@mastra/core 1.3.0 → 1.4.0

package/dist/chunk-NJ7TL3LQ.js

@@ -0,0 +1,1196 @@
+import { validateAndSaveScore } from './chunk-5Q5Y34SS.js';
+import { isSupportedLanguageModel } from './chunk-4XSAZPPS.js';
+import { MastraError } from './chunk-FJEVLHJT.js';
+import { isZodType } from '@mastra/schema-compat';
+import { zodToJsonSchema } from '@mastra/schema-compat/zod-to-json';
+
+// src/datasets/experiment/executor.ts
+async function executeScorer(scorer, item) {
+  try {
+    const result = await scorer.run(item.input);
+    const score = typeof result.score === "number" && !isNaN(result.score) ? result.score : null;
+    if (score === null && result.score !== void 0) {
+      console.warn(`Scorer ${scorer.id} returned invalid score: ${result.score}`);
+    }
+    return {
+      output: {
+        score,
+        reason: typeof result.reason === "string" ? result.reason : null
+      },
+      error: null,
+      traceId: null
+      // Scorers don't produce traces
+    };
+  } catch (error) {
+    return {
+      output: null,
+      error: {
+        message: error instanceof Error ? error.message : String(error),
+        stack: error instanceof Error ? error.stack : void 0
+      },
+      traceId: null
+    };
+  }
+}
+async function executeTarget(target, targetType, item, options) {
+  try {
+    const signal = options?.signal;
+    if (signal?.aborted) {
+      throw signal.reason ?? new DOMException("The operation was aborted.", "AbortError");
+    }
+    let executionPromise;
+    switch (targetType) {
+      case "agent":
+        executionPromise = executeAgent(target, item, signal);
+        break;
+      case "workflow":
+        executionPromise = executeWorkflow(target, item);
+        break;
+      case "scorer":
+        executionPromise = executeScorer(target, item);
+        break;
+      case "processor":
+        throw new Error(`Target type '${targetType}' not yet supported.`);
+      default:
+        throw new Error(`Unknown target type: ${targetType}`);
+    }
+    if (signal) {
+      return await raceWithSignal(executionPromise, signal);
+    }
+    return await executionPromise;
+  } catch (error) {
+    return {
+      output: null,
+      error: {
+        message: error instanceof Error ? error.message : String(error),
+        stack: error instanceof Error ? error.stack : void 0
+      },
+      traceId: null
+    };
+  }
+}
+function raceWithSignal(promise, signal) {
+  if (signal.aborted) {
+    return Promise.reject(signal.reason ?? new DOMException("The operation was aborted.", "AbortError"));
+  }
+  return new Promise((resolve, reject) => {
+    const onAbort = () => {
+      reject(signal.reason ?? new DOMException("The operation was aborted.", "AbortError"));
+    };
+    signal.addEventListener("abort", onAbort, { once: true });
+    promise.then(
+      (value) => {
+        signal.removeEventListener("abort", onAbort);
+        resolve(value);
+      },
+      (err) => {
+        signal.removeEventListener("abort", onAbort);
+        reject(err);
+      }
+    );
+  });
+}
+async function executeAgent(agent, item, signal) {
+  const model = await agent.getModel();
+  const result = isSupportedLanguageModel(model) ? await agent.generate(item.input, {
+    scorers: {},
+    returnScorerData: true,
+    abortSignal: signal
+  }) : await agent.generateLegacy?.(item.input, {
+    scorers: {},
+    returnScorerData: true
+  });
+  if (result == null) {
+    throw new Error(`Agent "${agent.name}" does not support generateLegacy for this model type`);
+  }
+  const traceId = result?.traceId ?? null;
+  const scoringData = result?.scoringData;
+  return {
+    output: result,
+    error: null,
+    traceId,
+    scorerInput: scoringData?.input,
+    scorerOutput: scoringData?.output
+  };
+}
+async function executeWorkflow(workflow, item) {
+  const run = await workflow.createRun({ disableScorers: true });
+  const result = await run.start({
+    inputData: item.input
+  });
+  const traceId = result?.traceId ?? null;
+  if (result.status === "success") {
+    return { output: result.result, error: null, traceId };
+  }
+  if (result.status === "failed") {
+    return {
+      output: null,
+      error: { message: result.error?.message ?? "Workflow failed", stack: result.error?.stack },
+      traceId
+    };
+  }
+  if (result.status === "tripwire") {
+    return {
+      output: null,
+      error: { message: `Workflow tripwire: ${result.tripwire?.reason ?? "Unknown reason"}` },
+      traceId
+    };
+  }
+  if (result.status === "suspended") {
+    return {
+      output: null,
+      error: { message: "Workflow suspended - not yet supported in dataset experiments" },
+      traceId
+    };
+  }
+  if (result.status === "paused") {
+    return { output: null, error: { message: "Workflow paused - not yet supported in dataset experiments" }, traceId };
+  }
+  const _exhaustiveCheck = result;
+  return {
+    output: null,
+    error: { message: `Workflow ended with unexpected status: ${_exhaustiveCheck.status}` },
+    traceId
+  };
+}
+
+// src/datasets/experiment/scorer.ts
+function resolveScorers(mastra, scorers) {
+  if (!scorers || scorers.length === 0) return [];
+  return scorers.map((scorer) => {
+    if (typeof scorer === "string") {
+      const resolved = mastra.getScorerById(scorer);
+      if (!resolved) {
+        console.warn(`Scorer not found: ${scorer}`);
+        return null;
+      }
+      return resolved;
+    }
+    return scorer;
+  }).filter((s) => s !== null);
+}
+async function runScorersForItem(scorers, item, output, storage, runId, targetType, targetId, itemId, scorerInput, scorerOutput) {
+  if (scorers.length === 0) return [];
+  const settled = await Promise.allSettled(
+    scorers.map(async (scorer) => {
+      const result = await runScorerSafe(scorer, item, output, scorerInput, scorerOutput);
+      if (storage && result.score !== null) {
+        try {
+          await validateAndSaveScore(storage, {
+            scorerId: scorer.id,
+            score: result.score,
+            reason: result.reason ?? void 0,
+            input: item.input,
+            output,
+            additionalContext: item.metadata,
+            entityType: targetType.toUpperCase(),
+            entityId: itemId,
+            source: "TEST",
+            runId,
+            scorer: {
+              id: scorer.id,
+              name: scorer.name,
+              description: scorer.description ?? ""
+            },
+            entity: {
+              id: targetId,
+              name: targetId
+            }
+          });
+        } catch (saveError) {
+          console.warn(`Failed to save score for scorer ${scorer.id}:`, saveError);
+        }
+      }
+      return result;
+    })
+  );
+  return settled.map(
+    (s, i) => s.status === "fulfilled" ? s.value : { scorerId: scorers[i].id, scorerName: scorers[i].name, score: null, reason: null, error: String(s.reason) }
+  );
+}
+async function runScorerSafe(scorer, item, output, scorerInput, scorerOutput) {
+  try {
+    const scoreResult = await scorer.run({
+      input: scorerInput ?? item.input,
+      output: scorerOutput ?? output,
+      groundTruth: item.groundTruth
+    });
+    const score = scoreResult.score;
+    const reason = scoreResult.reason;
+    return {
+      scorerId: scorer.id,
+      scorerName: scorer.name,
+      score: typeof score === "number" ? score : null,
+      reason: typeof reason === "string" ? reason : null,
+      error: null
+    };
+  } catch (error) {
+    return {
+      scorerId: scorer.id,
+      scorerName: scorer.name,
+      score: null,
+      reason: null,
+      error: error instanceof Error ? error.message : String(error)
+    };
+  }
+}
+
+// src/datasets/experiment/analytics/aggregate.ts
+function computeMean(values) {
+  if (values.length === 0) {
+    return 0;
+  }
+  const sum = values.reduce((acc, val) => acc + val, 0);
+  return sum / values.length;
+}
+function computeScorerStats(scores, passThreshold = 0.5) {
+  const totalItems = scores.length;
+  if (totalItems === 0) {
+    return {
+      errorRate: 0,
+      errorCount: 0,
+      passRate: 0,
+      passCount: 0,
+      avgScore: 0,
+      scoreCount: 0,
+      totalItems: 0
+    };
+  }
+  const validScores = [];
+  let errorCount = 0;
+  for (const score of scores) {
+    if (score.score === null || score.score === void 0) {
+      errorCount++;
+    } else {
+      validScores.push(score.score);
+    }
+  }
+  const scoreCount = validScores.length;
+  const errorRate = errorCount / totalItems;
+  const passCount = validScores.filter((s) => s >= passThreshold).length;
+  const passRate = scoreCount > 0 ? passCount / scoreCount : 0;
+  const avgScore = computeMean(validScores);
+  return {
+    errorRate,
+    errorCount,
+    passRate,
+    passCount,
+    avgScore,
+    scoreCount,
+    totalItems
+  };
+}
+function isRegression(delta, threshold, direction = "higher-is-better") {
+  if (direction === "higher-is-better") {
+    return delta < -threshold;
+  } else {
+    return delta > threshold;
+  }
+}
+
+// src/datasets/experiment/analytics/compare.ts
+var DEFAULT_THRESHOLD = {
+  value: 0,
+  direction: "higher-is-better"
+};
+var DEFAULT_PASS_THRESHOLD = 0.5;
+async function compareExperiments(mastra, config) {
+  const { experimentIdA, experimentIdB, thresholds = {} } = config;
+  const warnings = [];
+  const storage = mastra.getStorage();
+  if (!storage) {
+    throw new Error("Storage not configured. Configure storage in Mastra instance.");
+  }
+  const experimentsStore = await storage.getStore("experiments");
+  const scoresStore = await storage.getStore("scores");
+  if (!experimentsStore) {
+    throw new Error("ExperimentsStorage not configured.");
+  }
+  if (!scoresStore) {
+    throw new Error("ScoresStorage not configured.");
+  }
+  const [experimentA, experimentB] = await Promise.all([
+    experimentsStore.getExperimentById({ id: experimentIdA }),
+    experimentsStore.getExperimentById({ id: experimentIdB })
+  ]);
+  if (!experimentA) {
+    throw new Error(`Experiment not found: ${experimentIdA}`);
+  }
+  if (!experimentB) {
+    throw new Error(`Experiment not found: ${experimentIdB}`);
+  }
+  const versionMismatch = experimentA.datasetVersion !== experimentB.datasetVersion;
+  if (versionMismatch) {
+    warnings.push(
+      `Experiments have different dataset versions: ${experimentA.datasetVersion} vs ${experimentB.datasetVersion}`
+    );
+  }
+  const [resultsA, resultsB] = await Promise.all([
+    experimentsStore.listExperimentResults({ experimentId: experimentIdA, pagination: { page: 0, perPage: false } }),
+    experimentsStore.listExperimentResults({ experimentId: experimentIdB, pagination: { page: 0, perPage: false } })
+  ]);
+  const [scoresA, scoresB] = await Promise.all([
+    scoresStore.listScoresByRunId({ runId: experimentIdA, pagination: { page: 0, perPage: false } }),
+    scoresStore.listScoresByRunId({ runId: experimentIdB, pagination: { page: 0, perPage: false } })
+  ]);
+  if (resultsA.results.length === 0 && resultsB.results.length === 0) {
+    warnings.push("Both experiments have no results.");
+    return buildEmptyResult(experimentA, experimentB, versionMismatch, warnings);
+  }
+  if (resultsA.results.length === 0) {
+    warnings.push("Experiment A has no results.");
+  }
+  if (resultsB.results.length === 0) {
+    warnings.push("Experiment B has no results.");
+  }
+  const itemIdsA = new Set(resultsA.results.map((r) => r.itemId));
+  const itemIdsB = new Set(resultsB.results.map((r) => r.itemId));
+  const overlappingItemIds = [...itemIdsA].filter((id) => itemIdsB.has(id));
+  if (overlappingItemIds.length === 0) {
+    warnings.push("No overlapping items between experiments.");
+  }
+  const scoresMapA = groupScoresByScorerAndItem(scoresA.scores);
+  const scoresMapB = groupScoresByScorerAndItem(scoresB.scores);
+  const allScorerIds = /* @__PURE__ */ new Set([...Object.keys(scoresMapA), ...Object.keys(scoresMapB)]);
+  const scorers = {};
+  let hasRegression = false;
+  for (const scorerId of allScorerIds) {
+    const scorerScoresA = scoresMapA[scorerId] ?? {};
+    const scorerScoresB = scoresMapB[scorerId] ?? {};
+    const scoresArrayA = Object.values(scorerScoresA);
+    const scoresArrayB = Object.values(scorerScoresB);
+    const thresholdConfig = thresholds[scorerId] ?? DEFAULT_THRESHOLD;
+    const threshold = thresholdConfig.value;
+    const direction = thresholdConfig.direction ?? "higher-is-better";
+    const statsA = computeScorerStats(scoresArrayA, DEFAULT_PASS_THRESHOLD);
+    const statsB = computeScorerStats(scoresArrayB, DEFAULT_PASS_THRESHOLD);
+    const delta = statsB.avgScore - statsA.avgScore;
+    const regressed = isRegression(delta, threshold, direction);
+    if (regressed) {
+      hasRegression = true;
+    }
+    scorers[scorerId] = {
+      statsA,
+      statsB,
+      delta,
+      regressed,
+      threshold
+    };
+  }
+  const allItemIds = /* @__PURE__ */ new Set([...itemIdsA, ...itemIdsB]);
+  const items = [];
+  for (const itemId of allItemIds) {
+    const inBothExperiments = itemIdsA.has(itemId) && itemIdsB.has(itemId);
+    const itemScoresA = {};
+    const itemScoresB = {};
+    for (const scorerId of allScorerIds) {
+      const scoreA = scoresMapA[scorerId]?.[itemId];
+      const scoreB = scoresMapB[scorerId]?.[itemId];
+      itemScoresA[scorerId] = scoreA?.score ?? null;
+      itemScoresB[scorerId] = scoreB?.score ?? null;
+    }
+    items.push({
+      itemId,
+      inBothExperiments,
+      scoresA: itemScoresA,
+      scoresB: itemScoresB
+    });
+  }
+  return {
+    experimentA: {
+      id: experimentA.id,
+      datasetVersion: experimentA.datasetVersion
+    },
+    experimentB: {
+      id: experimentB.id,
+      datasetVersion: experimentB.datasetVersion
+    },
+    versionMismatch,
+    hasRegression,
+    scorers,
+    items,
+    warnings
+  };
+}
+function groupScoresByScorerAndItem(scores) {
+  const result = {};
+  for (const score of scores) {
+    const scorerId = score.scorerId;
+    const itemId = score.entityId;
+    if (!result[scorerId]) {
+      result[scorerId] = {};
+    }
+    result[scorerId][itemId] = score;
+  }
+  return result;
+}
+function buildEmptyResult(experimentA, experimentB, versionMismatch, warnings) {
+  return {
+    experimentA: {
+      id: experimentA.id,
+      datasetVersion: experimentA.datasetVersion
+    },
+    experimentB: {
+      id: experimentB.id,
+      datasetVersion: experimentB.datasetVersion
+    },
+    versionMismatch,
+    hasRegression: false,
+    scorers: {},
+    items: [],
+    warnings
+  };
+}
+
+// src/datasets/experiment/index.ts
+async function runExperiment(mastra, config) {
+  const {
+    datasetId,
+    targetType,
+    targetId,
+    scorers: scorerInput,
+    version,
+    maxConcurrency = 5,
+    signal,
+    itemTimeout,
+    maxRetries = 0,
+    experimentId: providedExperimentId,
+    name,
+    description,
+    metadata
+  } = config;
+  const startedAt = /* @__PURE__ */ new Date();
+  const experimentId = providedExperimentId ?? crypto.randomUUID();
+  const storage = mastra.getStorage();
+  const datasetsStore = await storage?.getStore("datasets");
+  const experimentsStore = await storage?.getStore("experiments");
+  let items;
+  let datasetVersion;
+  if (config.data) {
+    const rawData = typeof config.data === "function" ? await config.data() : config.data;
+    items = rawData.map((dataItem) => {
+      const id = dataItem.id ?? crypto.randomUUID();
+      return {
+        id,
+        datasetVersion: null,
+        input: dataItem.input,
+        groundTruth: dataItem.groundTruth,
+        metadata: dataItem.metadata
+      };
+    });
+    datasetVersion = null;
+  } else if (datasetId) {
+    if (!datasetsStore) {
+      throw new Error("DatasetsStorage not configured. Configure storage in Mastra instance.");
+    }
+    const dataset = await datasetsStore.getDatasetById({ id: datasetId });
+    if (!dataset) {
+      throw new Error(`Dataset not found: ${datasetId}`);
+    }
+    datasetVersion = version ?? dataset.version;
+    const versionItems = await datasetsStore.getItemsByVersion({
+      datasetId,
+      version: datasetVersion
+    });
+    if (versionItems.length === 0) {
+      throw new Error(`No items in dataset ${datasetId} at version ${datasetVersion}`);
+    }
+    items = versionItems.map((v) => ({
+      id: v.id,
+      datasetVersion: v.datasetVersion,
+      input: v.input,
+      groundTruth: v.groundTruth,
+      metadata: v.metadata
+    }));
+  } else {
+    throw new Error("No data source: provide datasetId or data");
+  }
+  let execFn;
+  if (config.task) {
+    const taskFn = config.task;
+    execFn = async (item, itemSignal) => {
+      try {
+        const result = await taskFn({
+          input: item.input,
+          mastra,
+          groundTruth: item.groundTruth,
+          metadata: item.metadata,
+          signal: itemSignal
+        });
+        return { output: result, error: null, traceId: null };
+      } catch (err) {
+        return {
+          output: null,
+          error: {
+            message: err instanceof Error ? err.message : String(err),
+            stack: err instanceof Error ? err.stack : void 0
+          },
+          traceId: null
+        };
+      }
+    };
+  } else if (targetType && targetId) {
+    const target = resolveTarget(mastra, targetType, targetId);
+    if (!target) {
+      throw new Error(`Target not found: ${targetType}/${targetId}`);
+    }
+    execFn = (item, itemSignal) => executeTarget(target, targetType, item, { signal: itemSignal });
+  } else {
+    throw new Error("No task: provide targetType+targetId or task");
+  }
+  const scorers = resolveScorers(mastra, scorerInput);
+  if (experimentsStore) {
+    if (!providedExperimentId) {
+      await experimentsStore.createExperiment({
+        id: experimentId,
+        name,
+        description,
+        metadata,
+        datasetId: datasetId ?? null,
+        datasetVersion,
+        targetType: targetType ?? "agent",
+        targetId: targetId ?? "inline",
+        totalItems: items.length
+      });
+    }
+    await experimentsStore.updateExperiment({
+      id: experimentId,
+      status: "running",
+      startedAt
+    });
+  }
+  let succeededCount = 0;
+  let failedCount = 0;
+  const results = new Array(items.length);
+  const PROGRESS_UPDATE_INTERVAL = 2e3;
+  let lastProgressUpdate = 0;
+  try {
+    const pMap = (await import('p-map')).default;
+    await pMap(
+      items.map((item, idx) => ({ item, idx })),
+      async ({ item, idx }) => {
+        if (signal?.aborted) {
+          throw new DOMException("Aborted", "AbortError");
+        }
+        const itemStartedAt = /* @__PURE__ */ new Date();
+        let itemSignal = signal;
+        if (itemTimeout) {
+          const timeoutSignal = AbortSignal.timeout(itemTimeout);
+          itemSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
+        }
+        let retryCount = 0;
+        let execResult = await execFn(item, itemSignal);
+        while (execResult.error && retryCount < maxRetries) {
+          if (execResult.error.message.toLowerCase().includes("abort")) break;
+          retryCount++;
+          const delay = Math.min(1e3 * Math.pow(2, retryCount - 1), 3e4);
+          const jitter = delay * 0.2 * Math.random();
+          await new Promise((r) => setTimeout(r, delay + jitter));
+          if (signal?.aborted) {
+            throw new DOMException("Aborted", "AbortError");
+          }
+          execResult = await execFn(item, itemSignal);
+        }
+        const itemCompletedAt = /* @__PURE__ */ new Date();
+        if (execResult.error) {
+          failedCount++;
+        } else {
+          succeededCount++;
+        }
+        const itemResult = {
+          itemId: item.id,
+          itemVersion: item.datasetVersion ?? 0,
+          input: item.input,
+          output: execResult.output,
+          groundTruth: item.groundTruth ?? null,
+          error: execResult.error,
+          startedAt: itemStartedAt,
+          completedAt: itemCompletedAt,
+          retryCount
+        };
+        const itemScores = await runScorersForItem(
+          scorers,
+          item,
+          execResult.output,
+          storage ?? null,
+          experimentId,
+          targetType ?? "agent",
+          targetId ?? "inline",
+          item.id,
+          execResult.scorerInput,
+          execResult.scorerOutput
+        );
+        if (experimentsStore) {
+          try {
+            await experimentsStore.addExperimentResult({
+              experimentId,
+              itemId: item.id,
+              itemDatasetVersion: item.datasetVersion,
+              input: item.input,
+              output: execResult.output,
+              groundTruth: item.groundTruth ?? null,
+              error: execResult.error,
+              startedAt: itemStartedAt,
+              completedAt: itemCompletedAt,
+              retryCount,
+              traceId: execResult.traceId
+            });
+          } catch (persistError) {
+            console.warn(`Failed to persist result for item ${item.id}:`, persistError);
+          }
+          const now = Date.now();
+          if (now - lastProgressUpdate >= PROGRESS_UPDATE_INTERVAL) {
+            lastProgressUpdate = now;
+            try {
+              await experimentsStore.updateExperiment({
+                id: experimentId,
+                succeededCount,
+                failedCount
+              });
+            } catch {
+            }
+          }
+        }
+        results[idx] = {
+          ...itemResult,
+          scores: itemScores
+        };
+      },
+      { concurrency: maxConcurrency }
+    );
+  } catch {
+    const completedAt2 = /* @__PURE__ */ new Date();
+    const skippedCount2 = items.length - succeededCount - failedCount;
+    if (experimentsStore) {
+      await experimentsStore.updateExperiment({
+        id: experimentId,
+        status: "failed",
+        succeededCount,
+        failedCount,
+        skippedCount: skippedCount2,
+        completedAt: completedAt2
+      });
+    }
+    return {
+      experimentId,
+      status: "failed",
+      totalItems: items.length,
+      succeededCount,
+      failedCount,
+      skippedCount: skippedCount2,
+      completedWithErrors: false,
+      startedAt,
+      completedAt: completedAt2,
+      results: results.filter(Boolean)
+    };
+  }
+  const completedAt = /* @__PURE__ */ new Date();
+  const status = failedCount === items.length ? "failed" : "completed";
+  const completedWithErrors = status === "completed" && failedCount > 0;
+  const skippedCount = items.length - succeededCount - failedCount;
+  if (experimentsStore) {
+    await experimentsStore.updateExperiment({
+      id: experimentId,
+      status,
+      succeededCount,
+      failedCount,
+      skippedCount,
+      completedAt
+    });
+  }
+  return {
+    experimentId,
+    status,
+    totalItems: items.length,
+    succeededCount,
+    failedCount,
+    skippedCount,
+    completedWithErrors,
+    startedAt,
+    completedAt,
+    results
+  };
+}
+function resolveTarget(mastra, targetType, targetId) {
+  switch (targetType) {
+    case "agent":
+      try {
+        return mastra.getAgentById(targetId);
+      } catch {
+        try {
+          return mastra.getAgent(targetId);
+        } catch {
+          return null;
+        }
+      }
+    case "workflow":
+      try {
+        return mastra.getWorkflowById(targetId);
+      } catch {
+        try {
+          return mastra.getWorkflow(targetId);
+        } catch {
+          return null;
+        }
+      }
+    case "scorer":
+      try {
+        return mastra.getScorerById(targetId) ?? null;
+      } catch {
+        return null;
+      }
+    case "processor":
+      return null;
+    default:
+      return null;
+  }
+}
+var Dataset = class {
+  id;
+  #mastra;
+  #datasetsStore;
+  #experimentsStore;
+  constructor(id, mastra) {
+    this.id = id;
+    this.#mastra = mastra;
+  }
+  // ---------------------------------------------------------------------------
+  // Lazy storage resolution
+  // ---------------------------------------------------------------------------
+  async #getDatasetsStore() {
+    if (this.#datasetsStore) return this.#datasetsStore;
+    const storage = this.#mastra.getStorage();
+    if (!storage) {
+      throw new MastraError({
+        id: "DATASETS_STORAGE_NOT_CONFIGURED",
+        text: "Storage not configured. Configure storage in Mastra instance.",
+        domain: "STORAGE",
+        category: "USER"
+      });
+    }
+    const store = await storage.getStore("datasets");
+    if (!store) {
+      throw new MastraError({
+        id: "DATASETS_STORE_NOT_AVAILABLE",
+        text: "Datasets store not available. Ensure your storage adapter provides a datasets domain.",
+        domain: "STORAGE",
+        category: "USER"
+      });
+    }
+    this.#datasetsStore = store;
+    return store;
+  }
+  async #getExperimentsStore() {
+    if (this.#experimentsStore) return this.#experimentsStore;
+    const storage = this.#mastra.getStorage();
+    if (!storage) {
+      throw new MastraError({
+        id: "DATASETS_STORAGE_NOT_CONFIGURED",
+        text: "Storage not configured. Configure storage in Mastra instance.",
+        domain: "STORAGE",
+        category: "USER"
+      });
+    }
+    const store = await storage.getStore("experiments");
+    if (!store) {
+      throw new MastraError({
+        id: "EXPERIMENTS_STORE_NOT_AVAILABLE",
+        text: "Experiments store not available. Ensure your storage adapter provides an experiments domain.",
+        domain: "STORAGE",
+        category: "USER"
+      });
+    }
+    this.#experimentsStore = store;
+    return store;
+  }
+  // ---------------------------------------------------------------------------
+  // Dataset metadata
+  // ---------------------------------------------------------------------------
+  /**
+   * Get the full dataset record from storage.
+   */
+  async getDetails() {
+    const store = await this.#getDatasetsStore();
+    const record = await store.getDatasetById({ id: this.id });
+    if (!record) {
+      throw new MastraError({
+        id: "DATASET_NOT_FOUND",
+        text: `Dataset not found: ${this.id}`,
+        domain: "STORAGE",
+        category: "USER"
+      });
+    }
+    return record;
+  }
+  /**
+   * Update dataset metadata and/or schemas.
+   * Zod schemas are automatically converted to JSON Schema.
+   */
+  async update(input) {
+    const store = await this.#getDatasetsStore();
+    let { inputSchema, groundTruthSchema, ...rest } = input;
+    if (inputSchema !== void 0 && isZodType(inputSchema)) {
+      inputSchema = zodToJsonSchema(inputSchema);
+    }
+    if (groundTruthSchema !== void 0 && isZodType(groundTruthSchema)) {
+      groundTruthSchema = zodToJsonSchema(groundTruthSchema);
+    }
+    return store.updateDataset({
+      id: this.id,
+      ...rest,
+      inputSchema,
+      groundTruthSchema
+    });
+  }
+  // ---------------------------------------------------------------------------
+  // Item CRUD
+  // ---------------------------------------------------------------------------
+  /**
+   * Add a single item to the dataset.
+   */
+  async addItem(input) {
+    const store = await this.#getDatasetsStore();
+    return store.addItem({
+      datasetId: this.id,
+      input: input.input,
+      groundTruth: input.groundTruth,
+      metadata: input.metadata
+    });
+  }
+  /**
+   * Add multiple items to the dataset in bulk.
+   */
+  async addItems(input) {
+    const store = await this.#getDatasetsStore();
+    return store.batchInsertItems({
+      datasetId: this.id,
+      items: input.items
+    });
+  }
+  /**
+   * Get a single item by ID, optionally at a specific version.
+   */
+  async getItem(args) {
+    const store = await this.#getDatasetsStore();
+    return store.getItemById({ id: args.itemId, datasetVersion: args.version });
+  }
+  /**
+   * List items in the dataset, optionally at a specific version.
+   */
+  async listItems(args) {
+    const store = await this.#getDatasetsStore();
+    if (args?.version) {
+      return store.getItemsByVersion({ datasetId: this.id, version: args.version });
+    }
+    return store.listItems({
+      datasetId: this.id,
+      search: args?.search,
+      pagination: { page: args?.page ?? 0, perPage: args?.perPage ?? 20 }
+    });
+  }
+  /**
+   * Update an existing item in the dataset.
+   */
+  async updateItem(input) {
+    const store = await this.#getDatasetsStore();
+    return store.updateItem({
+      id: input.itemId,
+      datasetId: this.id,
+      input: input.input,
+      groundTruth: input.groundTruth,
+      metadata: input.metadata
+    });
+  }
+  /**
+   * Delete a single item from the dataset.
+   */
+  async deleteItem(args) {
+    const store = await this.#getDatasetsStore();
+    return store.deleteItem({ id: args.itemId, datasetId: this.id });
+  }
+  /**
+   * Delete multiple items from the dataset in bulk.
+   */
+  async deleteItems(args) {
+    const store = await this.#getDatasetsStore();
+    return store.batchDeleteItems({ datasetId: this.id, itemIds: args.itemIds });
+  }
+  // ---------------------------------------------------------------------------
+  // Versioning
+  // ---------------------------------------------------------------------------
+  /**
+   * List all versions of this dataset.
+   */
+  async listVersions(args) {
+    const store = await this.#getDatasetsStore();
+    return store.listDatasetVersions({
+      datasetId: this.id,
+      pagination: { page: args?.page ?? 0, perPage: args?.perPage ?? 20 }
+    });
+  }
+  /**
+   * Get full SCD-2 history of a specific item across all dataset versions.
+   */
+  async getItemHistory(args) {
+    const store = await this.#getDatasetsStore();
+    return store.getItemHistory(args.itemId);
+  }
+  // ---------------------------------------------------------------------------
+  // Experiments
+  // ---------------------------------------------------------------------------
+  /**
+   * Run an experiment on this dataset and wait for completion.
+   */
+  async startExperiment(config) {
+    return runExperiment(this.#mastra, { datasetId: this.id, ...config });
+  }
+  /**
+   * Start an experiment asynchronously (fire-and-forget).
+   * Returns immediately with the experiment ID and pending status.
+   */
+  async startExperimentAsync(config) {
+    const experimentsStore = await this.#getExperimentsStore();
+    const datasetsStore = await this.#getDatasetsStore();
+    const dataset = await datasetsStore.getDatasetById({ id: this.id });
+    if (!dataset) {
+      throw new MastraError({
+        id: "DATASET_NOT_FOUND",
+        text: `Dataset not found: ${this.id}`,
+        domain: "STORAGE",
+        category: "USER"
+      });
+    }
+    const run = await experimentsStore.createExperiment({
+      datasetId: this.id,
+      datasetVersion: dataset.version,
+      targetType: config.targetType ?? "agent",
+      targetId: config.targetId ?? "inline",
+      totalItems: 0,
+      name: config.name,
+      description: config.description,
+      metadata: config.metadata
+    });
+    const experimentId = run.id;
+    void runExperiment(this.#mastra, {
+      datasetId: this.id,
+      experimentId,
+      ...config
+    }).catch(() => {
+    });
+    return { experimentId, status: "pending" };
+  }
+  /**
+   * List all experiments (runs) for this dataset.
+   */
+  async listExperiments(args) {
+    const experimentsStore = await this.#getExperimentsStore();
+    return experimentsStore.listExperiments({
+      datasetId: this.id,
+      pagination: { page: args?.page ?? 0, perPage: args?.perPage ?? 20 }
+    });
+  }
+  /**
+   * Get a specific experiment (run) by ID.
+   */
+  async getExperiment(args) {
+    const experimentsStore = await this.#getExperimentsStore();
+    return experimentsStore.getExperimentById({ id: args.experimentId });
+  }
+  /**
+   * List results for a specific experiment.
+   */
+  async listExperimentResults(args) {
+    const experimentsStore = await this.#getExperimentsStore();
+    return experimentsStore.listExperimentResults({
+      experimentId: args.experimentId,
+      pagination: { page: args?.page ?? 0, perPage: args?.perPage ?? 20 }
+    });
+  }
+  /**
+   * Delete an experiment (run) by ID.
+   */
+  async deleteExperiment(args) {
+    const experimentsStore = await this.#getExperimentsStore();
+    return experimentsStore.deleteExperiment({ id: args.experimentId });
+  }
+};
+var DatasetsManager = class {
+  #mastra;
+  #datasetsStore;
+  #experimentsStore;
+  constructor(mastra) {
+    this.#mastra = mastra;
+  }
+  // ---------------------------------------------------------------------------
+  // Lazy storage resolution
+  // ---------------------------------------------------------------------------
+  async #getDatasetsStore() {
+    if (this.#datasetsStore) return this.#datasetsStore;
+    const storage = this.#mastra.getStorage();
+    if (!storage) {
+      throw new MastraError({
+        id: "DATASETS_STORAGE_NOT_CONFIGURED",
+        text: "Storage not configured. Configure storage in Mastra instance.",
+        domain: "STORAGE",
+        category: "USER"
+      });
+    }
+    const store = await storage.getStore("datasets");
+    if (!store) {
+      throw new MastraError({
+        id: "DATASETS_STORE_NOT_AVAILABLE",
+        text: "Datasets store not available. Ensure your storage adapter provides a datasets domain.",
+        domain: "STORAGE",
+        category: "USER"
+      });
+    }
+    this.#datasetsStore = store;
+    return store;
+  }
+  async #getExperimentsStore() {
+    if (this.#experimentsStore) return this.#experimentsStore;
+    const storage = this.#mastra.getStorage();
+    if (!storage) {
+      throw new MastraError({
+        id: "DATASETS_STORAGE_NOT_CONFIGURED",
+        text: "Storage not configured. Configure storage in Mastra instance.",
+        domain: "STORAGE",
+        category: "USER"
+      });
+    }
+    const store = await storage.getStore("experiments");
+    if (!store) {
+      throw new MastraError({
+        id: "EXPERIMENTS_STORE_NOT_AVAILABLE",
+        text: "Experiments store not available. Ensure your storage adapter provides an experiments domain.",
+        domain: "STORAGE",
+        category: "USER"
+      });
+    }
+    this.#experimentsStore = store;
+    return store;
+  }
+  // ---------------------------------------------------------------------------
+  // Dataset CRUD
+  // ---------------------------------------------------------------------------
+  /**
+   * Create a new dataset.
+   * Zod schemas are automatically converted to JSON Schema.
+   */
+  async create(input) {
+    const store = await this.#getDatasetsStore();
+    let { inputSchema, groundTruthSchema, ...rest } = input;
+    if (inputSchema !== void 0 && isZodType(inputSchema)) {
+      inputSchema = zodToJsonSchema(inputSchema);
+    }
+    if (groundTruthSchema !== void 0 && isZodType(groundTruthSchema)) {
+      groundTruthSchema = zodToJsonSchema(groundTruthSchema);
+    }
+    const result = await store.createDataset({
+      ...rest,
+      inputSchema,
+      groundTruthSchema
+    });
+    return new Dataset(result.id, this.#mastra);
+  }
+  /**
+   * Get an existing dataset by ID.
+   * Throws if the dataset does not exist.
+   */
+  async get(args) {
+    const store = await this.#getDatasetsStore();
+    const record = await store.getDatasetById({ id: args.id });
+    if (!record) {
+      throw new MastraError({
+        id: "DATASET_NOT_FOUND",
+        text: "Dataset not found",
+        domain: "STORAGE",
+        category: "USER"
+      });
+    }
+    return new Dataset(args.id, this.#mastra);
+  }
+  /**
+   * List all datasets with pagination.
+   */
+  async list(args) {
+    const store = await this.#getDatasetsStore();
+    return store.listDatasets({
+      pagination: { page: args?.page ?? 0, perPage: args?.perPage ?? 20 }
+    });
+  }
+  /**
+   * Delete a dataset by ID.
+   */
+  async delete(args) {
+    const store = await this.#getDatasetsStore();
+    return store.deleteDataset({ id: args.id });
+  }
+  // ---------------------------------------------------------------------------
+  // Cross-dataset experiment operations
+  // ---------------------------------------------------------------------------
+  /**
+   * Get a specific experiment (run) by ID.
+   */
+  async getExperiment(args) {
+    const experimentsStore = await this.#getExperimentsStore();
+    return experimentsStore.getExperimentById({ id: args.experimentId });
+  }
+  /**
+   * Compare two or more experiments.
+   *
+   * Uses the internal `compareExperiments` function for pairwise comparison,
+   * then enriches results with per-item input/groundTruth/output data.
+   */
+  async compareExperiments(args) {
+    const { experimentIds, baselineId } = args;
+    if (experimentIds.length < 2) {
+      throw new MastraError({
+        id: "COMPARE_INVALID_INPUT",
+        text: "compareExperiments requires at least 2 experiment IDs.",
+        domain: "STORAGE",
+        category: "USER"
+      });
+    }
+    const resolvedBaseline = baselineId ?? experimentIds[0];
+    const otherExperimentId = experimentIds.find((id) => id !== resolvedBaseline) ?? experimentIds[1];
+    const internal = await compareExperiments(this.#mastra, {
+      experimentIdA: resolvedBaseline,
+      experimentIdB: otherExperimentId
+    });
+    const experimentsStore = await this.#getExperimentsStore();
+    const [resultsA, resultsB] = await Promise.all([
+      experimentsStore.listExperimentResults({
+        experimentId: resolvedBaseline,
+        pagination: { page: 0, perPage: false }
+      }),
+      experimentsStore.listExperimentResults({
+        experimentId: otherExperimentId,
+        pagination: { page: 0, perPage: false }
+      })
+    ]);
+    const resultsMapA = new Map(resultsA.results.map((r) => [r.itemId, r]));
+    const resultsMapB = new Map(resultsB.results.map((r) => [r.itemId, r]));
+    const items = internal.items.map((item) => {
+      const resultA = resultsMapA.get(item.itemId);
+      const resultB = resultsMapB.get(item.itemId);
+      return {
+        itemId: item.itemId,
+        input: resultA?.input ?? resultB?.input ?? null,
+        groundTruth: resultA?.groundTruth ?? resultB?.groundTruth ?? null,
+        results: {
+          [resolvedBaseline]: resultA ? { output: resultA.output, scores: item.scoresA } : null,
+          [otherExperimentId]: resultB ? { output: resultB.output, scores: item.scoresB } : null
+        }
+      };
+    });
+    return {
+      baselineId: resolvedBaseline,
+      items
+    };
+  }
+};
+
+export { Dataset, DatasetsManager, compareExperiments, computeMean, computeScorerStats, executeTarget, isRegression, resolveScorers, runExperiment, runScorersForItem };
+//# sourceMappingURL=chunk-NJ7TL3LQ.js.map
+//# sourceMappingURL=chunk-NJ7TL3LQ.js.map