@mastra/core 1.3.0 → 1.4.0

package/dist/datasets/dataset.d.ts

@@ -0,0 +1,153 @@
+import type { Mastra } from '../mastra/index.js';
+import type { DatasetRecord, DatasetItem, DatasetItemRow, DatasetVersion } from '../storage/types.js';
+import type { StartExperimentConfig, ExperimentSummary } from './experiment/types.js';
+/**
+ * Public API for interacting with a single dataset.
+ *
+ * Provides methods for item CRUD, versioning, and experiment management.
+ * Obtained via `DatasetsManager.get()` or `DatasetsManager.create()`.
+ */
+export declare class Dataset {
+    #private;
+    readonly id: string;
+    constructor(id: string, mastra: Mastra);
+    /**
+     * Get the full dataset record from storage.
+     */
+    getDetails(): Promise<DatasetRecord>;
+    /**
+     * Update dataset metadata and/or schemas.
+     * Zod schemas are automatically converted to JSON Schema.
+     */
+    update(input: {
+        name?: string;
+        description?: string;
+        metadata?: Record<string, unknown>;
+        inputSchema?: unknown;
+        groundTruthSchema?: unknown;
+    }): Promise<DatasetRecord>;
+    /**
+     * Add a single item to the dataset.
+     */
+    addItem(input: {
+        input: unknown;
+        groundTruth?: unknown;
+        metadata?: Record<string, unknown>;
+    }): Promise<DatasetItem>;
+    /**
+     * Add multiple items to the dataset in bulk.
+     */
+    addItems(input: {
+        items: Array<{
+            input: unknown;
+            groundTruth?: unknown;
+            metadata?: Record<string, unknown>;
+        }>;
+    }): Promise<DatasetItem[]>;
+    /**
+     * Get a single item by ID, optionally at a specific version.
+     */
+    getItem(args: {
+        itemId: string;
+        version?: number;
+    }): Promise<DatasetItem | null>;
+    /**
+     * List items in the dataset, optionally at a specific version.
+     */
+    listItems(args?: {
+        version?: number;
+        page?: number;
+        perPage?: number;
+        search?: string;
+    }): Promise<DatasetItem[] | {
+        items: DatasetItem[];
+        pagination: {
+            total: number;
+            page: number;
+            perPage: number | false;
+            hasMore: boolean;
+        };
+    }>;
+    /**
+     * Update an existing item in the dataset.
+     */
+    updateItem(input: {
+        itemId: string;
+        input?: unknown;
+        groundTruth?: unknown;
+        metadata?: Record<string, unknown>;
+    }): Promise<DatasetItem>;
+    /**
+     * Delete a single item from the dataset.
+     */
+    deleteItem(args: {
+        itemId: string;
+    }): Promise<void>;
+    /**
+     * Delete multiple items from the dataset in bulk.
+     */
+    deleteItems(args: {
+        itemIds: string[];
+    }): Promise<void>;
+    /**
+     * List all versions of this dataset.
+     */
+    listVersions(args?: {
+        page?: number;
+        perPage?: number;
+    }): Promise<{
+        versions: DatasetVersion[];
+        pagination: {
+            total: number;
+            page: number;
+            perPage: number | false;
+            hasMore: boolean;
+        };
+    }>;
+    /**
+     * Get full SCD-2 history of a specific item across all dataset versions.
+     */
+    getItemHistory(args: {
+        itemId: string;
+    }): Promise<DatasetItemRow[]>;
+    /**
+     * Run an experiment on this dataset and wait for completion.
+     */
+    startExperiment<I = unknown, O = unknown, E = unknown>(config: StartExperimentConfig<I, O, E>): Promise<ExperimentSummary>;
+    /**
+     * Start an experiment asynchronously (fire-and-forget).
+     * Returns immediately with the experiment ID and pending status.
+     */
+    startExperimentAsync<I = unknown, O = unknown, E = unknown>(config: StartExperimentConfig<I, O, E>): Promise<{
+        experimentId: string;
+        status: 'pending';
+    }>;
+    /**
+     * List all experiments (runs) for this dataset.
+     */
+    listExperiments(args?: {
+        page?: number;
+        perPage?: number;
+    }): Promise<import("../storage/types.js").ListExperimentsOutput>;
+    /**
+     * Get a specific experiment (run) by ID.
+     */
+    getExperiment(args: {
+        experimentId: string;
+    }): Promise<import("../storage/types.js").Experiment | null>;
+    /**
+     * List results for a specific experiment.
+     */
+    listExperimentResults(args: {
+        experimentId: string;
+        page?: number;
+        perPage?: number;
+    }): Promise<import("../storage/types.js").ListExperimentResultsOutput>;
+    /**
+     * Delete an experiment (run) by ID.
+     */
+    deleteExperiment(args: {
+        experimentId: string;
+    }): Promise<void>;
+}
+//# sourceMappingURL=dataset.d.ts.map

package/dist/datasets/dataset.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/datasets/dataset.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAEtG,OAAO,KAAK,EAAoB,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAExG;;;;;GAKG;AACH,qBAAa,OAAO;;IAClB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAKR,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IAmEtC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,aAAa,CAAC;IAc1C;;;OAGG;IACG,MAAM,CAAC,KAAK,EAAE;QAClB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACnC,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,iBAAiB,CAAC,EAAE,OAAO,CAAC;KAC7B,GAAG,OAAO,CAAC,aAAa,CAAC;IAwB1B;;OAEG;IACG,OAAO,CAAC,KAAK,EAAE;QACnB,KAAK,EAAE,OAAO,CAAC;QACf,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,GAAG,OAAO,CAAC,WAAW,CAAC;IAUxB;;OAEG;IACG,QAAQ,CAAC,KAAK,EAAE;QACpB,KAAK,EAAE,KAAK,CAAC;YACX,KAAK,EAAE,OAAO,CAAC;YACf,WAAW,CAAC,EAAE,OAAO,CAAC;YACtB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACpC,CAAC,CAAC;KACJ,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAQ1B;;OAEG;IACG,OAAO,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAKtF;;OAEG;IACG,SAAS,CAAC,IAAI,CAAC,EAAE;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,GAAG,OAAO,CACP,WAAW,EAAE,GACb;QAAE,KAAK,EAAE,WAAW,EAAE,CAAC;QAAC,UAAU,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,GAAG,KAAK,CAAC;YAAC,OAAO,EAAE,OAAO,CAAA;SAAE,CAAA;KAAE,CACnH;IAYD;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE;QACtB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,GAAG,OAAO,CAAC,WAAW,CAAC;IAWxB;;OAEG;IACG,UAAU,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAKzD;;OAEG;IACG,WAAW,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAS7D;;OAEG;IACG,YAAY,CAAC,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC;QACtE,QAAQ,EAAE,cAAc,EAAE,CAAC;QAC3B,UAAU,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,GAAG,KAAK,CAAC;YAAC,OAAO,EAAE,OAAO,CAAA;SAAE,CAAC;KACxF,CAAC;IAQF;;OAEG;IACG,cAAc,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IASzE;;OAEG;IACG,eAAe,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,EACzD,MAAM,EAAE,qBAAqB,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GACrC,OAAO,CAAC,iBAAiB,CAAC;IAI7B;;;OAGG;IACG,oBAAoB,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,EAC9D,MAAM,EAAE,qBAAqB,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GACrC,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,SAAS,CAAA;KAAE,CAAC;IAqCvD;;OAEG;IACG,eAAe,CAAC,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE;IAQhE;;OAEG;IACG,aAAa,CAAC,IAAI,EAAE;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE;IAKlD;;OAEG;IACG,qBAAqB,CAAC,IAAI,EAAE;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE;IAQ3F;;OAEG;IACG,gBAAgB,CAAC,IAAI,EAAE;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE;CAItD"}

package/dist/datasets/experiment/analytics/aggregate.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Experiment Analytics Aggregation Helpers
+ *
+ * Pure functions for computing statistics from raw score data.
+ * Used by compareExperiments to build ScorerStats and detect regressions.
+ */
+import type { ScoreRowData } from '../../../evals/types.js';
+import type { ScorerStats } from './types.js';
+/**
+ * Compute the arithmetic mean of an array of numbers.
+ *
+ * @param values - Array of numbers to average
+ * @returns Mean value, or 0 if array is empty
+ */
+export declare function computeMean(values: number[]): number;
+/**
+ * Compute aggregate statistics for a set of scores.
+ *
+ * Metrics:
+ * - errorRate: proportion of items with null scores (errors)
+ * - passRate: proportion of scored items meeting threshold
+ * - avgScore: mean of non-null scores
+ *
+ * @param scores - Score records from storage
+ * @param passThreshold - Absolute threshold for pass (score >= threshold)
+ * @returns ScorerStats with all computed metrics
+ */
+export declare function computeScorerStats(scores: ScoreRowData[], passThreshold?: number): ScorerStats;
+/**
+ * Determine if a score delta represents a regression.
+ *
+ * @param delta - Score difference (experiment B - experiment A)
+ * @param threshold - Absolute threshold for regression detection
+ * @param direction - Score direction ('higher-is-better' or 'lower-is-better')
+ * @returns True if delta represents a regression
+ *
+ * @example
+ * // Higher is better (default): negative delta is bad
+ * isRegression(-0.1, 0.05, 'higher-is-better') // true (dropped more than 0.05)
+ * isRegression(-0.01, 0.05, 'higher-is-better') // false (within tolerance)
+ *
+ * // Lower is better: positive delta is bad
+ * isRegression(0.1, 0.05, 'lower-is-better') // true (increased more than 0.05)
+ */
+export declare function isRegression(delta: number, threshold: number, direction?: 'higher-is-better' | 'lower-is-better'): boolean;
+//# sourceMappingURL=aggregate.d.ts.map

package/dist/datasets/experiment/analytics/aggregate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"aggregate.d.ts","sourceRoot":"","sources":["../../../../src/datasets/experiment/analytics/aggregate.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAMpD;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,YAAY,EAAE,EAAE,aAAa,GAAE,MAAY,GAAG,WAAW,CA8CnG;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,SAAS,GAAE,kBAAkB,GAAG,iBAAsC,GACrE,OAAO,CAUT"}

package/dist/datasets/experiment/analytics/compare.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Experiment Comparison
+ *
+ * Compare two experiments to detect score regressions.
+ * Returns per-scorer deltas and per-item score diffs.
+ */
+import type { Mastra } from '../../../mastra/index.js';
+import type { CompareExperimentsConfig, ComparisonResult } from './types.js';
+/**
+ * Compare two experiments to detect score regressions.
+ *
+ * @param mastra - Mastra instance for storage access
+ * @param config - Comparison configuration
+ * @returns ComparisonResult with per-scorer and per-item comparisons
+ *
+ * @example
+ * ```typescript
+ * const result = await compareExperiments(mastra, {
+ *   experimentIdA: 'baseline-experiment-id',
+ *   experimentIdB: 'candidate-experiment-id',
+ *   thresholds: {
+ *     'accuracy': { value: 0.05, direction: 'higher-is-better' },
+ *     'latency': { value: 100, direction: 'lower-is-better' },
+ *   },
+ * });
+ *
+ * if (result.hasRegression) {
+ *   console.log('Quality regression detected!');
+ * }
+ * ```
+ */
+export declare function compareExperiments(mastra: Mastra, config: CompareExperimentsConfig): Promise<ComparisonResult>;
+//# sourceMappingURL=compare.d.ts.map

package/dist/datasets/experiment/analytics/compare.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../../../../src/datasets/experiment/analytics/compare.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,KAAK,EACV,wBAAwB,EACxB,gBAAgB,EAIjB,MAAM,SAAS,CAAC;AAejB;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,wBAAwB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAiKpH"}

package/dist/datasets/experiment/analytics/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Experiment Analytics
+ *
+ * Compare experiments and compute aggregate statistics for regression detection.
+ */
+export * from './types.js';
+export * from './aggregate.js';
+export * from './compare.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/datasets/experiment/analytics/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/datasets/experiment/analytics/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,SAAS,CAAC;AACxB,cAAc,aAAa,CAAC;AAC5B,cAAc,WAAW,CAAC"}

package/dist/datasets/experiment/analytics/types.d.ts

@@ -0,0 +1,103 @@
+/**
+ * Experiment Analytics Types
+ *
+ * Types for comparing experiments and computing aggregate statistics.
+ * Supports regression detection for CI/CD quality gates.
+ */
+/**
+ * Aggregate statistics for a single scorer across an experiment.
+ */
+export interface ScorerStats {
+    /** Items with null score / total items */
+    errorRate: number;
+    /** Count of items with null score */
+    errorCount: number;
+    /** Items >= threshold / items with scores */
+    passRate: number;
+    /** Count of items that passed threshold */
+    passCount: number;
+    /** Mean of non-null scores */
+    avgScore: number;
+    /** Count of items with non-null scores */
+    scoreCount: number;
+    /** Total items evaluated by this scorer */
+    totalItems: number;
+}
+/**
+ * Per-scorer comparison between two experiments.
+ */
+export interface ScorerComparison {
+    /** Stats from experiment A (baseline) */
+    statsA: ScorerStats;
+    /** Stats from experiment B (candidate) */
+    statsB: ScorerStats;
+    /** avgScore difference: statsB.avgScore - statsA.avgScore */
+    delta: number;
+    /** Whether this scorer regressed (delta below threshold) */
+    regressed: boolean;
+    /** Threshold used for regression detection */
+    threshold: number;
+}
+/**
+ * Per-item comparison showing score differences.
+ */
+export interface ItemComparison {
+    /** Dataset item ID */
+    itemId: string;
+    /** Whether item exists in both experiments */
+    inBothExperiments: boolean;
+    /** Scores from experiment A by scorer ID (null if no score) */
+    scoresA: Record<string, number | null>;
+    /** Scores from experiment B by scorer ID (null if no score) */
+    scoresB: Record<string, number | null>;
+}
+/**
+ * Top-level comparison result.
+ */
+export interface ComparisonResult {
+    /** Experiment A metadata */
+    experimentA: {
+        id: string;
+        datasetVersion: number | null;
+    };
+    /** Experiment B metadata */
+    experimentB: {
+        id: string;
+        datasetVersion: number | null;
+    };
+    /** True if experiments used different dataset versions */
+    versionMismatch: boolean;
+    /** True if any scorer regressed (for CI quick check) */
+    hasRegression: boolean;
+    /** Per-scorer comparison results */
+    scorers: Record<string, ScorerComparison>;
+    /** Per-item comparison details */
+    items: ItemComparison[];
+    /** Warning messages (e.g., version mismatch, no overlap) */
+    warnings: string[];
+}
+/**
+ * Threshold configuration for a single scorer.
+ */
+export interface ScorerThreshold {
+    /** Absolute threshold value for regression detection */
+    value: number;
+    /** Score direction: 'higher-is-better' (default) or 'lower-is-better' */
+    direction?: 'higher-is-better' | 'lower-is-better';
+}
+/**
+ * Configuration for compareExperiments function.
+ */
+export interface CompareExperimentsConfig {
+    /** ID of experiment A (baseline) */
+    experimentIdA: string;
+    /** ID of experiment B (candidate) */
+    experimentIdB: string;
+    /**
+     * Per-scorer thresholds for regression detection.
+     * Key is scorer ID, value is threshold config.
+     * Default when not specified: { value: 0, direction: 'higher-is-better' }
+     */
+    thresholds?: Record<string, ScorerThreshold>;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/datasets/experiment/analytics/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/datasets/experiment/analytics/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,6CAA6C;IAC7C,QAAQ,EAAE,MAAM,CAAC;IACjB,2CAA2C;IAC3C,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,0CAA0C;IAC1C,UAAU,EAAE,MAAM,CAAC;IACnB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,MAAM,EAAE,WAAW,CAAC;IACpB,0CAA0C;IAC1C,MAAM,EAAE,WAAW,CAAC;IACpB,6DAA6D;IAC7D,KAAK,EAAE,MAAM,CAAC;IACd,4DAA4D;IAC5D,SAAS,EAAE,OAAO,CAAC;IACnB,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,sBAAsB;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,iBAAiB,EAAE,OAAO,CAAC;IAC3B,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;IACvC,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,4BAA4B;IAC5B,WAAW,EAAE;QACX,EAAE,EAAE,MAAM,CAAC;QACX,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;KAC/B,CAAC;IACF,4BAA4B;IAC5B,WAAW,EAAE;QACX,EAAE,EAAE,MAAM,CAAC;QACX,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;KAC/B,CAAC;IACF,0DAA0D;IAC1D,eAAe,EAAE,OAAO,CAAC;IACzB,wDAAwD;IACxD,aAAa,EAAE,OAAO,CAAC;IACvB,oCAAoC;IACpC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC1C,kCAAkC;IAClC,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,4DAA4D;IAC5D,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAMD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,wDAAwD;IACxD,KAAK,EAAE,MAAM,CAAC;IACd,yEAAyE;IACzE,SAAS,CAAC,EAAE,kBAAkB,GAAG,iBAAiB,CAAC;CACpD;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,oCAAoC;IACpC,aAAa,EAAE,MAAM,CAAC;IACtB,qCAAqC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;CAC9C"}

package/dist/datasets/experiment/executor.d.ts

@@ -0,0 +1,40 @@
+import type { Agent } from '../../agent/index.js';
+import type { MastraScorer } from '../../evals/base.js';
+import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '../../evals/types.js';
+import type { TargetType } from '../../storage/types.js';
+import type { Workflow } from '../../workflows/index.js';
+/**
+ * Target types supported for dataset execution.
+ * Agent and Workflow are Phase 2; scorer and processor are Phase 4.
+ */
+export type Target = Agent | Workflow | MastraScorer<any, any, any, any>;
+/**
+ * Result from executing a target against a dataset item.
+ */
+export interface ExecutionResult {
+    /** Output from the target (null if failed) */
+    output: unknown;
+    /** Structured error if execution failed */
+    error: {
+        message: string;
+        stack?: string;
+        code?: string;
+    } | null;
+    /** Trace ID from agent/workflow execution (null for scorers or errors) */
+    traceId: string | null;
+    /** Structured input for scorers (extracted from agent scoring data) */
+    scorerInput?: ScorerRunInputForAgent;
+    /** Structured output for scorers (extracted from agent scoring data) */
+    scorerOutput?: ScorerRunOutputForAgent;
+}
+/**
+ * Execute a dataset item against a target (agent, workflow, scorer, processor).
+ * Phase 2: agent/workflow. Phase 4: scorer. Processor deferred.
+ */
+export declare function executeTarget(target: Target, targetType: TargetType, item: {
+    input: unknown;
+    groundTruth?: unknown;
+}, options?: {
+    signal?: AbortSignal;
+}): Promise<ExecutionResult>;
+//# sourceMappingURL=executor.d.ts.map

package/dist/datasets/experiment/executor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/datasets/experiment/executor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AACzF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAEhD;;;GAGG;AACH,MAAM,MAAM,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AAEzE;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,8CAA8C;IAC9C,MAAM,EAAE,OAAO,CAAC;IAChB,2CAA2C;IAC3C,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACjE,0EAA0E;IAC1E,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,uEAAuE;IACvE,WAAW,CAAC,EAAE,sBAAsB,CAAC;IACrC,wEAAwE;IACxE,YAAY,CAAC,EAAE,uBAAuB,CAAC;CACxC;AA2CD;;;GAGG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,UAAU,EACtB,IAAI,EAAE;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,WAAW,CAAC,EAAE,OAAO,CAAA;CAAE,EAC/C,OAAO,CAAC,EAAE;IAAE,MAAM,CAAC,EAAE,WAAW,CAAA;CAAE,GACjC,OAAO,CAAC,eAAe,CAAC,CA2C1B"}

package/dist/datasets/experiment/index.d.ts

@@ -0,0 +1,31 @@
+import type { Mastra } from '../../mastra/index.js';
+import type { ExperimentConfig, ExperimentSummary } from './types.js';
+export type { DataItem, ExperimentConfig, ExperimentSummary, ItemWithScores, ItemResult, ScorerResult, StartExperimentConfig, } from './types.js';
+export { executeTarget, type Target, type ExecutionResult } from './executor.js';
+export { resolveScorers, runScorersForItem } from './scorer.js';
+export * from './analytics/index.js';
+/**
+ * Run a dataset experiment against a target with optional scoring.
+ *
+ * Executes all items in the dataset concurrently (up to maxConcurrency) against
+ * the specified target (agent or workflow). Optionally applies scorers to each
+ * result and persists both results and scores to storage.
+ *
+ * @param mastra - Mastra instance for storage and target resolution
+ * @param config - Experiment configuration
+ * @returns ExperimentSummary with results and scores
+ *
+ * @example
+ * ```typescript
+ * const summary = await runExperiment(mastra, {
+ *   datasetId: 'my-dataset',
+ *   targetType: 'agent',
+ *   targetId: 'my-agent',
+ *   scorers: [accuracyScorer, latencyScorer],
+ *   maxConcurrency: 10,
+ * });
+ * console.log(`${summary.succeededCount}/${summary.totalItems} succeeded`);
+ * ```
+ */
+export declare function runExperiment(mastra: Mastra, config: ExperimentConfig): Promise<ExperimentSummary>;
+//# sourceMappingURL=index.d.ts.map

package/dist/datasets/experiment/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/datasets/experiment/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAa3C,OAAO,KAAK,EAAE,gBAAgB,EAAE,iBAAiB,EAA8B,MAAM,SAAS,CAAC;AAG/F,YAAY,EACV,QAAQ,EACR,gBAAgB,EAChB,iBAAiB,EACjB,cAAc,EACd,UAAU,EACV,YAAY,EACZ,qBAAqB,GACtB,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,aAAa,EAAE,KAAK,MAAM,EAAE,KAAK,eAAe,EAAE,MAAM,YAAY,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAG7D,cAAc,aAAa,CAAC;AAE5B;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAsB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CA0UxG"}

package/dist/datasets/experiment/scorer.d.ts

@@ -0,0 +1,21 @@
+import type { MastraScorer } from '../../evals/base.js';
+import type { ScorerRunInputForAgent, ScorerRunOutputForAgent } from '../../evals/types.js';
+import type { Mastra } from '../../mastra/index.js';
+import type { MastraCompositeStore } from '../../storage/base.js';
+import type { TargetType } from '../../storage/types.js';
+import type { ScorerResult } from './types.js';
+/**
+ * Resolve scorers from mixed array of instances and string IDs.
+ * String IDs are looked up from Mastra's scorer registry.
+ */
+export declare function resolveScorers(mastra: Mastra, scorers?: (MastraScorer<any, any, any, any> | string)[]): MastraScorer<any, any, any, any>[];
+/**
+ * Run all scorers for a single item result.
+ * Errors are isolated per scorer - one failing scorer doesn't affect others.
+ */
+export declare function runScorersForItem(scorers: MastraScorer<any, any, any, any>[], item: {
+    input: unknown;
+    groundTruth?: unknown;
+    metadata?: Record<string, unknown>;
+}, output: unknown, storage: MastraCompositeStore | null, runId: string, targetType: TargetType, targetId: string, itemId: string, scorerInput?: ScorerRunInputForAgent, scorerOutput?: ScorerRunOutputForAgent): Promise<ScorerResult[]>;
+//# sourceMappingURL=scorer.d.ts.map

package/dist/datasets/experiment/scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorer.d.ts","sourceRoot":"","sources":["../../../src/datasets/experiment/scorer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AACzF,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAE3C,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC/D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAE5C;;;GAGG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC,EAAE,GACtD,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAgBpC;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,EAC3C,IAAI,EAAE;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,EACnF,MAAM,EAAE,OAAO,EACf,OAAO,EAAE,oBAAoB,GAAG,IAAI,EACpC,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,UAAU,EACtB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,WAAW,CAAC,EAAE,sBAAsB,EACpC,YAAY,CAAC,EAAE,uBAAuB,GACrC,OAAO,CAAC,YAAY,EAAE,CAAC,CA8CzB"}

package/dist/datasets/experiment/types.d.ts

@@ -0,0 +1,140 @@
+import type { MastraScorer } from '../../evals/base.js';
+import type { Mastra } from '../../mastra/index.js';
+import type { TargetType, ExperimentStatus } from '../../storage/types.js';
+/**
+ * A single data item for inline experiment data.
+ * Internal — not publicly exported from @mastra/core.
+ */
+export interface DataItem<I = unknown, E = unknown> {
+    /** Unique ID (auto-generated if omitted) */
+    id?: string;
+    /** Input data passed to task */
+    input: I;
+    /** Ground truth for scoring */
+    groundTruth?: E;
+    /** Additional metadata */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Internal configuration for running a dataset experiment.
+ * Not publicly exported — users interact via Dataset.startExperiment().
+ * All new fields are optional — existing internal callers are unaffected.
+ */
+export interface ExperimentConfig<I = unknown, O = unknown, E = unknown> {
+    /** ID of dataset in storage (injected by Dataset) */
+    datasetId?: string;
+    /** Override data source — inline array or async factory (bypasses storage load) */
+    data?: DataItem<I, E>[] | (() => Promise<DataItem<I, E>[]>);
+    /** Registry-based target type (existing) */
+    targetType?: TargetType;
+    /** Registry-based target ID (existing) */
+    targetId?: string;
+    /** Inline task function (sync or async) */
+    task?: (args: {
+        input: I;
+        mastra: Mastra;
+        groundTruth?: E;
+        metadata?: Record<string, unknown>;
+        signal?: AbortSignal;
+    }) => O | Promise<O>;
+    /** Scorers — MastraScorer instances or string IDs */
+    scorers?: (MastraScorer<any, any, any, any> | string)[];
+    /** Pin to specific dataset version (default: latest). Only applies when datasetId is used. */
+    version?: number;
+    /** Maximum concurrent executions (default: 5) */
+    maxConcurrency?: number;
+    /** AbortSignal for cancellation */
+    signal?: AbortSignal;
+    /** Per-item execution timeout in milliseconds */
+    itemTimeout?: number;
+    /** Maximum retries per item on failure (default: 0 = no retries). Abort errors are never retried. */
+    maxRetries?: number;
+    /** Pre-created experiment ID (for async trigger — skips experiment creation). */
+    experimentId?: string;
+    /** Experiment name (used for display / grouping) */
+    name?: string;
+    /** Experiment description */
+    description?: string;
+    /** Arbitrary metadata for the experiment */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Configuration for starting an experiment on a dataset.
+ * The dataset is always the data source — no datasetId/data needed.
+ */
+export type StartExperimentConfig<I = unknown, O = unknown, E = unknown> = Omit<ExperimentConfig<I, O, E>, 'datasetId' | 'data' | 'experimentId'>;
+/**
+ * Result of executing a single dataset item.
+ */
+export interface ItemResult {
+    /** ID of the dataset item */
+    itemId: string;
+    /** Dataset version of the item when executed */
+    itemVersion: number;
+    /** Input data that was passed to the target */
+    input: unknown;
+    /** Output from the target (null if failed) */
+    output: unknown | null;
+    /** Expected output from the dataset item */
+    groundTruth: unknown | null;
+    /** Structured error if execution failed */
+    error: {
+        message: string;
+        stack?: string;
+        code?: string;
+    } | null;
+    /** When execution started */
+    startedAt: Date;
+    /** When execution completed */
+    completedAt: Date;
+    /** Number of retry attempts */
+    retryCount: number;
+}
+/**
+ * Result from a single scorer for an item.
+ */
+export interface ScorerResult {
+    /** ID of the scorer */
+    scorerId: string;
+    /** Display name of the scorer */
+    scorerName: string;
+    /** Computed score (null if scorer failed) */
+    score: number | null;
+    /** Reason/explanation for the score */
+    reason: string | null;
+    /** Error message if scorer failed */
+    error: string | null;
+}
+/**
+ * Item result with all scorer results attached.
+ */
+export interface ItemWithScores extends ItemResult {
+    /** Results from all scorers for this item */
+    scores: ScorerResult[];
+}
+/**
+ * Summary of an entire dataset experiment.
+ */
+export interface ExperimentSummary {
+    /** Unique ID of this experiment */
+    experimentId: string;
+    /** Final status of the experiment */
+    status: ExperimentStatus;
+    /** Total number of items in the dataset */
+    totalItems: number;
+    /** Number of items that succeeded */
+    succeededCount: number;
+    /** Number of items that failed */
+    failedCount: number;
+    /** Number of items skipped (e.g. due to abort) */
+    skippedCount: number;
+    /** True if run completed but some items failed */
+    completedWithErrors: boolean;
+    /** When the experiment started */
+    startedAt: Date;
+    /** When the experiment completed */
+    completedAt: Date;
+    /** All item results with their scores */
+    results: ItemWithScores[];
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/datasets/experiment/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/datasets/experiment/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAExE;;;GAGG;AACH,MAAM,WAAW,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO;IAChD,4CAA4C;IAC5C,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,gCAAgC;IAChC,KAAK,EAAE,CAAC,CAAC;IACT,+BAA+B;IAC/B,WAAW,CAAC,EAAE,CAAC,CAAC;IAChB,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAgB,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO;IAGrE,qDAAqD;IACrD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mFAAmF;IACnF,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAI5D,4CAA4C;IAC5C,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2CAA2C;IAC3C,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE;QACZ,KAAK,EAAE,CAAC,CAAC;QACT,MAAM,EAAE,MAAM,CAAC;QACf,WAAW,CAAC,EAAE,CAAC,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACnC,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAIrB,qDAAqD;IACrD,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC,EAAE,CAAC;IAIxD,8FAA8F;IAC9F,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,mCAAmC;IACnC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,qGAAqG;IACrG,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iFAAiF;IACjF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oDAAoD;IACpD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6BAA6B;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4CAA4C;IAC5C,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;;GAGG;AACH,MAAM,MAAM,qBAAqB,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,IAAI,IAAI,CAC7E,gBAAgB,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EACzB,WAAW,GAAG,MAAM,GAAG,cAAc,CACtC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,6BAA6B;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,WAAW,EAAE,MAAM,CAAC;IACpB,+CAA+C;IAC/C,KAAK,EAAE,OAAO,CAAC;IACf,8CAA8C;IAC9C,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC;IACvB,4CAA4C;IAC5C,WAAW,EAAE,OAAO,GAAG,IAAI,CAAC;IAC5B,2CAA2C;IAC3C,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACjE,6BAA6B;IAC7B,SAAS,EAAE,IAAI,CAAC;IAChB,+BAA+B;IAC/B,WAAW,EAAE,IAAI,CAAC;IAClB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,uBAAuB;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,6CAA6C;IAC7C,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,uCAAuC;IACvC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,qCAAqC;IACrC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,cAAe,SAAQ,UAAU;IAChD,6CAA6C;IAC7C,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,mCAAmC;IACnC,YAAY,EAAE,MAAM,CAAC;IACrB,qCAAqC;IACrC,MAAM,EAAE,gBAAgB,CAAC;IACzB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,qCAAqC;IACrC,cAAc,EAAE,MAAM,CAAC;IACvB,kCAAkC;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,kDAAkD;IAClD,YAAY,EAAE,MAAM,CAAC;IACrB,kDAAkD;IAClD,mBAAmB,EAAE,OAAO,CAAC;IAC7B,kCAAkC;IAClC,SAAS,EAAE,IAAI,CAAC;IAChB,oCAAoC;IACpC,WAAW,EAAE,IAAI,CAAC;IAClB,yCAAyC;IACzC,OAAO,EAAE,cAAc,EAAE,CAAC;CAC3B"}