@m4trix/evals 0.11.0 → 0.13.0

package/dist/index.d.ts

@@ -91,6 +91,8 @@ interface RunnerDiscoveryConfig {
 interface RunnerConfig {
     discovery: RunnerDiscoveryConfig;
     artifactDirectory: string;
+    /** Max concurrent test cases per run. Default: 1 (sequential). */
+    maxConcurrency: number;
 }
 type RunnerConfigOverrides = Omit<Partial<RunnerConfig>, 'discovery'> & {
     discovery?: Partial<RunnerDiscoveryConfig>;
@@ -108,6 +110,8 @@ interface M4trixEvalConfigDiscovery {
 interface M4trixEvalConfig {
     discovery?: M4trixEvalConfigDiscovery;
     artifactDirectory?: string;
+    /** Max concurrent test cases per run. Default: 1 (sequential). */
+    maxConcurrency?: number;
 }
 type ConfigType = M4trixEvalConfig;
 type M4trixEvalConfigFactory<TConfig extends ConfigType = ConfigType> = () => TConfig;
@@ -124,6 +128,7 @@ type InputOrBuilder<T> = T | (() => T);
 interface TestCaseDescribeConfig<TI extends Schema.Schema.Any, TO extends Schema.Schema.Any = Schema.Schema<unknown>> {
     name: string;
     tags: string[];
+    reruns?: number;
     inputSchema: TI;
     input: InputOrBuilder<Schema.Schema.Type<TI>>;
     outputSchema?: TO;
@@ -133,6 +138,7 @@ declare class TestCase<TInput = unknown, TOutput = unknown> {
     private readonly _config;
     private constructor();
     static describe<TI extends Schema.Schema.Any, TO extends Schema.Schema.Any = Schema.Schema<unknown>>(config: TestCaseDescribeConfig<TI, TO>): TestCase<Schema.Schema.Type<TI>, Schema.Schema.Type<TO>>;
+    getReruns(): number;
     getName(): string;
     getTags(): string[];
     getInputSchema(): Schema.Schema.Any;
@@ -225,17 +231,22 @@ interface MetricItem<TData = unknown> {
     readonly id: string;
     readonly data: TData;
 }
+interface FormatMetricOptions {
+    isAggregated?: boolean;
+}
 interface MetricDef<TData = unknown> {
     readonly id: string;
     readonly name?: string;
-    format(data: TData): string;
+    readonly aggregate?: (values: ReadonlyArray<TData>) => TData;
+    format(data: TData, options?: FormatMetricOptions): string;
     make(data: TData): MetricItem<TData>;
 }
 declare const Metric: {
     of<TData>(config: {
         id: string;
         name?: string | undefined;
-        format: (data: TData) => string;
+        format: (data: TData, options?: FormatMetricOptions) => string;
+        aggregate?: ((values: readonly TData[]) => TData) | undefined;
     }): MetricDef<TData>;
 };
 declare function getMetricById(id: string): MetricDef<unknown> | undefined;
@@ -246,11 +257,15 @@ interface ScoreItem<TData = unknown> {
     readonly data: TData;
     readonly passed?: boolean;
 }
+interface FormatScoreOptions {
+    isAggregated?: boolean;
+}
 interface ScoreDef<TData = unknown> {
     readonly id: string;
     readonly name?: string;
     readonly displayStrategy: ScoreDisplayStrategy;
-    format(data: TData): string;
+    readonly aggregate?: (values: ReadonlyArray<TData>) => TData;
+    format(data: TData, options?: FormatScoreOptions): string;
     make(data: TData, options?: {
         definePassed?: (data: TData) => boolean;
     }): ScoreItem<TData>;
@@ -260,7 +275,8 @@ declare const Score: {
         id: string;
         name?: string | undefined;
         displayStrategy: ScoreDisplayStrategy;
-        format: (data: TData) => string;
+        format: (data: TData, options?: FormatScoreOptions) => string;
+        aggregate?: ((values: readonly TData[]) => TData) | undefined;
     }): ScoreDef<TData>;
 };
 declare function getScoreById(id: string): ScoreDef<unknown> | undefined;
@@ -326,6 +342,8 @@ type RunnerEvent = {
     testCaseName: string;
     completedTestCases: number;
     totalTestCases: number;
+    rerunIndex: number;
+    rerunTotal: number;
     passed: boolean;
     durationMs: number;
     evaluatorScores: ReadonlyArray<{
@@ -371,6 +389,7 @@ interface RunnerApi {
     subscribeRunEvents(listener: (event: RunnerEvent) => void, options?: SubscribeOptions): () => void;
     getRunSnapshot(runId: string): RunSnapshot | undefined;
     getAllRunSnapshots(): ReadonlyArray<RunSnapshot>;
+    loadRunSnapshotsFromArtifacts(): Promise<ReadonlyArray<RunSnapshot>>;
     shutdown(): Promise<void>;
 }
 declare function createRunner(overrides?: RunnerConfigOverrides): RunnerApi;
@@ -400,4 +419,4 @@ interface BinaryScoreData {
 }
 declare const binaryScore: ScoreDef<BinaryScoreData>;
 
-export { BinaryScoreData, CliState, CollectedDataset, CollectedEvaluator, CollectedTestCase, ConfigType, Dataset, EvalDataset, EvalMiddleware, EvalRun, EvalsData, EvaluateArgs, Evaluator, EvaluatorOption, LatencyData, M4trixEvalConfig, M4trixEvalConfigDiscovery, Metric, MetricDef, MetricItem, PathMatcher, PercentScoreData, PrintJsonDiffOptions, RunDatasetRequest, RunSnapshot, RunnerApi, RunnerConfig, RunnerConfigOverrides, RunnerDiscoveryConfig, RunnerEvent, Score, ScoreDef, ScoreDisplayStrategy, ScoreItem, SearchTestCasesQuery, StartupArgs, TagMatcher, TestCase, TokenCountData, ViewLevel, binaryScore, createRunner, defaultRunnerConfig, defineConfig, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, printJsonDiff, tokenCountMetric, withRunnerConfig };
+export { BinaryScoreData, CliState, CollectedDataset, CollectedEvaluator, CollectedTestCase, ConfigType, Dataset, EvalDataset, EvalMiddleware, EvalRun, EvalsData, EvaluateArgs, Evaluator, EvaluatorOption, FormatMetricOptions, FormatScoreOptions, LatencyData, M4trixEvalConfig, M4trixEvalConfigDiscovery, Metric, MetricDef, MetricItem, PathMatcher, PercentScoreData, PrintJsonDiffOptions, RunDatasetRequest, RunSnapshot, RunnerApi, RunnerConfig, RunnerConfigOverrides, RunnerDiscoveryConfig, RunnerEvent, Score, ScoreDef, ScoreDisplayStrategy, ScoreItem, SearchTestCasesQuery, StartupArgs, TagMatcher, TestCase, TokenCountData, ViewLevel, binaryScore, createRunner, defaultRunnerConfig, defineConfig, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, printJsonDiff, tokenCountMetric, withRunnerConfig };