@m4trix/evals 0.5.0 → 0.8.0

package/dist/cli-simple.cjs

@@ -633,9 +633,18 @@ function mergeRunnerOverrides(base, next) {
   if (!base) {
     return next;
   }
-  {
+  if (!next) {
     return base;
   }
+  const discovery = base.discovery || next.discovery ? {
+    ...base.discovery ?? {},
+    ...next.discovery ?? {}
+  } : void 0;
+  return {
+    ...base,
+    ...next,
+    discovery
+  };
 }
 function createRunner(overrides) {
   const fileOverrides = loadRunnerConfigFile();

package/dist/cli-simple.js

@@ -2,7 +2,7 @@
 import { randomUUID } from 'crypto';
 import { Effect, PubSub, Queue, Fiber } from 'effect';
 import { existsSync } from 'fs';
-import { resolve, parse, join, relative, dirname } from 'path';
+import { resolve, relative, join, parse, dirname } from 'path';
 import * as jitiModule from 'jiti';
 import { writeFile, mkdir, appendFile, readdir } from 'fs/promises';
 import { pathToFileURL } from 'url';
@@ -610,9 +610,18 @@ function mergeRunnerOverrides(base, next) {
   if (!base) {
     return next;
   }
-  {
+  if (!next) {
     return base;
   }
+  const discovery = base.discovery || next.discovery ? {
+    ...base.discovery ?? {},
+    ...next.discovery ?? {}
+  } : void 0;
+  return {
+    ...base,
+    ...next,
+    discovery
+  };
 }
 function createRunner(overrides) {
   const fileOverrides = loadRunnerConfigFile();

package/dist/cli.cjs

@@ -1899,9 +1899,18 @@ function mergeRunnerOverrides(base, next) {
   if (!base) {
     return next;
   }
-  {
+  if (!next) {
     return base;
   }
+  const discovery = base.discovery || next.discovery ? {
+    ...base.discovery ?? {},
+    ...next.discovery ?? {}
+  } : void 0;
+  return {
+    ...base,
+    ...next,
+    discovery
+  };
 }
 function createRunner(overrides) {
   const fileOverrides = loadRunnerConfigFile();

package/dist/cli.js

@@ -1873,9 +1873,18 @@ function mergeRunnerOverrides(base, next) {
   if (!base) {
     return next;
   }
-  {
+  if (!next) {
     return base;
   }
+  const discovery = base.discovery || next.discovery ? {
+    ...base.discovery ?? {},
+    ...next.discovery ?? {}
+  } : void 0;
+  return {
+    ...base,
+    ...next,
+    discovery
+  };
 }
 function createRunner(overrides) {
   const fileOverrides = loadRunnerConfigFile();

package/dist/index.cjs

@@ -1318,7 +1318,7 @@ var EffectRunner = class {
   }
 };
 
-Object.defineProperty(exports, "S", {
+Object.defineProperty(exports, 'S', {
   enumerable: true,
   get: function () { return effect.Schema; }
 });

package/dist/index.d.ts

@@ -210,7 +210,7 @@ interface MetricDef<TData = unknown> {
 declare const Metric: {
     of<TData>(config: {
         id: string;
-        name?: string;
+        name?: string | undefined;
         format: (data: TData) => string;
     }): MetricDef<TData>;
 };
@@ -234,7 +234,7 @@ interface ScoreDef<TData = unknown> {
 declare const Score: {
     of<TData>(config: {
         id: string;
-        name?: string;
+        name?: string | undefined;
         displayStrategy: ScoreDisplayStrategy;
         format: (data: TData) => string;
     }): ScoreDef<TData>;
@@ -375,4 +375,4 @@ interface BinaryScoreData {
 }
 declare const binaryScore: ScoreDef<BinaryScoreData>;
 
-export { type BinaryScoreData, type CliState, type CollectedDataset, type CollectedEvaluator, type CollectedTestCase, type ConfigType, Dataset, type EvalDataset, type EvalMiddleware, type EvalRun, type EvalsData, type EvaluateArgs, Evaluator, type EvaluatorOption, type LatencyData, type M4trixEvalConfig, type M4trixEvalConfigDiscovery, Metric, type MetricDef, type MetricItem, type PathMatcher, type PercentScoreData, type RunDatasetRequest, type RunSnapshot, type RunnerApi, type RunnerConfig, type RunnerConfigOverrides, type RunnerDiscoveryConfig, type RunnerEvent, Score, type ScoreDef, type ScoreDisplayStrategy, type ScoreItem, type SearchTestCasesQuery, type StartupArgs, type TagMatcher, TestCase, type TokenCountData, type ViewLevel, binaryScore, createRunner, defaultRunnerConfig, defineConfig, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, tokenCountMetric, withRunnerConfig };
+export { BinaryScoreData, CliState, CollectedDataset, CollectedEvaluator, CollectedTestCase, ConfigType, Dataset, EvalDataset, EvalMiddleware, EvalRun, EvalsData, EvaluateArgs, Evaluator, EvaluatorOption, LatencyData, M4trixEvalConfig, M4trixEvalConfigDiscovery, Metric, MetricDef, MetricItem, PathMatcher, PercentScoreData, RunDatasetRequest, RunSnapshot, RunnerApi, RunnerConfig, RunnerConfigOverrides, RunnerDiscoveryConfig, RunnerEvent, Score, ScoreDef, ScoreDisplayStrategy, ScoreItem, SearchTestCasesQuery, StartupArgs, TagMatcher, TestCase, TokenCountData, ViewLevel, binaryScore, createRunner, defaultRunnerConfig, defineConfig, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, tokenCountMetric, withRunnerConfig };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@m4trix/evals",
-  "version": "0.5.0",
+  "version": "0.8.0",
   "description": "Ink-based evaluation console for m4trix",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -27,7 +27,8 @@
     "dev": "tsup --watch",
     "lint": "eslint src --ext .ts,.tsx",
     "prepare": "pnpm run build",
-    "publish-package": "npm publish --no-git-checks"
+    "publish-package": "npm publish --no-git-checks",
+    "release": "pnpm -C ../.. run release:bump:evals"
   },
   "dependencies": {
     "effect": "^3.16.10",

package/dist/cli-simple.d.cts

@@ -1 +0,0 @@
-#!/usr/bin/env node

package/dist/cli.d.cts

@@ -1 +0,0 @@
-#!/usr/bin/env node

package/dist/index.d.cts

@@ -1,378 +0,0 @@
-import { Schema } from 'effect';
-export { Schema as S } from 'effect';
-
-type EvalStatus = 'PASS' | 'FAILED' | 'RUNNING';
-interface EvalDimension {
-    name: string;
-    score: number;
-}
-interface EvalCheck {
-    name: string;
-    passed: boolean;
-    detail?: string;
-}
-interface EvalFailure {
-    title: string;
-}
-interface EvalPerformance {
-    passRate: number;
-    avgScore: number;
-    latencyP95Ms: number;
-    latencyAvgMs: number;
-    tokensAvg: number;
-    tokensP95: number;
-    costUsd: number;
-    /** Per-sample latency in ms for sparkline (e.g. last N requests) */
-    latencyHistoryMs?: number[];
-}
-interface EvalRunMeta {
-    model: string;
-    provider: string;
-    commit: string;
-    branch: string;
-    seed: number;
-    concurrency: number;
-    duration: string;
-    artifact: string;
-}
-interface EvalRun {
-    id: string;
-    label: string;
-    status: EvalStatus;
-    performance: EvalPerformance;
-    dimensions: EvalDimension[];
-    checks: EvalCheck[];
-    failures: EvalFailure[];
-    meta: EvalRunMeta;
-}
-interface EvalDataset {
-    id: string;
-    name: string;
-    overview: string;
-    runs: EvalRun[];
-}
-interface EvaluatorOption {
-    id: string;
-    name: string;
-    configPreview: string;
-}
-interface EvalsData {
-    datasets: EvalDataset[];
-    evaluators: EvaluatorOption[];
-}
-type PaneFocus = 'left' | 'right';
-type ViewLevel = 'datasets' | 'runs' | 'details' | 'new-evaluation';
-interface StartupArgs {
-    datasetId?: string;
-    runId?: string;
-    search?: string;
-    unknownArgs: string[];
-}
-interface CliState {
-    level: ViewLevel;
-    focus: PaneFocus;
-    datasetMenuIndex: number;
-    runMenuIndex: number;
-    detailsScrollOffset: number;
-    selectedEvaluatorIds: string[];
-    evaluatorMenuIndex: number;
-    searchQuery: string;
-    searchMode: boolean;
-    startupWarnings: string[];
-}
-
-interface RunnerDiscoveryConfig {
-    rootDir: string;
-    datasetSuffixes: ReadonlyArray<string>;
-    evaluatorSuffixes: ReadonlyArray<string>;
-    testCaseSuffixes: ReadonlyArray<string>;
-    excludeDirectories: ReadonlyArray<string>;
-}
-interface RunnerConfig {
-    discovery: RunnerDiscoveryConfig;
-    artifactDirectory: string;
-}
-type RunnerConfigOverrides = Omit<Partial<RunnerConfig>, 'discovery'> & {
-    discovery?: Partial<RunnerDiscoveryConfig>;
-};
-interface M4trixEvalConfigDiscovery {
-    rootDir?: string;
-    datasetFilePatterns?: ReadonlyArray<string>;
-    evaluatorFilePatterns?: ReadonlyArray<string>;
-    testCaseFilePatterns?: ReadonlyArray<string>;
-    datasetSuffixes?: ReadonlyArray<string>;
-    evaluatorSuffixes?: ReadonlyArray<string>;
-    testCaseSuffixes?: ReadonlyArray<string>;
-    excludeDirectories?: ReadonlyArray<string>;
-}
-interface M4trixEvalConfig {
-    discovery?: M4trixEvalConfigDiscovery;
-    artifactDirectory?: string;
-}
-type ConfigType = M4trixEvalConfig;
-type M4trixEvalConfigFactory<TConfig extends ConfigType = ConfigType> = () => TConfig;
-declare function defineConfig<TConfig extends ConfigType>(factory: M4trixEvalConfigFactory<TConfig>): M4trixEvalConfigFactory<TConfig>;
-declare const defaultRunnerConfig: RunnerConfig;
-declare function withRunnerConfig(overrides?: RunnerConfigOverrides): RunnerConfig;
-
-/** Matches a tag by exact string equality or regex test */
-type TagMatcher = string | RegExp;
-/** Matches a file path by glob string or regex test */
-type PathMatcher = string | RegExp;
-
-type InputOrBuilder<T> = T | (() => T);
-interface TestCaseDescribeConfig<TI extends Schema.Schema.Any, TO extends Schema.Schema.Any = Schema.Schema<unknown>> {
-    name: string;
-    tags: string[];
-    inputSchema: TI;
-    input: InputOrBuilder<Schema.Schema.Type<TI>>;
-    outputSchema?: TO;
-    output?: InputOrBuilder<Schema.Schema.Type<TO>>;
-}
-declare class TestCase<TInput = unknown, TOutput = unknown> {
-    private readonly _config;
-    private constructor();
-    static describe<TI extends Schema.Schema.Any, TO extends Schema.Schema.Any = Schema.Schema<unknown>>(config: TestCaseDescribeConfig<TI, TO>): TestCase<Schema.Schema.Type<TI>, Schema.Schema.Type<TO>>;
-    getName(): string;
-    getTags(): string[];
-    getInputSchema(): Schema.Schema.Any;
-    getInput(): TInput;
-    getOutputSchema(): Schema.Schema.Any | undefined;
-    getOutput(): TOutput | undefined;
-}
-
-interface DatasetDefineConfig {
-    name: string;
-    includedTags?: TagMatcher[];
-    excludedTags?: TagMatcher[];
-    includedPaths?: PathMatcher[];
-    excludedPaths?: PathMatcher[];
-}
-declare class Dataset {
-    private readonly _config;
-    private constructor();
-    static define(config: DatasetDefineConfig): Dataset;
-    getName(): string;
-    getIncludedTags(): ReadonlyArray<TagMatcher>;
-    getExcludedTags(): ReadonlyArray<TagMatcher>;
-    getIncludedPaths(): ReadonlyArray<PathMatcher>;
-    getExcludedPaths(): ReadonlyArray<PathMatcher>;
-    matchesTestCase(testCase: TestCase<unknown>, filePath: string): boolean;
-}
-
-interface EvalMiddleware<TCtx> {
-    name: string;
-    resolve: () => TCtx | Promise<TCtx>;
-}
-interface EvaluateArgs<TInput, TCtx> {
-    input: TInput;
-    ctx: TCtx;
-    output?: unknown;
-}
-type EvaluateFn<TInput, TScore, TCtx> = (args: EvaluateArgs<TInput, TCtx>) => TScore | Promise<TScore>;
-interface EvaluatorDefineConfig<TI extends Schema.Schema.Any, TO extends Schema.Schema.Any, TS extends Schema.Schema.Any> {
-    name: string;
-    inputSchema: TI;
-    outputSchema: TO;
-    scoreSchema: TS;
-    passThreshold?: number;
-    passCriterion?: (score: unknown) => boolean;
-}
-declare class Evaluator<TInput = unknown, TOutput = unknown, TScore = unknown, TCtx = Record<string, never>> {
-    private readonly _config;
-    private constructor();
-    private getState;
-    static use<TCtx>(middleware: EvalMiddleware<TCtx>): Evaluator<unknown, unknown, unknown, TCtx>;
-    use<TNew>(middleware: EvalMiddleware<TNew>): Evaluator<TInput, TOutput, TScore, TCtx & TNew>;
-    define<TI extends Schema.Schema.Any, TO extends Schema.Schema.Any, TS extends Schema.Schema.Any>(config: EvaluatorDefineConfig<TI, TO, TS>): Evaluator<Schema.Schema.Type<TI>, Schema.Schema.Type<TO>, Schema.Schema.Type<TS>, TCtx>;
-    evaluate(fn: EvaluateFn<TInput, TScore, TCtx>): Evaluator<TInput, TOutput, TScore, TCtx>;
-    getName(): string | undefined;
-    getInputSchema(): Schema.Schema.Any | undefined;
-    getOutputSchema(): Schema.Schema.Any | undefined;
-    getScoreSchema(): Schema.Schema.Any | undefined;
-    getMiddlewares(): ReadonlyArray<EvalMiddleware<unknown>>;
-    getEvaluateFn(): EvaluateFn<TInput, TScore, TCtx> | undefined;
-    getPassThreshold(): number | undefined;
-    getPassCriterion(): ((score: unknown) => boolean) | undefined;
-    resolveContext(): Promise<TCtx>;
-}
-
-interface MetricItem<TData = unknown> {
-    readonly id: string;
-    readonly data: TData;
-}
-interface MetricDef<TData = unknown> {
-    readonly id: string;
-    readonly name?: string;
-    format(data: TData): string;
-    make(data: TData): MetricItem<TData>;
-}
-declare const Metric: {
-    of<TData>(config: {
-        id: string;
-        name?: string;
-        format: (data: TData) => string;
-    }): MetricDef<TData>;
-};
-declare function getMetricById(id: string): MetricDef<unknown> | undefined;
-
-type ScoreDisplayStrategy = 'bar' | 'number' | 'passFail';
-interface ScoreItem<TData = unknown> {
-    readonly id: string;
-    readonly data: TData;
-    readonly passed?: boolean;
-}
-interface ScoreDef<TData = unknown> {
-    readonly id: string;
-    readonly name?: string;
-    readonly displayStrategy: ScoreDisplayStrategy;
-    format(data: TData): string;
-    make(data: TData, options?: {
-        definePassed?: (data: TData) => boolean;
-    }): ScoreItem<TData>;
-}
-declare const Score: {
-    of<TData>(config: {
-        id: string;
-        name?: string;
-        displayStrategy: ScoreDisplayStrategy;
-        format: (data: TData) => string;
-    }): ScoreDef<TData>;
-};
-declare function getScoreById(id: string): ScoreDef<unknown> | undefined;
-
-interface CollectedDataset {
-    id: string;
-    filePath: string;
-    dataset: Dataset;
-}
-interface CollectedEvaluator {
-    id: string;
-    filePath: string;
-    evaluator: Evaluator<unknown, unknown, unknown, unknown>;
-}
-interface CollectedTestCase {
-    id: string;
-    filePath: string;
-    testCase: TestCase<unknown, unknown>;
-}
-interface SearchTestCasesQuery {
-    includedTags?: ReadonlyArray<string | RegExp>;
-    excludedTags?: ReadonlyArray<string | RegExp>;
-    includedPaths?: ReadonlyArray<string | RegExp>;
-    excludedPaths?: ReadonlyArray<string | RegExp>;
-}
-interface RunDatasetRequest {
-    datasetId: string;
-    evaluatorIds: ReadonlyArray<string>;
-    concurrency?: number;
-}
-interface RunSnapshot {
-    runId: string;
-    datasetId: string;
-    datasetName: string;
-    evaluatorIds: ReadonlyArray<string>;
-    queuedAt: number;
-    startedAt?: number;
-    finishedAt?: number;
-    totalTestCases: number;
-    completedTestCases: number;
-    passedTestCases: number;
-    failedTestCases: number;
-    status: 'queued' | 'running' | 'completed' | 'failed';
-    artifactPath: string;
-    errorMessage?: string;
-}
-type RunnerEvent = {
-    type: 'RunQueued';
-    runId: string;
-    datasetId: string;
-    datasetName: string;
-    evaluatorIds: ReadonlyArray<string>;
-    totalTestCases: number;
-    artifactPath: string;
-} | {
-    type: 'RunStarted';
-    runId: string;
-    startedAt: number;
-} | {
-    type: 'TestCaseProgress';
-    runId: string;
-    testCaseId: string;
-    testCaseName: string;
-    completedTestCases: number;
-    totalTestCases: number;
-    passed: boolean;
-    durationMs: number;
-    evaluatorScores: ReadonlyArray<{
-        evaluatorId: string;
-        scores: ReadonlyArray<ScoreItem>;
-        passed: boolean;
-        metrics?: ReadonlyArray<MetricItem>;
-    }>;
-    output?: unknown;
-    errorMessage?: string;
-} | {
-    type: 'RunCompleted';
-    runId: string;
-    finishedAt: number;
-    passedTestCases: number;
-    failedTestCases: number;
-    totalTestCases: number;
-    artifactPath: string;
-} | {
-    type: 'RunFailed';
-    runId: string;
-    finishedAt: number;
-    errorMessage: string;
-    artifactPath: string;
-} | {
-    type: 'ArtifactFlushed';
-    runId: string;
-    artifactPath: string;
-};
-
-interface SubscribeOptions {
-    runId?: string;
-}
-interface RunnerApi {
-    collectDatasets(): Promise<ReadonlyArray<CollectedDataset>>;
-    collectEvaluators(): Promise<ReadonlyArray<CollectedEvaluator>>;
-    resolveDatasetByName(name: string): Promise<CollectedDataset | undefined>;
-    resolveEvaluatorsByNamePattern(pattern: string): Promise<ReadonlyArray<CollectedEvaluator>>;
-    searchTestCases(query?: SearchTestCasesQuery): Promise<ReadonlyArray<CollectedTestCase>>;
-    collectDatasetTestCases(datasetId: string): Promise<ReadonlyArray<CollectedTestCase>>;
-    runDatasetWith(request: RunDatasetRequest): Promise<RunSnapshot>;
-    subscribeRunEvents(listener: (event: RunnerEvent) => void, options?: SubscribeOptions): () => void;
-    getRunSnapshot(runId: string): RunSnapshot | undefined;
-    getAllRunSnapshots(): ReadonlyArray<RunSnapshot>;
-    shutdown(): Promise<void>;
-}
-declare function createRunner(overrides?: RunnerConfigOverrides): RunnerApi;
-
-declare function loadMockData(): EvalsData;
-declare function loadRunnerData(runner: RunnerApi): Promise<EvalsData>;
-declare function parseStartupArgs(argv: string[]): StartupArgs;
-
-interface TokenCountData {
-    input?: number;
-    output?: number;
-    inputCached?: number;
-    outputCached?: number;
-}
-declare const tokenCountMetric: MetricDef<TokenCountData>;
-interface LatencyData {
-    ms: number;
-}
-declare const latencyMetric: MetricDef<LatencyData>;
-
-interface PercentScoreData {
-    value: number;
-}
-declare const percentScore: ScoreDef<PercentScoreData>;
-interface BinaryScoreData {
-    passed: boolean;
-}
-declare const binaryScore: ScoreDef<BinaryScoreData>;
-
-export { type BinaryScoreData, type CliState, type CollectedDataset, type CollectedEvaluator, type CollectedTestCase, type ConfigType, Dataset, type EvalDataset, type EvalMiddleware, type EvalRun, type EvalsData, type EvaluateArgs, Evaluator, type EvaluatorOption, type LatencyData, type M4trixEvalConfig, type M4trixEvalConfigDiscovery, Metric, type MetricDef, type MetricItem, type PathMatcher, type PercentScoreData, type RunDatasetRequest, type RunSnapshot, type RunnerApi, type RunnerConfig, type RunnerConfigOverrides, type RunnerDiscoveryConfig, type RunnerEvent, Score, type ScoreDef, type ScoreDisplayStrategy, type ScoreItem, type SearchTestCasesQuery, type StartupArgs, type TagMatcher, TestCase, type TokenCountData, type ViewLevel, binaryScore, createRunner, defaultRunnerConfig, defineConfig, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, tokenCountMetric, withRunnerConfig };