@m4trix/evals 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,347 @@
+import { Schema } from 'effect';
+
+type EvalStatus = 'PASS' | 'FAILED' | 'RUNNING';
+interface EvalDimension {
+    name: string;
+    score: number;
+}
+interface EvalCheck {
+    name: string;
+    passed: boolean;
+    detail?: string;
+}
+interface EvalFailure {
+    title: string;
+}
+interface EvalPerformance {
+    passRate: number;
+    avgScore: number;
+    latencyP95Ms: number;
+    latencyAvgMs: number;
+    tokensAvg: number;
+    tokensP95: number;
+    costUsd: number;
+    /** Per-sample latency in ms for sparkline (e.g. last N requests) */
+    latencyHistoryMs?: number[];
+}
+interface EvalRunMeta {
+    model: string;
+    provider: string;
+    commit: string;
+    branch: string;
+    seed: number;
+    concurrency: number;
+    duration: string;
+    artifact: string;
+}
+interface EvalRun {
+    id: string;
+    label: string;
+    status: EvalStatus;
+    performance: EvalPerformance;
+    dimensions: EvalDimension[];
+    checks: EvalCheck[];
+    failures: EvalFailure[];
+    meta: EvalRunMeta;
+}
+interface EvalDataset {
+    id: string;
+    name: string;
+    overview: string;
+    runs: EvalRun[];
+}
+interface EvaluatorOption {
+    id: string;
+    name: string;
+    configPreview: string;
+}
+interface EvalsData {
+    datasets: EvalDataset[];
+    evaluators: EvaluatorOption[];
+}
+type PaneFocus = 'left' | 'right';
+type ViewLevel = 'datasets' | 'runs' | 'details' | 'new-evaluation';
+interface StartupArgs {
+    datasetId?: string;
+    runId?: string;
+    search?: string;
+    unknownArgs: string[];
+}
+interface CliState {
+    level: ViewLevel;
+    focus: PaneFocus;
+    datasetMenuIndex: number;
+    runMenuIndex: number;
+    detailsScrollOffset: number;
+    selectedEvaluatorIds: string[];
+    evaluatorMenuIndex: number;
+    searchQuery: string;
+    searchMode: boolean;
+    startupWarnings: string[];
+}
+
+interface RunnerDiscoveryConfig {
+    rootDir: string;
+    datasetSuffixes: ReadonlyArray<string>;
+    evaluatorSuffixes: ReadonlyArray<string>;
+    testCaseSuffixes: ReadonlyArray<string>;
+    excludeDirectories: ReadonlyArray<string>;
+}
+interface RunnerConfig {
+    discovery: RunnerDiscoveryConfig;
+    artifactDirectory: string;
+}
+declare const defaultRunnerConfig: RunnerConfig;
+declare function withRunnerConfig(overrides?: Partial<RunnerConfig>): RunnerConfig;
+
+/** Matches a tag by exact string equality or regex test */
+type TagMatcher = string | RegExp;
+/** Matches a file path by glob string or regex test */
+type PathMatcher = string | RegExp;
+
+type InputOrBuilder<T> = T | (() => T);
+interface TestCaseDescribeConfig<TI extends Schema.Schema.Any> {
+    name: string;
+    tags: string[];
+    inputSchema: TI;
+    input: InputOrBuilder<Schema.Schema.Type<TI>>;
+}
+declare class TestCase<TInput = unknown> {
+    private readonly _config;
+    private constructor();
+    static describe<TI extends Schema.Schema.Any>(config: TestCaseDescribeConfig<TI>): TestCase<Schema.Schema.Type<TI>>;
+    getName(): string;
+    getTags(): string[];
+    getInputSchema(): Schema.Schema.Any;
+    getInput(): TInput;
+}
+
+interface DatasetDefineConfig {
+    name: string;
+    includedTags?: TagMatcher[];
+    excludedTags?: TagMatcher[];
+    includedPaths?: PathMatcher[];
+    excludedPaths?: PathMatcher[];
+}
+declare class Dataset {
+    private readonly _config;
+    private constructor();
+    static define(config: DatasetDefineConfig): Dataset;
+    getName(): string;
+    getIncludedTags(): ReadonlyArray<TagMatcher>;
+    getExcludedTags(): ReadonlyArray<TagMatcher>;
+    getIncludedPaths(): ReadonlyArray<PathMatcher>;
+    getExcludedPaths(): ReadonlyArray<PathMatcher>;
+    matchesTestCase(testCase: TestCase<unknown>, filePath: string): boolean;
+}
+
+interface EvalMiddleware<TCtx> {
+    name: string;
+    resolve: () => TCtx | Promise<TCtx>;
+}
+type EvaluateFn<TInput, TScore, TCtx> = (input: TInput, ctx: TCtx) => TScore | Promise<TScore>;
+interface EvaluatorDefineConfig<TI extends Schema.Schema.Any, TO extends Schema.Schema.Any, TS extends Schema.Schema.Any> {
+    name: string;
+    inputSchema: TI;
+    outputSchema: TO;
+    scoreSchema: TS;
+    passThreshold?: number;
+    passCriterion?: (score: unknown) => boolean;
+}
+declare class Evaluator<TInput = unknown, TOutput = unknown, TScore = unknown, TCtx = Record<string, never>> {
+    private readonly _config;
+    private constructor();
+    private getState;
+    static use<TCtx>(middleware: EvalMiddleware<TCtx>): Evaluator<unknown, unknown, unknown, TCtx>;
+    use<TNew>(middleware: EvalMiddleware<TNew>): Evaluator<TInput, TOutput, TScore, TCtx & TNew>;
+    define<TI extends Schema.Schema.Any, TO extends Schema.Schema.Any, TS extends Schema.Schema.Any>(config: EvaluatorDefineConfig<TI, TO, TS>): Evaluator<Schema.Schema.Type<TI>, Schema.Schema.Type<TO>, Schema.Schema.Type<TS>, TCtx>;
+    evaluate(fn: EvaluateFn<TInput, TScore, TCtx>): Evaluator<TInput, TOutput, TScore, TCtx>;
+    getName(): string | undefined;
+    getInputSchema(): Schema.Schema.Any | undefined;
+    getOutputSchema(): Schema.Schema.Any | undefined;
+    getScoreSchema(): Schema.Schema.Any | undefined;
+    getMiddlewares(): ReadonlyArray<EvalMiddleware<unknown>>;
+    getEvaluateFn(): EvaluateFn<TInput, TScore, TCtx> | undefined;
+    getPassThreshold(): number | undefined;
+    getPassCriterion(): ((score: unknown) => boolean) | undefined;
+    resolveContext(): Promise<TCtx>;
+}
+
+interface MetricItem<TData = unknown> {
+    readonly id: string;
+    readonly data: TData;
+}
+interface MetricDef<TData = unknown> {
+    readonly id: string;
+    readonly name?: string;
+    format(data: TData): string;
+    make(data: TData): MetricItem<TData>;
+}
+declare const Metric: {
+    of<TData>(config: {
+        id: string;
+        name?: string;
+        format: (data: TData) => string;
+    }): MetricDef<TData>;
+};
+declare function getMetricById(id: string): MetricDef<unknown> | undefined;
+
+type ScoreDisplayStrategy = 'bar' | 'number' | 'passFail';
+interface ScoreItem<TData = unknown> {
+    readonly id: string;
+    readonly data: TData;
+    readonly passed?: boolean;
+}
+interface ScoreDef<TData = unknown> {
+    readonly id: string;
+    readonly name?: string;
+    readonly displayStrategy: ScoreDisplayStrategy;
+    format(data: TData): string;
+    make(data: TData, options?: {
+        definePassed?: (data: TData) => boolean;
+    }): ScoreItem<TData>;
+}
+declare const Score: {
+    of<TData>(config: {
+        id: string;
+        name?: string;
+        displayStrategy: ScoreDisplayStrategy;
+        format: (data: TData) => string;
+    }): ScoreDef<TData>;
+};
+declare function getScoreById(id: string): ScoreDef<unknown> | undefined;
+
+interface CollectedDataset {
+    id: string;
+    filePath: string;
+    dataset: Dataset;
+}
+interface CollectedEvaluator {
+    id: string;
+    filePath: string;
+    evaluator: Evaluator<unknown, unknown, unknown, unknown>;
+}
+interface CollectedTestCase {
+    id: string;
+    filePath: string;
+    testCase: TestCase<unknown>;
+}
+interface SearchTestCasesQuery {
+    includedTags?: ReadonlyArray<string | RegExp>;
+    excludedTags?: ReadonlyArray<string | RegExp>;
+    includedPaths?: ReadonlyArray<string | RegExp>;
+    excludedPaths?: ReadonlyArray<string | RegExp>;
+}
+interface RunDatasetRequest {
+    datasetId: string;
+    evaluatorIds: ReadonlyArray<string>;
+    concurrency?: number;
+}
+interface RunSnapshot {
+    runId: string;
+    datasetId: string;
+    datasetName: string;
+    evaluatorIds: ReadonlyArray<string>;
+    queuedAt: number;
+    startedAt?: number;
+    finishedAt?: number;
+    totalTestCases: number;
+    completedTestCases: number;
+    passedTestCases: number;
+    failedTestCases: number;
+    status: 'queued' | 'running' | 'completed' | 'failed';
+    artifactPath: string;
+    errorMessage?: string;
+}
+type RunnerEvent = {
+    type: 'RunQueued';
+    runId: string;
+    datasetId: string;
+    datasetName: string;
+    evaluatorIds: ReadonlyArray<string>;
+    totalTestCases: number;
+    artifactPath: string;
+} | {
+    type: 'RunStarted';
+    runId: string;
+    startedAt: number;
+} | {
+    type: 'TestCaseProgress';
+    runId: string;
+    testCaseId: string;
+    testCaseName: string;
+    completedTestCases: number;
+    totalTestCases: number;
+    passed: boolean;
+    durationMs: number;
+    evaluatorScores: ReadonlyArray<{
+        evaluatorId: string;
+        scores: ReadonlyArray<ScoreItem>;
+        passed: boolean;
+        metrics?: ReadonlyArray<MetricItem>;
+    }>;
+    errorMessage?: string;
+} | {
+    type: 'RunCompleted';
+    runId: string;
+    finishedAt: number;
+    passedTestCases: number;
+    failedTestCases: number;
+    totalTestCases: number;
+    artifactPath: string;
+} | {
+    type: 'RunFailed';
+    runId: string;
+    finishedAt: number;
+    errorMessage: string;
+    artifactPath: string;
+} | {
+    type: 'ArtifactFlushed';
+    runId: string;
+    artifactPath: string;
+};
+
+interface SubscribeOptions {
+    runId?: string;
+}
+interface RunnerApi {
+    collectDatasets(): Promise<ReadonlyArray<CollectedDataset>>;
+    collectEvaluators(): Promise<ReadonlyArray<CollectedEvaluator>>;
+    resolveDatasetByName(name: string): Promise<CollectedDataset | undefined>;
+    resolveEvaluatorsByNamePattern(pattern: string): Promise<ReadonlyArray<CollectedEvaluator>>;
+    searchTestCases(query?: SearchTestCasesQuery): Promise<ReadonlyArray<CollectedTestCase>>;
+    collectDatasetTestCases(datasetId: string): Promise<ReadonlyArray<CollectedTestCase>>;
+    runDatasetWith(request: RunDatasetRequest): Promise<RunSnapshot>;
+    subscribeRunEvents(listener: (event: RunnerEvent) => void, options?: SubscribeOptions): () => void;
+    getRunSnapshot(runId: string): RunSnapshot | undefined;
+    getAllRunSnapshots(): ReadonlyArray<RunSnapshot>;
+    shutdown(): Promise<void>;
+}
+declare function createRunner(overrides?: Partial<RunnerConfig>): RunnerApi;
+
+declare function loadMockData(): EvalsData;
+declare function loadRunnerData(runner: RunnerApi): Promise<EvalsData>;
+declare function parseStartupArgs(argv: string[]): StartupArgs;
+
+interface TokenCountData {
+    input?: number;
+    output?: number;
+    inputCached?: number;
+    outputCached?: number;
+}
+declare const tokenCountMetric: MetricDef<TokenCountData>;
+interface LatencyData {
+    ms: number;
+}
+declare const latencyMetric: MetricDef<LatencyData>;
+
+interface PercentScoreData {
+    value: number;
+}
+declare const percentScore: ScoreDef<PercentScoreData>;
+interface BinaryScoreData {
+    passed: boolean;
+}
+declare const binaryScore: ScoreDef<BinaryScoreData>;
+
+export { type BinaryScoreData, type CliState, type CollectedDataset, type CollectedEvaluator, type CollectedTestCase, Dataset, type EvalDataset, type EvalMiddleware, type EvalRun, type EvalsData, Evaluator, type EvaluatorOption, type LatencyData, Metric, type MetricDef, type MetricItem, type PathMatcher, type PercentScoreData, type RunDatasetRequest, type RunSnapshot, type RunnerApi, type RunnerConfig, type RunnerDiscoveryConfig, type RunnerEvent, Score, type ScoreDef, type ScoreDisplayStrategy, type ScoreItem, type SearchTestCasesQuery, type StartupArgs, type TagMatcher, TestCase, type TokenCountData, type ViewLevel, binaryScore, createRunner, defaultRunnerConfig, getMetricById, getScoreById, latencyMetric, loadMockData, loadRunnerData, parseStartupArgs, percentScore, tokenCountMetric, withRunnerConfig };