@aliou/pi-evals 0.0.1

package/dist/index.d.ts

@@ -0,0 +1,344 @@
+/**
+ * Core type definitions for pi-eval
+ */
+/**
+ * Message from the agent conversation.
+ * We use unknown since we just pass these through to scorers.
+ */
+type Message = unknown;
+/**
+ * Pi configuration for running evals
+ */
+interface PiConfig {
+    model: string;
+    provider: string;
+    extensions?: string[];
+    env?: Record<string, string>;
+}
+/**
+ * Global configuration loaded from pi-eval.config.ts
+ */
+interface GlobalConfig {
+    /** Default Pi configuration */
+    defaults?: Partial<PiConfig>;
+    /** Directory containing eval files (default: ./evals) */
+    evalsDir?: string;
+    /** Delay between test cases in ms (default: 500) */
+    delayBetweenTests?: number;
+    /** Default timeout per test in ms (default: 60000) */
+    timeout?: number;
+    /** Warn if more than this many test cases (default: 30) */
+    warnTestCount?: number;
+}
+/**
+ * Setup configuration for test workspace
+ */
+interface TestSetup {
+    /** Files to create in workspace before running */
+    files?: Record<string, string>;
+    /** Shell commands to run before the eval */
+    commands?: string[];
+}
+/**
+ * Expected outcome for scoring
+ */
+interface Expected {
+    /** Expected files and their content (substring match) */
+    files?: Record<string, string>;
+    /** Expected substring in output */
+    output?: string;
+}
+/**
+ * A single test case
+ */
+interface TestCase<TExpected = Expected> {
+    /** Prompt to send to pi */
+    input: string;
+    /** Expected outcome for scorers */
+    expected?: TExpected;
+    /** Optional workspace setup */
+    setup?: TestSetup;
+    /** Run only this test case */
+    only?: boolean;
+    /** Skip this test case */
+    skip?: boolean;
+    /** Override timeout for this case */
+    timeout?: number;
+}
+/**
+ * Token usage statistics
+ */
+interface TokenStats {
+    input: number;
+    output: number;
+    total: number;
+}
+/**
+ * Session statistics from pi
+ */
+interface SessionStats {
+    tokens: TokenStats;
+    cost: number;
+}
+/**
+ * A tool call captured from the session
+ */
+interface ToolCall {
+    name: string;
+    args: Record<string, unknown>;
+}
+/**
+ * Context passed to scorers
+ */
+interface ScoreContext<TExpected = Expected> {
+    /** Original input prompt */
+    input: string;
+    /** Agent's final response text */
+    output: string;
+    /** Expected outcome */
+    expected?: TExpected;
+    /** Workspace directory */
+    cwd: string;
+    /** Full conversation messages */
+    messages: Message[];
+    /** Tool calls made during the session */
+    toolCalls: ToolCall[];
+    /** Token and cost stats */
+    stats: SessionStats;
+}
+/**
+ * Result from a scorer
+ */
+interface ScoreResult {
+    /** Scorer name */
+    name: string;
+    /** Score from 0-1 */
+    score: number;
+    /** Explanation of the score */
+    reason?: string;
+}
+/**
+ * A scorer (evaluator) function
+ */
+interface Scorer<TExpected = Expected> {
+    /** Display name */
+    name: string;
+    /** Scoring function */
+    score: (ctx: ScoreContext<TExpected>) => Promise<ScoreResult>;
+}
+/**
+ * Options for defining an eval
+ */
+interface EvalOptions<TExpected = Expected> {
+    /** Pi configuration */
+    config: PiConfig;
+    /** Test cases */
+    data: TestCase<TExpected>[];
+    /** Scorers to run */
+    scorers: Scorer<TExpected>[];
+    /** Timeout per test case in ms */
+    timeout?: number;
+}
+/**
+ * Internal representation of a registered eval
+ */
+interface EvalDefinition<TExpected = Expected> {
+    /** Eval name */
+    name: string;
+    /** Eval options */
+    options: EvalOptions<TExpected>;
+    /** Source file path */
+    file: string;
+}
+/**
+ * Result of a single test case
+ */
+interface TestResult {
+    /** Eval name */
+    evalName: string;
+    /** Test input */
+    input: string;
+    /** Score results from all scorers */
+    scores: ScoreResult[];
+    /** Whether the test passed (all scores >= 0.5) */
+    passed: boolean;
+    /** Duration in ms */
+    duration: number;
+    /** Token usage */
+    tokens: TokenStats;
+    /** Cost in USD */
+    cost: number;
+    /** Error message if failed */
+    error?: string;
+}
+/**
+ * Summary of an eval run
+ */
+interface EvalRunSummary {
+    /** All test results */
+    results: TestResult[];
+    /** Total tests */
+    total: number;
+    /** Passed tests */
+    passed: number;
+    /** Failed tests */
+    failed: number;
+    /** Total duration in ms */
+    duration: number;
+    /** Total tokens used */
+    totalTokens: number;
+    /** Total cost in USD */
+    totalCost: number;
+}
+/**
+ * CLI options
+ */
+interface CliOptions {
+    /** Filter evals by name substring */
+    filter?: string;
+    /** Output JSON instead of pretty print */
+    json?: boolean;
+    /** Verbose output */
+    verbose?: boolean;
+    /** Minimum pass percentage to exit 0 */
+    threshold?: number;
+    /** Config file path */
+    config?: string;
+    /** Override model */
+    model?: string;
+    /** Override provider */
+    provider?: string;
+}
+
+/**
+ * Helper for defining config with type inference
+ */
+declare function defineConfig(config: GlobalConfig): GlobalConfig;
+
+interface BashOptions {
+    /** Expected exit code (default: 0) */
+    exitCode?: number;
+    /** Timeout in ms (default: 30000) */
+    timeout?: number;
+}
+/**
+ * Creates a scorer that runs a bash command and checks the exit code.
+ * Useful for running tests, linters, or other validation commands.
+ */
+declare function bash(command: string, options?: BashOptions): Scorer<Expected>;
+
+/**
+ * Output contains scorer - checks that output contains expected substring
+ */
+
+/**
+ * Creates a scorer that checks if the agent's output contains expected.output
+ */
+declare function outputContains(): Scorer<Expected>;
+
+/**
+ * Creates a scorer that checks if expected files exist and contain expected content.
+ * Uses substring matching for content comparison.
+ */
+declare function files(): Scorer<Expected>;
+
+/**
+ * LLM Judge scorer - uses an LLM to evaluate the output
+ */
+
+interface LlmJudgeOptions {
+    /** Criteria for the LLM to evaluate against */
+    criteria: string;
+    /** Model to use (default: gpt-4o-mini) */
+    model?: string;
+    /** Provider (default: openai) */
+    provider?: string;
+}
+/**
+ * Creates a scorer that uses an LLM to evaluate the output against criteria.
+ * Uses a cheap, fast model by default.
+ *
+ * Note: Requires OPENAI_API_KEY or appropriate provider API key.
+ */
+declare function llmJudge(options: LlmJudgeOptions): Scorer<Expected>;
+
+/**
+ * Regex scorer - checks that output matches a pattern
+ */
+
+/**
+ * Creates a scorer that checks if the agent's output matches a regex pattern
+ */
+declare function outputMatches(pattern: RegExp): Scorer<Expected>;
+
+/**
+ * Tool called scorer - checks that a specific tool was called during the session
+ */
+
+/**
+ * Creates a scorer that checks if a specific tool was called.
+ *
+ * @param name - The tool name to check for (e.g., "read", "bash", "linkup_web_search")
+ */
+declare function toolCalled(name: string): Scorer<Expected>;
+
+/**
+ * Creates a scorer that checks if a tool was called with specific arguments.
+ *
+ * For `path` arguments, both expected and actual values are resolved to
+ * absolute paths before comparison. All other arguments use direct equality.
+ *
+ * @param name - The tool name to check for
+ * @param expectedArgs - Key-value pairs the tool call args must contain
+ */
+declare function toolCalledWith(name: string, expectedArgs: Record<string, unknown>): Scorer<Expected>;
+
+/**
+ * Built-in scorers for pi-eval
+ */
+
+type ScorersModule_BashOptions = BashOptions;
+type ScorersModule_LlmJudgeOptions = LlmJudgeOptions;
+declare const ScorersModule_bash: typeof bash;
+declare const ScorersModule_files: typeof files;
+declare const ScorersModule_llmJudge: typeof llmJudge;
+declare const ScorersModule_outputContains: typeof outputContains;
+declare const ScorersModule_outputMatches: typeof outputMatches;
+declare const ScorersModule_toolCalled: typeof toolCalled;
+declare const ScorersModule_toolCalledWith: typeof toolCalledWith;
+declare namespace ScorersModule {
+  export { type ScorersModule_BashOptions as BashOptions, type ScorersModule_LlmJudgeOptions as LlmJudgeOptions, ScorersModule_bash as bash, ScorersModule_files as files, ScorersModule_llmJudge as llmJudge, ScorersModule_outputContains as outputContains, ScorersModule_outputMatches as outputMatches, ScorersModule_toolCalled as toolCalled, ScorersModule_toolCalledWith as toolCalledWith };
+}
+
+/**
+ * pi-eval - Eval framework for pi coding agent
+ */
+
+declare const Scorers: typeof ScorersModule;
+
+/**
+ * Define and register an eval.
+ * This is the main API for creating evals.
+ *
+ * @example
+ * ```typescript
+ * import { evaluate, Scorers } from "@aliou/pi-evals";
+ *
+ * evaluate("Create hello file", {
+ *   config: {
+ *     model: "claude-sonnet-4-20250514",
+ *     provider: "anthropic",
+ *   },
+ *   data: [
+ *     {
+ *       input: 'Create a file called hello.txt containing "Hello World"',
+ *       expected: { files: { "hello.txt": "Hello World" } },
+ *     },
+ *   ],
+ *   scorers: [Scorers.files()],
+ * });
+ * ```
+ */
+declare function evaluate<TExpected = Expected>(name: string, options: EvalOptions<TExpected>): void;
+
+export { type BashOptions, type CliOptions, type EvalDefinition, type EvalOptions, type EvalRunSummary, type Expected, type GlobalConfig, type LlmJudgeOptions, type PiConfig, type ScoreContext, type ScoreResult, type Scorer, Scorers, type SessionStats, type TestCase, type TestResult, type TestSetup, type TokenStats, type ToolCall, defineConfig, evaluate };