evalkit 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,402 @@
+/** Standard result returned by every evaluator. */
+interface EvalResult {
+    /** Evaluator identifier */
+    key: string;
+    /** Binary pass/fail */
+    passed: boolean;
+    /** Human-readable explanation */
+    details: string;
+}
+/** Result for tool selection check. */
+interface ToolSelectionResult extends EvalResult {
+    key: 'tool_selection';
+    expected: string[];
+    actual: string[];
+}
+/** Result for content match check. */
+interface ContentMatchResult extends EvalResult {
+    key: 'content_match';
+    missing: string[];
+}
+/** Result for negative match check. */
+interface NegativeMatchResult extends EvalResult {
+    key: 'negative_match';
+    found: string[];
+}
+/** Result for latency check. */
+interface LatencyResult extends EvalResult {
+    key: 'latency';
+    ms: number;
+    threshold: number;
+}
+/** Result for JSON validity check. */
+interface JsonValidResult extends EvalResult {
+    key: 'json_valid';
+}
+/** Result for schema match check. */
+interface SchemaMatchResult extends EvalResult {
+    key: 'schema_match';
+    missingKeys: string[];
+    typeErrors: string[];
+}
+/** Result for non-empty check. */
+interface NonEmptyResult extends EvalResult {
+    key: 'non_empty';
+}
+/** Result for length bounds check. */
+interface LengthBoundsResult extends EvalResult {
+    key: 'length_bounds';
+    length: number;
+    min?: number;
+    max?: number;
+}
+/** Result for regex match check. */
+interface RegexMatchResult extends EvalResult {
+    key: 'regex_match';
+    failedPatterns: string[];
+}
+/** Result for tool call count check. */
+interface ToolCallCountResult extends EvalResult {
+    key: 'tool_call_count';
+    count: number;
+    min?: number;
+    max?: number;
+}
+/** Result for cost budget check. */
+interface CostBudgetResult extends EvalResult {
+    key: 'cost_budget';
+    actual: number;
+    budget: number;
+}
+/** Summary result from runChecks(). */
+interface CheckSuiteResult {
+    passed: boolean;
+    results: EvalResult[];
+    summary: string;
+}
+
+interface ToolSelectionInput {
+    expected: string[];
+    actual: string[];
+}
+/**
+ * Strict set equality between expected and actual tool names.
+ * Order-independent, deduplicates. Reports missing and unexpected tools.
+ */
+declare function toolSelection(input: ToolSelectionInput): ToolSelectionResult;
+/**
+ * Factory: creates a reusable toolSelection evaluator with fixed expected tools.
+ */
+declare function createToolSelectionEvaluator(config: {
+    expected: string[];
+}): (input: {
+    actual: string[];
+}) => ToolSelectionResult;
+
+interface ContentMatchInput {
+    responseText: string;
+    mustContain: string[];
+}
+/**
+ * Case-insensitive substring matching: all mustContain strings must appear in responseText.
+ */
+declare function contentMatch(input: ContentMatchInput): ContentMatchResult;
+/**
+ * Factory: creates a reusable contentMatch evaluator with fixed mustContain list.
+ */
+declare function createContentMatchEvaluator(config: {
+    mustContain: string[];
+}): (input: {
+    responseText: string;
+}) => ContentMatchResult;
+
+interface NegativeMatchInput {
+    responseText: string;
+    mustNotContain: string[];
+}
+/**
+ * Case-insensitive substring matching: no mustNotContain strings may appear in responseText.
+ */
+declare function negativeMatch(input: NegativeMatchInput): NegativeMatchResult;
+/**
+ * Factory: creates a reusable negativeMatch evaluator with fixed mustNotContain list.
+ */
+declare function createNegativeMatchEvaluator(config: {
+    mustNotContain: string[];
+}): (input: {
+    responseText: string;
+}) => NegativeMatchResult;
+
+interface LatencyInput {
+    latencyMs: number;
+    thresholdMs?: number;
+}
+/**
+ * Simple threshold check: latencyMs <= thresholdMs.
+ * Default threshold: 20,000ms.
+ */
+declare function latency(input: LatencyInput): LatencyResult;
+/**
+ * Factory: creates a reusable latency evaluator with a fixed threshold.
+ */
+declare function createLatencyEvaluator(config: {
+    thresholdMs: number;
+}): (input: {
+    latencyMs: number;
+}) => LatencyResult;
+
+interface JsonValidInput {
+    text: string;
+    /** If true, the parsed result must be an object or array (not a bare primitive). */
+    requireObject?: boolean;
+}
+/**
+ * Validates that a string is parseable JSON.
+ * Optionally checks that the parsed result is an object or array.
+ */
+declare function jsonValid(input: JsonValidInput): JsonValidResult;
+/**
+ * Factory: creates a reusable jsonValid evaluator with fixed options.
+ */
+declare function createJsonValidEvaluator(config?: {
+    requireObject?: boolean;
+}): (input: {
+    text: string;
+}) => JsonValidResult;
+
+interface SchemaMatchInput {
+    /** The object to validate. */
+    data: Record<string, unknown>;
+    /** Keys that must be present. */
+    requiredKeys: string[];
+    /** Optional type constraints: key -> expected typeof result. */
+    typeChecks?: Record<string, string>;
+}
+/**
+ * Validates that a parsed object contains required keys and optionally checks value types.
+ * Zero-dep — just checks key presence and typeof.
+ */
+declare function schemaMatch(input: SchemaMatchInput): SchemaMatchResult;
+/**
+ * Factory: creates a reusable schemaMatch evaluator with fixed schema expectations.
+ */
+declare function createSchemaMatchEvaluator(config: {
+    requiredKeys: string[];
+    typeChecks?: Record<string, string>;
+}): (input: {
+    data: Record<string, unknown>;
+}) => SchemaMatchResult;
+
+interface NonEmptyInput {
+    responseText: string;
+    /** Custom cop-out phrases to check against (case-insensitive exact match after trim). */
+    copOutPhrases?: string[];
+}
+/**
+ * Checks that the response is not empty, not just whitespace, and not a cop-out phrase.
+ */
+declare function nonEmpty(input: NonEmptyInput): NonEmptyResult;
+/**
+ * Factory: creates a reusable nonEmpty evaluator with fixed cop-out phrases.
+ */
+declare function createNonEmptyEvaluator(config?: {
+    copOutPhrases?: string[];
+}): (input: {
+    responseText: string;
+}) => NonEmptyResult;
+
+interface LengthBoundsInput {
+    responseText: string;
+    min?: number;
+    max?: number;
+}
+/**
+ * Checks that the response length (character count) falls within min and/or max bounds.
+ */
+declare function lengthBounds(input: LengthBoundsInput): LengthBoundsResult;
+/**
+ * Factory: creates a reusable lengthBounds evaluator with fixed bounds.
+ */
+declare function createLengthBoundsEvaluator(config: {
+    min?: number;
+    max?: number;
+}): (input: {
+    responseText: string;
+}) => LengthBoundsResult;
+
+interface RegexMatchInput {
+    responseText: string;
+    patterns: (string | RegExp)[];
+    /** If 'all' (default), every pattern must match. If 'any', at least one must match. */
+    mode?: 'all' | 'any';
+}
+/**
+ * Tests the response against one or more regex patterns.
+ */
+declare function regexMatch(input: RegexMatchInput): RegexMatchResult;
+/**
+ * Factory: creates a reusable regexMatch evaluator with fixed patterns and mode.
+ */
+declare function createRegexMatchEvaluator(config: {
+    patterns: (string | RegExp)[];
+    mode?: 'all' | 'any';
+}): (input: {
+    responseText: string;
+}) => RegexMatchResult;
+
+interface ToolCallCountInput {
+    count: number;
+    min?: number;
+    max?: number;
+}
+/**
+ * Checks that the number of tool calls falls within expected min/max bounds.
+ */
+declare function toolCallCount(input: ToolCallCountInput): ToolCallCountResult;
+/**
+ * Factory: creates a reusable toolCallCount evaluator with fixed bounds.
+ */
+declare function createToolCallCountEvaluator(config: {
+    min?: number;
+    max?: number;
+}): (input: {
+    count: number;
+}) => ToolCallCountResult;
+
+interface CostBudgetInput {
+    /** Actual cost — can be token count or dollar amount. */
+    actual: number;
+    /** Budget threshold — same unit as actual. */
+    budget: number;
+}
+/**
+ * Checks that token count or dollar cost stays under a threshold.
+ */
+declare function costBudget(input: CostBudgetInput): CostBudgetResult;
+/**
+ * Factory: creates a reusable costBudget evaluator with a fixed budget.
+ */
+declare function createCostBudgetEvaluator(config: {
+    budget: number;
+}): (input: {
+    actual: number;
+}) => CostBudgetResult;
+
+interface RunChecksInput {
+    responseText?: string;
+    expectedTools?: string[];
+    actualTools?: string[];
+    mustContain?: string[];
+    mustNotContain?: string[];
+    latencyMs?: number;
+    thresholdMs?: number;
+    json?: {
+        text: string;
+        requireObject?: boolean;
+    };
+    schema?: {
+        data: Record<string, unknown>;
+        requiredKeys: string[];
+        typeChecks?: Record<string, string>;
+    };
+    copOutPhrases?: string[];
+    lengthMin?: number;
+    lengthMax?: number;
+    regexPatterns?: (string | RegExp)[];
+    regexMode?: 'all' | 'any';
+    toolCallCountValue?: number;
+    toolCallMin?: number;
+    toolCallMax?: number;
+    costActual?: number;
+    costBudget?: number;
+}
+/**
+ * Runs any combination of checks at once and returns a summary.
+ * Only runs checks for which the relevant inputs are provided.
+ */
+declare function runChecks(input: RunChecksInput): CheckSuiteResult;
+
+/** TypeScript types for the YAML/JSON test case format. */
+interface TestCase {
+    id: string;
+    query: string;
+    checks: TestCaseChecks;
+    metadata?: Record<string, unknown>;
+}
+interface TestCaseChecks {
+    expectedTools?: string[];
+    mustContain?: string[];
+    mustNotContain?: string[];
+    thresholdMs?: number;
+    json?: {
+        requireObject?: boolean;
+    };
+    schema?: {
+        requiredKeys: string[];
+        typeChecks?: Record<string, string>;
+    };
+    copOutPhrases?: string[];
+    lengthMin?: number;
+    lengthMax?: number;
+    regexPatterns?: string[];
+    regexMode?: 'all' | 'any';
+    toolCallMin?: number;
+    toolCallMax?: number;
+    costBudget?: number;
+}
+interface SuiteConfig {
+    name?: string;
+    test_cases: TestCase[];
+}
+
+interface AgentResult {
+    responseText: string;
+    actualTools?: string[];
+    latencyMs?: number;
+    toolCallCount?: number;
+    cost?: number;
+}
+type AgentFn = (query: string) => Promise<AgentResult>;
+interface CaseResult {
+    id: string;
+    query: string;
+    passed: boolean;
+    checks: CheckSuiteResult;
+    metadata?: Record<string, unknown>;
+    agentResult: AgentResult;
+}
+interface SuiteResult {
+    name: string;
+    passed: number;
+    failed: number;
+    total: number;
+    cases: CaseResult[];
+    duration: number;
+}
+
+/** Parse a YAML string into a SuiteConfig. */
+declare function parseYaml(content: string): SuiteConfig;
+/** Parse a JSON string into a SuiteConfig. */
+declare function parseJson(content: string): SuiteConfig;
+/** Read a file from disk and parse by extension. */
+declare function loadFile(filePath: string): SuiteConfig;
+/**
+ * Main entry point. Accepts a file path (string) or an inline SuiteConfig.
+ * If a string is provided, delegates to loadFile.
+ */
+declare function loadCases(source: string | SuiteConfig): SuiteConfig;
+
+interface RunSuiteOptions {
+    cases: string | SuiteConfig;
+    agent: AgentFn;
+    name?: string;
+    concurrency?: number;
+    onCaseComplete?: (result: CaseResult) => void;
+}
+/** Run a suite of test cases against an agent function. */
+declare function runSuite(options: RunSuiteOptions): Promise<SuiteResult>;
+
+/** Prints a human-readable summary of a suite run to the console. */
+declare function printSuiteResult(result: SuiteResult): void;
+
+export { type AgentFn, type AgentResult, type CaseResult, type CheckSuiteResult, type ContentMatchResult, type CostBudgetResult, type EvalResult, type JsonValidResult, type LatencyResult, type LengthBoundsResult, type NegativeMatchResult, type NonEmptyResult, type RegexMatchResult, type RunSuiteOptions, type SchemaMatchResult, type SuiteConfig, type SuiteResult, type TestCase, type TestCaseChecks, type ToolCallCountResult, type ToolSelectionResult, contentMatch, costBudget, createContentMatchEvaluator, createCostBudgetEvaluator, createJsonValidEvaluator, createLatencyEvaluator, createLengthBoundsEvaluator, createNegativeMatchEvaluator, createNonEmptyEvaluator, createRegexMatchEvaluator, createSchemaMatchEvaluator, createToolCallCountEvaluator, createToolSelectionEvaluator, jsonValid, latency, lengthBounds, loadCases, loadFile, negativeMatch, nonEmpty, parseJson, parseYaml, printSuiteResult, regexMatch, runChecks, runSuite, schemaMatch, toolCallCount, toolSelection };