evalsense 0.2.0

package/dist/metrics/index.d.cts

@@ -0,0 +1,299 @@
+export { FaithfulnessConfig, HallucinationConfig, RelevanceConfig, ToxicityConfig, faithfulness, hallucination, relevance, toxicity } from './opinionated/index.cjs';
+import { M as MetricFn, a as MetricConfig, b as MetricOutput, L as LLMClient, J as JSONSchema } from '../types-C71p0wzM.cjs';
+
+/**
+ * Custom metric registration
+ */
+
+/**
+ * Registers a custom metric
+ *
+ * @example
+ * ```ts
+ * registerMetric("custom-relevance", async ({ outputs, query }) => {
+ *   // Custom evaluation logic
+ *   return outputs.map(o => ({
+ *     id: o.id,
+ *     metric: "custom-relevance",
+ *     score: evaluateRelevance(o.output, query),
+ *   }));
+ * });
+ * ```
+ */
+declare function registerMetric(name: string, fn: MetricFn): void;
+/**
+ * Gets a registered custom metric
+ */
+declare function getMetric(name: string): MetricFn | undefined;
+/**
+ * Runs a registered metric
+ */
+declare function runMetric(name: string, config: MetricConfig): Promise<MetricOutput[]>;
+/**
+ * Lists all registered custom metrics
+ */
+declare function listMetrics(): string[];
+/**
+ * Unregisters a metric (mainly for testing)
+ */
+declare function unregisterMetric(name: string): boolean;
+/**
+ * Clears all registered metrics (mainly for testing)
+ */
+declare function clearMetrics(): void;
+/**
+ * Creates a simple string-matching metric
+ */
+declare function createPatternMetric(name: string, patterns: RegExp[], options?: {
+    matchScore?: number;
+    noMatchScore?: number;
+}): MetricFn;
+/**
+ * Creates a keyword-based metric
+ */
+declare function createKeywordMetric(name: string, keywords: string[], options?: {
+    caseSensitive?: boolean;
+    threshold?: number;
+}): MetricFn;
+
+/**
+ * Metric utilities
+ */
+
+/**
+ * Normalizes a score to 0-1 range
+ */
+declare function normalizeScore(score: number, min?: number, max?: number): number;
+/**
+ * Converts a numeric score to a label based on thresholds
+ */
+declare function scoreToLabel(score: number, thresholds: {
+    label: string;
+    min: number;
+}[]): string;
+/**
+ * Creates a metric output from a score
+ */
+declare function createMetricOutput(id: string, metric: string, score: number, labelThresholds?: {
+    label: string;
+    min: number;
+}[]): MetricOutput;
+/**
+ * Default thresholds for binary metrics
+ */
+declare const BINARY_THRESHOLDS: {
+    label: string;
+    min: number;
+}[];
+/**
+ * Default thresholds for severity metrics
+ */
+declare const SEVERITY_THRESHOLDS: {
+    label: string;
+    min: number;
+}[];
+/**
+ * Batches items for parallel processing
+ */
+declare function batch<T>(items: T[], size: number): T[][];
+/**
+ * Delays execution
+ */
+declare function delay(ms: number): Promise<void>;
+
+/**
+ * LLM client management for metric evaluation
+ *
+ * Provides a global LLM client that can be configured once and used
+ * across all LLM-based metrics, with support for per-call overrides.
+ */
+
+/**
+ * Sets the global LLM client for all metrics
+ *
+ * @example
+ * ```ts
+ * import { setLLMClient } from "evalsense/metrics";
+ *
+ * setLLMClient({
+ *   async complete(prompt) {
+ *     return await yourLLM.generate(prompt);
+ *   }
+ * });
+ * ```
+ */
+declare function setLLMClient(client: LLMClient): void;
+/**
+ * Gets the current global LLM client
+ *
+ * @returns The global client or null if not set
+ */
+declare function getLLMClient(): LLMClient | null;
+/**
+ * Resets the global LLM client
+ *
+ * Useful for testing or switching between different LLM providers.
+ */
+declare function resetLLMClient(): void;
+/**
+ * Validates that an LLM client is available
+ *
+ * @param client - Optional client override
+ * @param metricName - Name of the metric for error messages
+ * @throws Error if no client is configured
+ * @returns The client to use (override or global)
+ */
+declare function requireLLMClient(client: LLMClient | undefined, metricName: string): LLMClient;
+
+/**
+ * Utilities for LLM-based metric evaluation
+ *
+ * Provides helpers for prompt templating, response parsing, validation, and error handling.
+ */
+
+/**
+ * Fills a prompt template with variables
+ *
+ * @example
+ * ```ts
+ * const prompt = fillPrompt(
+ *   "Context: {context}\nOutput: {output}",
+ *   { context: "Paris is the capital", output: "France's capital is Paris" }
+ * );
+ * ```
+ */
+declare function fillPrompt(template: string, variables: Record<string, string>): string;
+/**
+ * Parses a JSON response from an LLM, with fallback handling
+ *
+ * Handles:
+ * - Plain JSON strings
+ * - JSON wrapped in markdown code blocks
+ * - Malformed JSON with helpful error messages
+ *
+ * @example
+ * ```ts
+ * const result = parseJSONResponse<{ score: number }>(llmResponse);
+ * ```
+ */
+declare function parseJSONResponse<T>(response: string): T;
+/**
+ * Validates that a parsed JSON response has required fields
+ *
+ * @example
+ * ```ts
+ * validateResponse(result, ["score", "reasoning"], "hallucination");
+ * ```
+ */
+declare function validateResponse(response: unknown, requiredFields: string[], metricName: string): void;
+/**
+ * Extracts a score from various formats (number, string, object with score field)
+ */
+declare function extractScore(value: unknown, defaultScore?: number): number;
+/**
+ * Creates a JSON schema for structured LLM outputs
+ *
+ * @example
+ * ```ts
+ * const schema = createJSONSchema({
+ *   score: "number",
+ *   reasoning: "string"
+ * });
+ * ```
+ */
+declare function createJSONSchema(properties: Record<string, string>, required?: string[]): JSONSchema;
+/**
+ * Batches an array of items into chunks
+ *
+ * Useful for batch evaluation mode to control batch size.
+ */
+declare function batchItems<T>(items: T[], batchSize: number): T[][];
+/**
+ * Creates a consistent error message for LLM metric failures
+ */
+declare function createLLMError(metricName: string, operation: string, error: unknown, context?: {
+    id?: string;
+    index?: number;
+}): Error;
+/**
+ * Waits for a promise with a timeout
+ */
+declare function withTimeout<T>(promise: Promise<T>, timeoutMs: number, operation: string): Promise<T>;
+
+/**
+ * Mock LLM client for testing
+ *
+ * Provides a configurable mock implementation of LLMClient for unit tests.
+ */
+
+/**
+ * Configuration for mock LLM client
+ */
+interface MockLLMConfig {
+    /** Fixed response to return (can be string or object for JSON mode) */
+    response?: string | Record<string, unknown>;
+    /** Multiple responses for sequential calls */
+    responses?: Array<string | Record<string, unknown>>;
+    /** Delay in milliseconds before responding */
+    delay?: number;
+    /** Whether to throw an error */
+    shouldError?: boolean;
+    /** Error message to throw */
+    errorMessage?: string;
+    /** Function to validate prompts */
+    onPrompt?: (prompt: string) => void;
+}
+/**
+ * Creates a mock LLM client for testing
+ *
+ * @example
+ * ```ts
+ * const mock = createMockLLMClient({
+ *   response: JSON.stringify({ score: 0.8, reasoning: "test" }),
+ *   delay: 100
+ * });
+ *
+ * setLLMClient(mock);
+ * ```
+ */
+declare function createMockLLMClient(config?: MockLLMConfig): LLMClient;
+/**
+ * Creates a mock client that returns sequential responses
+ *
+ * Useful for testing multiple calls with different responses.
+ *
+ * @example
+ * ```ts
+ * const mock = createSequentialMockClient([
+ *   { score: 0.2, reasoning: "First call" },
+ *   { score: 0.8, reasoning: "Second call" }
+ * ]);
+ * ```
+ */
+declare function createSequentialMockClient(responses: Array<string | Record<string, unknown>>, options?: {
+    delay?: number;
+}): LLMClient;
+/**
+ * Creates a mock client that always errors
+ *
+ * Useful for testing error handling.
+ */
+declare function createErrorMockClient(errorMessage?: string): LLMClient;
+/**
+ * Creates a spy mock client that records all prompts
+ *
+ * Useful for testing what prompts are being sent to the LLM.
+ *
+ * @example
+ * ```ts
+ * const { client, prompts } = createSpyMockClient({ score: 0.5 });
+ * await metric({ outputs, context, llmClient: client });
+ * console.log(prompts); // See all prompts that were sent
+ * ```
+ */
+declare function createSpyMockClient(response: string | Record<string, unknown>): {
+    client: LLMClient;
+    prompts: string[];
+};
+
+export { BINARY_THRESHOLDS, SEVERITY_THRESHOLDS, batch, batchItems, clearMetrics, createErrorMockClient, createJSONSchema, createKeywordMetric, createLLMError, createMetricOutput, createMockLLMClient, createPatternMetric, createSequentialMockClient, createSpyMockClient, delay, extractScore, fillPrompt, getLLMClient, getMetric, listMetrics, normalizeScore, parseJSONResponse, registerMetric, requireLLMClient, resetLLMClient, runMetric, scoreToLabel, setLLMClient, unregisterMetric, validateResponse, withTimeout };

package/dist/metrics/index.d.ts

@@ -0,0 +1,299 @@
+export { FaithfulnessConfig, HallucinationConfig, RelevanceConfig, ToxicityConfig, faithfulness, hallucination, relevance, toxicity } from './opinionated/index.js';
+import { M as MetricFn, a as MetricConfig, b as MetricOutput, L as LLMClient, J as JSONSchema } from '../types-C71p0wzM.js';
+
+/**
+ * Custom metric registration
+ */
+
+/**
+ * Registers a custom metric
+ *
+ * @example
+ * ```ts
+ * registerMetric("custom-relevance", async ({ outputs, query }) => {
+ *   // Custom evaluation logic
+ *   return outputs.map(o => ({
+ *     id: o.id,
+ *     metric: "custom-relevance",
+ *     score: evaluateRelevance(o.output, query),
+ *   }));
+ * });
+ * ```
+ */
+declare function registerMetric(name: string, fn: MetricFn): void;
+/**
+ * Gets a registered custom metric
+ */
+declare function getMetric(name: string): MetricFn | undefined;
+/**
+ * Runs a registered metric
+ */
+declare function runMetric(name: string, config: MetricConfig): Promise<MetricOutput[]>;
+/**
+ * Lists all registered custom metrics
+ */
+declare function listMetrics(): string[];
+/**
+ * Unregisters a metric (mainly for testing)
+ */
+declare function unregisterMetric(name: string): boolean;
+/**
+ * Clears all registered metrics (mainly for testing)
+ */
+declare function clearMetrics(): void;
+/**
+ * Creates a simple string-matching metric
+ */
+declare function createPatternMetric(name: string, patterns: RegExp[], options?: {
+    matchScore?: number;
+    noMatchScore?: number;
+}): MetricFn;
+/**
+ * Creates a keyword-based metric
+ */
+declare function createKeywordMetric(name: string, keywords: string[], options?: {
+    caseSensitive?: boolean;
+    threshold?: number;
+}): MetricFn;
+
+/**
+ * Metric utilities
+ */
+
+/**
+ * Normalizes a score to 0-1 range
+ */
+declare function normalizeScore(score: number, min?: number, max?: number): number;
+/**
+ * Converts a numeric score to a label based on thresholds
+ */
+declare function scoreToLabel(score: number, thresholds: {
+    label: string;
+    min: number;
+}[]): string;
+/**
+ * Creates a metric output from a score
+ */
+declare function createMetricOutput(id: string, metric: string, score: number, labelThresholds?: {
+    label: string;
+    min: number;
+}[]): MetricOutput;
+/**
+ * Default thresholds for binary metrics
+ */
+declare const BINARY_THRESHOLDS: {
+    label: string;
+    min: number;
+}[];
+/**
+ * Default thresholds for severity metrics
+ */
+declare const SEVERITY_THRESHOLDS: {
+    label: string;
+    min: number;
+}[];
+/**
+ * Batches items for parallel processing
+ */
+declare function batch<T>(items: T[], size: number): T[][];
+/**
+ * Delays execution
+ */
+declare function delay(ms: number): Promise<void>;
+
+/**
+ * LLM client management for metric evaluation
+ *
+ * Provides a global LLM client that can be configured once and used
+ * across all LLM-based metrics, with support for per-call overrides.
+ */
+
+/**
+ * Sets the global LLM client for all metrics
+ *
+ * @example
+ * ```ts
+ * import { setLLMClient } from "evalsense/metrics";
+ *
+ * setLLMClient({
+ *   async complete(prompt) {
+ *     return await yourLLM.generate(prompt);
+ *   }
+ * });
+ * ```
+ */
+declare function setLLMClient(client: LLMClient): void;
+/**
+ * Gets the current global LLM client
+ *
+ * @returns The global client or null if not set
+ */
+declare function getLLMClient(): LLMClient | null;
+/**
+ * Resets the global LLM client
+ *
+ * Useful for testing or switching between different LLM providers.
+ */
+declare function resetLLMClient(): void;
+/**
+ * Validates that an LLM client is available
+ *
+ * @param client - Optional client override
+ * @param metricName - Name of the metric for error messages
+ * @throws Error if no client is configured
+ * @returns The client to use (override or global)
+ */
+declare function requireLLMClient(client: LLMClient | undefined, metricName: string): LLMClient;
+
+/**
+ * Utilities for LLM-based metric evaluation
+ *
+ * Provides helpers for prompt templating, response parsing, validation, and error handling.
+ */
+
+/**
+ * Fills a prompt template with variables
+ *
+ * @example
+ * ```ts
+ * const prompt = fillPrompt(
+ *   "Context: {context}\nOutput: {output}",
+ *   { context: "Paris is the capital", output: "France's capital is Paris" }
+ * );
+ * ```
+ */
+declare function fillPrompt(template: string, variables: Record<string, string>): string;
+/**
+ * Parses a JSON response from an LLM, with fallback handling
+ *
+ * Handles:
+ * - Plain JSON strings
+ * - JSON wrapped in markdown code blocks
+ * - Malformed JSON with helpful error messages
+ *
+ * @example
+ * ```ts
+ * const result = parseJSONResponse<{ score: number }>(llmResponse);
+ * ```
+ */
+declare function parseJSONResponse<T>(response: string): T;
+/**
+ * Validates that a parsed JSON response has required fields
+ *
+ * @example
+ * ```ts
+ * validateResponse(result, ["score", "reasoning"], "hallucination");
+ * ```
+ */
+declare function validateResponse(response: unknown, requiredFields: string[], metricName: string): void;
+/**
+ * Extracts a score from various formats (number, string, object with score field)
+ */
+declare function extractScore(value: unknown, defaultScore?: number): number;
+/**
+ * Creates a JSON schema for structured LLM outputs
+ *
+ * @example
+ * ```ts
+ * const schema = createJSONSchema({
+ *   score: "number",
+ *   reasoning: "string"
+ * });
+ * ```
+ */
+declare function createJSONSchema(properties: Record<string, string>, required?: string[]): JSONSchema;
+/**
+ * Batches an array of items into chunks
+ *
+ * Useful for batch evaluation mode to control batch size.
+ */
+declare function batchItems<T>(items: T[], batchSize: number): T[][];
+/**
+ * Creates a consistent error message for LLM metric failures
+ */
+declare function createLLMError(metricName: string, operation: string, error: unknown, context?: {
+    id?: string;
+    index?: number;
+}): Error;
+/**
+ * Waits for a promise with a timeout
+ */
+declare function withTimeout<T>(promise: Promise<T>, timeoutMs: number, operation: string): Promise<T>;
+
+/**
+ * Mock LLM client for testing
+ *
+ * Provides a configurable mock implementation of LLMClient for unit tests.
+ */
+
+/**
+ * Configuration for mock LLM client
+ */
+interface MockLLMConfig {
+    /** Fixed response to return (can be string or object for JSON mode) */
+    response?: string | Record<string, unknown>;
+    /** Multiple responses for sequential calls */
+    responses?: Array<string | Record<string, unknown>>;
+    /** Delay in milliseconds before responding */
+    delay?: number;
+    /** Whether to throw an error */
+    shouldError?: boolean;
+    /** Error message to throw */
+    errorMessage?: string;
+    /** Function to validate prompts */
+    onPrompt?: (prompt: string) => void;
+}
+/**
+ * Creates a mock LLM client for testing
+ *
+ * @example
+ * ```ts
+ * const mock = createMockLLMClient({
+ *   response: JSON.stringify({ score: 0.8, reasoning: "test" }),
+ *   delay: 100
+ * });
+ *
+ * setLLMClient(mock);
+ * ```
+ */
+declare function createMockLLMClient(config?: MockLLMConfig): LLMClient;
+/**
+ * Creates a mock client that returns sequential responses
+ *
+ * Useful for testing multiple calls with different responses.
+ *
+ * @example
+ * ```ts
+ * const mock = createSequentialMockClient([
+ *   { score: 0.2, reasoning: "First call" },
+ *   { score: 0.8, reasoning: "Second call" }
+ * ]);
+ * ```
+ */
+declare function createSequentialMockClient(responses: Array<string | Record<string, unknown>>, options?: {
+    delay?: number;
+}): LLMClient;
+/**
+ * Creates a mock client that always errors
+ *
+ * Useful for testing error handling.
+ */
+declare function createErrorMockClient(errorMessage?: string): LLMClient;
+/**
+ * Creates a spy mock client that records all prompts
+ *
+ * Useful for testing what prompts are being sent to the LLM.
+ *
+ * @example
+ * ```ts
+ * const { client, prompts } = createSpyMockClient({ score: 0.5 });
+ * await metric({ outputs, context, llmClient: client });
+ * console.log(prompts); // See all prompts that were sent
+ * ```
+ */
+declare function createSpyMockClient(response: string | Record<string, unknown>): {
+    client: LLMClient;
+    prompts: string[];
+};
+
+export { BINARY_THRESHOLDS, SEVERITY_THRESHOLDS, batch, batchItems, clearMetrics, createErrorMockClient, createJSONSchema, createKeywordMetric, createLLMError, createMetricOutput, createMockLLMClient, createPatternMetric, createSequentialMockClient, createSpyMockClient, delay, extractScore, fillPrompt, getLLMClient, getMetric, listMetrics, normalizeScore, parseJSONResponse, registerMetric, requireLLMClient, resetLLMClient, runMetric, scoreToLabel, setLLMClient, unregisterMetric, validateResponse, withTimeout };