evalsense 0.3.2 → 0.4.0

package/dist/index-CATqAHNK.d.cts

@@ -0,0 +1,416 @@
+import { L as LLMClient, a as MetricOutput } from './types-D0hzfyKm.cjs';
+
+/**
+ * Type definitions for the LLM metric factory
+ *
+ * Provides a declarative API for creating LLM-based evaluation metrics
+ * with unified record input (no more parallel arrays).
+ */
+
+/**
+ * A record with id and arbitrary fields for evaluation
+ *
+ * All LLM metrics expect unified records where each record contains
+ * all fields needed for evaluation (output, context, query, etc.).
+ *
+ * @example
+ * ```ts
+ * // Hallucination metric (needs output + context)
+ * const records: EvalRecord[] = [
+ *   { id: "1", output: "Paris has 50M people", context: "Paris has 2.1M residents" },
+ *   { id: "2", output: "Berlin is Germany's capital", context: "Berlin is the capital of Germany" },
+ * ];
+ *
+ * // Toxicity metric (needs only output)
+ * const records: EvalRecord[] = [
+ *   { id: "1", output: "Thank you for your question" },
+ * ];
+ * ```
+ */
+interface EvalRecord {
+    id: string;
+    [field: string]: unknown;
+}
+/**
+ * Input field specification for createLLMMetric
+ *
+ * Can be a string (required field) or an object with explicit required flag.
+ *
+ * @example
+ * ```ts
+ * // Required fields
+ * inputs: ["output", "context"]
+ *
+ * // Optional context field
+ * inputs: ["output", { name: "context", required: false }]
+ * ```
+ */
+type InputSpec = string | {
+    name: string;
+    required: boolean;
+};
+/**
+ * Response field type for JSON schema generation
+ */
+type ResponseFieldType = "string" | "number" | "boolean" | "array";
+/**
+ * Label threshold configuration
+ *
+ * Sorted by min descending at runtime to find matching label.
+ *
+ * @example
+ * ```ts
+ * labels: [
+ *   { min: 0.7, label: "high" },
+ *   { min: 0.4, label: "medium" },
+ *   { min: 0, label: "low" },
+ * ]
+ * ```
+ */
+interface LabelThreshold {
+    min: number;
+    label: string;
+}
+/**
+ * Configuration for creating an LLM-based metric
+ *
+ * This declarative configuration replaces 90+ lines of boilerplate
+ * with ~15 lines of configuration.
+ *
+ * @example
+ * ```ts
+ * const answerCorrectness = createLLMMetric({
+ *   name: "answer-correctness",
+ *   inputs: ["output", "reference"],
+ *   prompt: ANSWER_CORRECTNESS_PROMPT,
+ *   responseFields: { score: "number", reasoning: "string" },
+ *   labels: [
+ *     { min: 0.8, label: "correct" },
+ *     { min: 0.5, label: "partial" },
+ *     { min: 0, label: "incorrect" },
+ *   ],
+ * });
+ * ```
+ */
+interface LLMMetricConfig {
+    /**
+     * Metric name - used in MetricOutput and error messages
+     */
+    name: string;
+    /**
+     * Field names to extract from records for prompt filling.
+     *
+     * - string: required field (e.g., "output")
+     * - { name: string, required: boolean }: explicit requirement
+     *
+     * The "output" field is always required and should be first.
+     *
+     * @example
+     * ```ts
+     * // Toxicity: only output needed
+     * inputs: ["output"]
+     *
+     * // Hallucination: output + context
+     * inputs: ["output", "context"]
+     *
+     * // Relevance: output + query
+     * inputs: ["output", "query"]
+     *
+     * // Faithfulness: output + source
+     * inputs: ["output", "source"]
+     *
+     * // Optional context
+     * inputs: ["output", { name: "context", required: false }]
+     * ```
+     */
+    inputs: InputSpec[];
+    /**
+     * Prompt template with {variable} placeholders.
+     *
+     * Variables are filled from record fields using fillPrompt().
+     * Use the same names as specified in `inputs`.
+     *
+     * @example
+     * ```ts
+     * prompt: `
+     * Context: {context}
+     * Output: {output}
+     *
+     * Evaluate for hallucinations...
+     * `
+     * ```
+     */
+    prompt: string;
+    /**
+     * Optional batch prompt template.
+     *
+     * Uses {items} placeholder which receives JSON array of all records.
+     * If not provided, batch mode falls back to per-row evaluation.
+     */
+    batchPrompt?: string;
+    /**
+     * Response fields and their types.
+     *
+     * Generates JSON schema for structured LLM outputs.
+     * All fields listed here become required in the schema.
+     *
+     * @example
+     * ```ts
+     * responseFields: {
+     *   score: "number",
+     *   reasoning: "string",
+     *   categories: "array",
+     * }
+     * ```
+     */
+    responseFields: Record<string, ResponseFieldType>;
+    /**
+     * Which response field to use as the primary score.
+     *
+     * @default "score"
+     */
+    scoreField?: string;
+    /**
+     * Optional label field from response to use directly.
+     *
+     * If specified, uses this field from LLM response as the label
+     * instead of computing from score thresholds.
+     *
+     * @example
+     * ```ts
+     * // Toxicity uses "severity" field directly
+     * labelField: "severity"
+     * ```
+     */
+    labelField?: string;
+    /**
+     * Label thresholds for score-to-label conversion.
+     *
+     * Applied in descending min order. Ignored if labelField is set.
+     *
+     * @example
+     * ```ts
+     * labels: [
+     *   { min: 0.7, label: "high" },
+     *   { min: 0.4, label: "medium" },
+     *   { min: 0, label: "low" },
+     * ]
+     * ```
+     */
+    labels?: LabelThreshold[];
+    /**
+     * Default evaluation mode for this metric.
+     *
+     * - "per-row": One LLM call per record (higher accuracy)
+     * - "batch": Single LLM call for all records (lower cost)
+     *
+     * @default "per-row"
+     */
+    defaultMode?: "per-row" | "batch";
+}
+/**
+ * Options for calling an LLM metric
+ */
+interface LLMMetricOptions {
+    /**
+     * Override the default evaluation mode
+     */
+    evaluationMode?: "per-row" | "batch";
+    /**
+     * Override the global LLM client for this call
+     */
+    llmClient?: LLMClient;
+    /**
+     * Custom prompt template override
+     */
+    customPrompt?: string;
+}
+/**
+ * The resulting metric function from createLLMMetric
+ *
+ * Takes unified records and returns MetricOutput array.
+ *
+ * @example
+ * ```ts
+ * const results = await hallucination([
+ *   { id: "1", output: "Paris has 50M people", context: "Paris has 2.1M residents" },
+ * ]);
+ * ```
+ */
+type LLMMetric = (records: EvalRecord[], options?: LLMMetricOptions) => Promise<MetricOutput[]>;
+
+/**
+ * Hallucination detection metric (LLM-based)
+ *
+ * Detects statements in the output that are not supported by the provided context.
+ * Uses LLM evaluation for accurate hallucination detection.
+ *
+ * @example
+ * ```ts
+ * import { setLLMClient, hallucination } from "evalsense/metrics";
+ *
+ * setLLMClient({ async complete(prompt) { ... } });
+ *
+ * const results = await hallucination([
+ *   { id: "1", output: "The capital of France is Paris.", context: "France is in Europe. Its capital is Paris." },
+ * ]);
+ * ```
+ */
+/**
+ * Detects potential hallucinations by checking if output content
+ * is supported by the provided context.
+ *
+ * Score interpretation:
+ * - 0.0 = No hallucinations (all claims fully supported)
+ * - 0.5 = Some unsupported claims
+ * - 1.0 = Severe hallucinations (most/all claims unsupported)
+ *
+ * Labels:
+ * - "true" = Hallucination detected (score >= 0.5)
+ * - "false" = No hallucination (score < 0.5)
+ *
+ * @example
+ * ```ts
+ * const results = await hallucination([
+ *   { id: "1", output: "Paris has 50M people", context: "Paris has 2.1M residents" },
+ *   { id: "2", output: "Berlin is Germany's capital", context: "Berlin is the capital of Germany" },
+ * ]);
+ *
+ * // With batch mode for lower cost:
+ * const batchResults = await hallucination(records, { evaluationMode: "batch" });
+ * ```
+ */
+declare const hallucination: LLMMetric;
+
+/**
+ * Relevance metric (LLM-based)
+ *
+ * Measures how relevant the output is to the input query.
+ * Uses LLM evaluation for accurate relevance assessment.
+ *
+ * @example
+ * ```ts
+ * import { setLLMClient, relevance } from "evalsense/metrics";
+ *
+ * setLLMClient({ async complete(prompt) { ... } });
+ *
+ * const results = await relevance([
+ *   { id: "1", output: "Paris is the capital of France.", query: "What is the capital of France?" },
+ * ]);
+ * ```
+ */
+/**
+ * Measures the relevance of outputs to their queries.
+ *
+ * Score interpretation:
+ * - 0.0 = Completely irrelevant (doesn't address the query at all)
+ * - 0.5 = Partially relevant (addresses some aspects but misses key points)
+ * - 1.0 = Highly relevant (fully addresses the query)
+ *
+ * Labels:
+ * - "high" = High relevance (score >= 0.7)
+ * - "medium" = Medium relevance (score >= 0.4)
+ * - "low" = Low relevance (score < 0.4)
+ *
+ * @example
+ * ```ts
+ * const results = await relevance([
+ *   { id: "1", output: "Paris is the capital of France.", query: "What is the capital of France?" },
+ *   { id: "2", output: "I like pizza.", query: "What is the weather today?" },
+ * ]);
+ *
+ * // With batch mode for lower cost:
+ * const batchResults = await relevance(records, { evaluationMode: "batch" });
+ * ```
+ */
+declare const relevance: LLMMetric;
+
+/**
+ * Faithfulness metric (LLM-based)
+ *
+ * Measures how faithful the output is to the source material.
+ * Uses LLM evaluation to detect contradictions and misrepresentations.
+ *
+ * @example
+ * ```ts
+ * import { setLLMClient, faithfulness } from "evalsense/metrics";
+ *
+ * setLLMClient({ async complete(prompt) { ... } });
+ *
+ * const results = await faithfulness([
+ *   { id: "1", output: "The document discusses climate change.", source: "This report covers the impacts of climate change on biodiversity." },
+ * ]);
+ * ```
+ */
+/**
+ * Measures the faithfulness of outputs to their source material.
+ *
+ * Score interpretation:
+ * - 0.0 = Unfaithful (contradicts or misrepresents source)
+ * - 0.5 = Partially faithful (some accurate, some distortions)
+ * - 1.0 = Fully faithful (accurate representation of source)
+ *
+ * Labels:
+ * - "high" = High faithfulness (score >= 0.7)
+ * - "medium" = Medium faithfulness (score >= 0.4)
+ * - "low" = Low faithfulness (score < 0.4)
+ *
+ * @example
+ * ```ts
+ * const results = await faithfulness([
+ *   { id: "1", output: "Revenue increased by 15%", source: "Revenue increased by 15% in Q4" },
+ *   { id: "2", output: "Sales dropped sharply", source: "Sales increased modestly" },
+ * ]);
+ *
+ * // With batch mode for lower cost:
+ * const batchResults = await faithfulness(records, { evaluationMode: "batch" });
+ * ```
+ */
+declare const faithfulness: LLMMetric;
+
+/**
+ * Toxicity detection metric (LLM-based)
+ *
+ * Detects potentially toxic, harmful, or inappropriate content.
+ * Uses LLM evaluation for nuanced toxicity detection.
+ *
+ * @example
+ * ```ts
+ * import { setLLMClient, toxicity } from "evalsense/metrics";
+ *
+ * setLLMClient({ async complete(prompt) { ... } });
+ *
+ * const results = await toxicity([
+ *   { id: "1", output: "This is a friendly message." },
+ * ]);
+ * ```
+ */
+/**
+ * Detects potential toxicity in outputs.
+ *
+ * Score interpretation:
+ * - 0.0 = No toxicity (completely safe, appropriate)
+ * - 0.3 = Mild (minor profanity or slightly inappropriate)
+ * - 0.6 = Moderate (clear toxic elements)
+ * - 1.0 = Severe (extremely toxic, harmful, or dangerous)
+ *
+ * Labels (from LLM's severity field):
+ * - "none" = No toxicity
+ * - "mild" = Mild toxicity
+ * - "moderate" = Moderate toxicity
+ * - "severe" = Severe toxicity
+ *
+ * @example
+ * ```ts
+ * const results = await toxicity([
+ *   { id: "1", output: "Thank you for your question" },
+ *   { id: "2", output: "That's a stupid question" },
+ * ]);
+ *
+ * // With batch mode for lower cost:
+ * const batchResults = await toxicity(records, { evaluationMode: "batch" });
+ * ```
+ */
+declare const toxicity: LLMMetric;
+
+export { type EvalRecord as E, type InputSpec as I, type LLMMetricConfig as L, type ResponseFieldType as R, type LLMMetric as a, type LLMMetricOptions as b, type LabelThreshold as c, faithfulness as f, hallucination as h, relevance as r, toxicity as t };