judgeval 0.2.4 → 0.2.5

package/dist/types/exporters/otel-exporter.d.ts

@@ -0,0 +1,16 @@
+import { ExportResult } from '@opentelemetry/core';
+import { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-base";
+interface OtelExporterConfig {
+    apiKey: string;
+    organizationId: string;
+    serviceName?: string;
+}
+export declare class JudgevalExporter implements SpanExporter {
+    private serviceName;
+    private apiKey;
+    private organizationId;
+    constructor(config: OtelExporterConfig);
+    export(spans: ReadableSpan[], resultCallback: (result: ExportResult) => void): Promise<void>;
+    shutdown(): Promise<void>;
+}
+export {};

package/dist/types/judges/index.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Interface for judge models that can generate text
+ */
+export interface Judge {
+    /**
+     * Generate text synchronously
+     */
+    generate(prompt: string): string;
+    /**
+     * Generate text asynchronously
+     */
+    aGenerate(prompt: string): Promise<string>;
+    /**
+     * Get the name of the model
+     */
+    getModelName(): string;
+}
+/**
+ * Default judge implementation using OpenAI API
+ */
+export declare class DefaultJudge implements Judge {
+    private modelName;
+    private apiKey?;
+    private user?;
+    constructor(modelName?: string, apiKey?: string, user?: string);
+    generate(prompt: string): string;
+    aGenerate(prompt: string): Promise<string>;
+    getModelName(): string;
+}
+/**
+ * Together AI judge implementation
+ */
+export declare class TogetherJudge implements Judge {
+    private modelName;
+    private apiKey?;
+    constructor(modelName?: string, apiKey?: string);
+    generate(prompt: string): string;
+    aGenerate(prompt: string): Promise<string>;
+    getModelName(): string;
+}
+/**
+ * Create a judge instance
+ * @param model Model name or Judge instance
+ * @param user Optional user identifier
+ * @returns Judge instance and whether it's a native model
+ */
+export declare function createJudge(model?: string | Judge, user?: string): {
+    judge: Judge;
+    usingNativeModel: boolean;
+};

package/dist/types/scorers/metrics/answer-correctness/answer-correctness.d.ts

@@ -0,0 +1,99 @@
+import { Example } from '../../../data/example.js';
+import { ScorerData } from '../../../data/result.js';
+import { JudgevalScorer } from '../../base-scorer.js';
+export interface Judge {
+    generate(prompt: string): string;
+    aGenerate(prompt: string): Promise<string>;
+    getModelName(): string;
+}
+export declare class DefaultJudge implements Judge {
+    private modelName;
+    private apiKey?;
+    private user?;
+    constructor(modelName?: string, apiKey?: string, user?: string);
+    generate(prompt: string): string;
+    aGenerate(prompt: string): Promise<string>;
+    getModelName(): string;
+}
+export declare function createJudge(model?: string | Judge, user?: string): {
+    judge: Judge;
+    usingNativeModel: boolean;
+};
+/**
+ * AnswerCorrectnessScorer evaluates how well an actual output matches an expected output
+ * by breaking down the expected output into statements and checking if each statement
+ * is correctly represented in the actual output.
+ */
+export declare class AnswerCorrectnessScorer extends JudgevalScorer {
+    private model;
+    private usingNativeModel;
+    private statements?;
+    private verdicts?;
+    evaluation_cost?: number;
+    reason?: string;
+    /**
+     * Constructor for AnswerCorrectnessScorer
+     * @param threshold Minimum score to consider the evaluation successful (default: 0.5)
+     * @param model LLM to use for evaluation (string or Judge instance)
+     * @param include_reason Whether to generate a reason for the score
+     * @param async_mode Whether to use asynchronous evaluation
+     * @param strict_mode If true, sets threshold to 1.0 (requiring perfect match)
+     * @param verbose_mode Enables detailed logging
+     * @param user Optional user identifier for the LLM
+     * @param additional_metadata Additional metadata to include in the result
+     */
+    constructor(threshold?: number, model?: string | Judge, include_reason?: boolean, async_mode?: boolean, strict_mode?: boolean, verbose_mode?: boolean, user?: string, additional_metadata?: Record<string, any>);
+    /**
+     * Get statements from expected output asynchronously
+     */
+    private _aGetStatements;
+    /**
+     * Get statements from expected output synchronously
+     */
+    private _getStatements;
+    /**
+     * Get verdicts for statements against actual output asynchronously
+     */
+    private _aGetVerdicts;
+    /**
+     * Get verdicts for statements against actual output synchronously
+     */
+    private _getVerdicts;
+    /**
+     * Get reason for the score asynchronously
+     */
+    private _aGetReason;
+    /**
+     * Get reason for the score synchronously
+     */
+    private _getReason;
+    /**
+     * Compute score based on verdicts
+     */
+    private _computeScore;
+    /**
+     * Calculate token costs for model usage
+     */
+    private _calculateTokenCosts;
+    /**
+     * Check if example has required parameters
+     */
+    private _checkExampleParams;
+    /**
+     * Create verbose logs for debugging
+     */
+    private _createVerboseLogs;
+    /**
+     * Score an example synchronously - this is for compatibility with the Python SDK
+     */
+    syncScoreExample(example: Example): ScorerData;
+    /**
+     * Score an example - this is the main method that should be called
+     * It will use async or sync methods based on the async_mode setting
+     */
+    scoreExample(example: Example): Promise<ScorerData>;
+    /**
+     * Get the name of the scorer
+     */
+    get name(): string;
+}

package/dist/types/scorers/metrics/answer-correctness/index.d.ts

@@ -0,0 +1,2 @@
+export * from './answer-correctness.js';
+export * from './prompts.js';

package/dist/types/scorers/metrics/answer-correctness/prompts.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Utility prompts for AnswerCorrectnessScorer
+ */
+import { z } from 'zod';
+export declare const StatementsSchema: z.ZodObject<{
+    statements: z.ZodArray<z.ZodString, "many">;
+}, "strip", z.ZodTypeAny, {
+    statements: string[];
+}, {
+    statements: string[];
+}>;
+export type Statements = z.infer<typeof StatementsSchema>;
+export declare const ACVerdictSchema: z.ZodObject<{
+    verdict: z.ZodString;
+    reason: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    reason: string;
+    verdict: string;
+}, {
+    reason: string;
+    verdict: string;
+}>;
+export type ACVerdict = z.infer<typeof ACVerdictSchema>;
+export declare const VerdictsSchema: z.ZodObject<{
+    verdicts: z.ZodArray<z.ZodObject<{
+        verdict: z.ZodString;
+        reason: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        reason: string;
+        verdict: string;
+    }, {
+        reason: string;
+        verdict: string;
+    }>, "many">;
+}, "strip", z.ZodTypeAny, {
+    verdicts: {
+        reason: string;
+        verdict: string;
+    }[];
+}, {
+    verdicts: {
+        reason: string;
+        verdict: string;
+    }[];
+}>;
+export type Verdicts = z.infer<typeof VerdictsSchema>;
+export declare const ReasonSchema: z.ZodObject<{
+    reason: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    reason: string;
+}, {
+    reason: string;
+}>;
+export type Reason = z.infer<typeof ReasonSchema>;
+/**
+ * Template prompts for the AnswerCorrectnessScorer
+ */
+export declare class AnswerCorrectnessTemplate {
+    /**
+     * Generate a prompt to extract statements from the expected output
+     */
+    static deduceStatements(expectedOutput: string): string;
+    /**
+     * Generate a prompt to evaluate statements against the actual output
+     */
+    static generateVerdicts(statements: string[], actualOutput: string): string;
+    /**
+     * Generate a prompt to explain the score based on incorrect statements
+     */
+    static generateReason(incorrectStatements: [string, string][], score: string): string;
+}

package/dist/types/scorers/metrics/answer-relevancy/answer-relevancy.d.ts

@@ -0,0 +1,78 @@
+import { Example } from '../../../data/example.js';
+import { ScorerData } from '../../../data/result.js';
+import { JudgevalScorer } from '../../base-scorer.js';
+import { Judge } from '../answer-correctness/answer-correctness.js';
+/**
+ * AnswerRelevancyScorer evaluates how relevant the actual output is to the input
+ * by breaking down the actual output into statements and checking if each statement
+ * is relevant to the input.
+ */
+export declare class AnswerRelevancyScorer extends JudgevalScorer {
+    private model;
+    private usingNativeModel;
+    private statements?;
+    private verdicts?;
+    evaluation_cost?: number;
+    reason?: string;
+    /**
+     * Constructor for AnswerRelevancyScorer
+     * @param threshold Minimum score to consider the evaluation successful (default: 0.5)
+     * @param model LLM to use for evaluation (string or Judge instance)
+     * @param include_reason Whether to generate a reason for the score
+     * @param async_mode Whether to use asynchronous evaluation
+     * @param strict_mode If true, sets threshold to 1.0 (requiring perfect match)
+     * @param verbose_mode Enables detailed logging
+     * @param user Optional user identifier for the LLM
+     * @param additional_metadata Additional metadata to include in the result
+     */
+    constructor(threshold?: number, model?: string | Judge, include_reason?: boolean, async_mode?: boolean, strict_mode?: boolean, verbose_mode?: boolean, user?: string, additional_metadata?: Record<string, any>);
+    /**
+     * Get statements from actual output asynchronously
+     */
+    private _aGetStatements;
+    /**
+     * Get statements from actual output synchronously
+     */
+    private _getStatements;
+    /**
+     * Get verdicts for statements against input asynchronously
+     */
+    private _aGetVerdicts;
+    /**
+     * Get verdicts for statements against input synchronously
+     */
+    private _getVerdicts;
+    /**
+     * Get reason for the score asynchronously
+     */
+    private _aGetReason;
+    /**
+     * Get reason for the score synchronously
+     */
+    private _getReason;
+    /**
+     * Compute score based on verdicts
+     */
+    private _computeScore;
+    /**
+     * Check if example has required parameters
+     */
+    private _checkExampleParams;
+    /**
+     * Create verbose logs for debugging
+     */
+    private _createVerboseLogs;
+    /**
+     * Score an example synchronously - this is for compatibility with the Python SDK
+     */
+    syncScoreExample(example: Example): ScorerData;
+    /**
+     * Score an example - this is the main method that should be called
+     * It will use async or sync methods based on the async_mode setting
+     */
+    scoreExample(example: Example): Promise<ScorerData>;
+    /**
+     * Get the name of the scorer
+     */
+    get name(): string;
+}

package/dist/types/scorers/metrics/answer-relevancy/index.d.ts

@@ -0,0 +1,2 @@
+export * from './answer-relevancy.js';
+export * from './prompts.js';

package/dist/types/scorers/metrics/answer-relevancy/prompts.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Utility prompts for AnswerRelevancyScorer
+ */
+import { z } from 'zod';
+export declare const StatementsSchema: z.ZodObject<{
+    statements: z.ZodArray<z.ZodString, "many">;
+}, "strip", z.ZodTypeAny, {
+    statements: string[];
+}, {
+    statements: string[];
+}>;
+export type Statements = z.infer<typeof StatementsSchema>;
+export declare const ARVerdictSchema: z.ZodObject<{
+    verdict: z.ZodString;
+    reason: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    reason: string;
+    verdict: string;
+}, {
+    reason: string;
+    verdict: string;
+}>;
+export type ARVerdict = z.infer<typeof ARVerdictSchema>;
+export declare const VerdictsSchema: z.ZodObject<{
+    verdicts: z.ZodArray<z.ZodObject<{
+        verdict: z.ZodString;
+        reason: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        reason: string;
+        verdict: string;
+    }, {
+        reason: string;
+        verdict: string;
+    }>, "many">;
+}, "strip", z.ZodTypeAny, {
+    verdicts: {
+        reason: string;
+        verdict: string;
+    }[];
+}, {
+    verdicts: {
+        reason: string;
+        verdict: string;
+    }[];
+}>;
+export type Verdicts = z.infer<typeof VerdictsSchema>;
+export declare const ReasonSchema: z.ZodObject<{
+    reason: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    reason: string;
+}, {
+    reason: string;
+}>;
+export type Reason = z.infer<typeof ReasonSchema>;
+/**
+ * Template prompts for the AnswerRelevancyScorer
+ */
+export declare class AnswerRelevancyTemplate {
+    /**
+     * Generate a prompt to extract statements from the actual output
+     */
+    static deduceStatements(actualOutput: string): string;
+    /**
+     * Generate a prompt to evaluate statements against the input
+     */
+    static generateVerdicts(input: string, statements: string[]): string;
+    /**
+     * Generate a prompt to explain the score based on irrelevant statements
+     */
+    static generateReason(irrelevantStatements: [string, string][], input: string, score: string): string;
+}

package/dist/types/scorers/metrics/faithfulness/faithfulness.d.ts

@@ -0,0 +1,77 @@
+import { Example } from '../../../data/example.js';
+import { ScorerData } from '../../../data/result.js';
+import { JudgevalScorer } from '../../base-scorer.js';
+import { Judge } from '../../../judges/index.js';
+/**
+ * FaithfulnessScorer evaluates how well the actual output is supported by the retrieval context
+ * by extracting claims from the output and checking if each claim is supported by the context.
+ */
+export declare class FaithfulnessScorer extends JudgevalScorer {
+    private model;
+    private usingNativeModel;
+    private claims?;
+    private claimsWithQuotes?;
+    private verdicts?;
+    evaluation_cost?: number;
+    reason?: string;
+    /**
+     * Constructor for FaithfulnessScorer
+     * @param threshold Minimum score to consider the evaluation successful (default: 0.5)
+     * @param model LLM to use for evaluation (string or Judge instance)
+     * @param include_reason Whether to generate a reason for the score
+     * @param async_mode Whether to use asynchronous evaluation
+     * @param strict_mode If true, sets threshold to 1.0 (requiring perfect match)
+     * @param verbose_mode Enables detailed logging
+     * @param user Optional user identifier for the LLM
+     * @param additional_metadata Additional metadata to include in the result
+     */
+    constructor(threshold?: number, model?: string | Judge, include_reason?: boolean, async_mode?: boolean, strict_mode?: boolean, verbose_mode?: boolean, user?: string, additional_metadata?: Record<string, any>);
+    /**
+     * Generate claims from actual output asynchronously
+     */
+    private _aGenerateClaims;
+    /**
+     * Generate claims from actual output synchronously
+     */
+    private _generateClaims;
+    /**
+     * Generate verdicts for claims against retrieval context asynchronously
+     */
+    private _aGenerateVerdicts;
+    /**
+     * Generate verdicts for claims against retrieval context synchronously
+     */
+    private _generateVerdicts;
+    /**
+     * Generate reason for the score asynchronously
+     */
+    private _aGenerateReason;
+    /**
+     * Generate reason for the score synchronously
+     */
+    private _generateReason;
+    /**
+     * Compute score based on verdicts
+     */
+    private _computeScore;
+    /**
+     * Check if example has required parameters
+     */
+    private _checkExampleParams;
+    /**
+     * Create verbose logs for debugging
+     */
+    private _createVerboseLogs;
+    /**
+     * Score an example synchronously
+     */
+    syncScoreExample(example: Example, allClaims?: boolean): ScorerData;
+    /**
+     * Score an example asynchronously
+     */
+    scoreExample(example: Example, allClaims?: boolean): Promise<ScorerData>;
+    /**
+     * Get the name of the scorer
+     */
+    get name(): string;
+}

package/dist/types/scorers/metrics/faithfulness/index.d.ts

@@ -0,0 +1,2 @@
+export * from './faithfulness.js';
+export * from './prompts.js';

package/dist/types/scorers/metrics/faithfulness/prompts.d.ts

@@ -0,0 +1,94 @@
+/**
+ * Utility prompts for FaithfulnessScorer
+ */
+import { z } from 'zod';
+export declare const FaithfulnessVerdictSchema: z.ZodObject<{
+    verdict: z.ZodString;
+    reason: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    verdict: string;
+    reason?: string | undefined;
+}, {
+    verdict: string;
+    reason?: string | undefined;
+}>;
+export type FaithfulnessVerdict = z.infer<typeof FaithfulnessVerdictSchema>;
+export declare const VerdictsSchema: z.ZodObject<{
+    verdicts: z.ZodArray<z.ZodObject<{
+        verdict: z.ZodString;
+        reason: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        verdict: string;
+        reason?: string | undefined;
+    }, {
+        verdict: string;
+        reason?: string | undefined;
+    }>, "many">;
+}, "strip", z.ZodTypeAny, {
+    verdicts: {
+        verdict: string;
+        reason?: string | undefined;
+    }[];
+}, {
+    verdicts: {
+        verdict: string;
+        reason?: string | undefined;
+    }[];
+}>;
+export type Verdicts = z.infer<typeof VerdictsSchema>;
+export declare const TruthsSchema: z.ZodObject<{
+    truths: z.ZodArray<z.ZodString, "many">;
+}, "strip", z.ZodTypeAny, {
+    truths: string[];
+}, {
+    truths: string[];
+}>;
+export type Truths = z.infer<typeof TruthsSchema>;
+export declare const ClaimsSchema: z.ZodObject<{
+    claims: z.ZodArray<z.ZodObject<{
+        claim: z.ZodString;
+        quote: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        claim: string;
+        quote: string;
+    }, {
+        claim: string;
+        quote: string;
+    }>, "many">;
+}, "strip", z.ZodTypeAny, {
+    claims: {
+        claim: string;
+        quote: string;
+    }[];
+}, {
+    claims: {
+        claim: string;
+        quote: string;
+    }[];
+}>;
+export type Claims = z.infer<typeof ClaimsSchema>;
+export declare const ReasonSchema: z.ZodObject<{
+    reason: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    reason: string;
+}, {
+    reason: string;
+}>;
+export type Reason = z.infer<typeof ReasonSchema>;
+/**
+ * Template prompts for the FaithfulnessScorer
+ */
+export declare class FaithfulnessTemplate {
+    /**
+     * Generate a prompt to extract claims from the actual output
+     */
+    static findClaims(text: string, allClaims?: boolean): string;
+    /**
+     * Generate a prompt to evaluate claims against the retrieval context
+     */
+    static generateVerdicts(claims: string[], retrievalContext: string): string;
+    /**
+     * Generate a prompt to explain the score based on verdicts
+     */
+    static generateReason(verdicts: FaithfulnessVerdict[], score: string): string;
+}

package/dist/types/scorers/metrics/hallucination/hallucination.d.ts

@@ -0,0 +1,67 @@
+import { Example } from '../../../data/example.js';
+import { ScorerData } from '../../../data/result.js';
+import { JudgevalScorer } from '../../base-scorer.js';
+import { Judge } from '../../../judges/index.js';
+/**
+ * HallucinationScorer evaluates whether an LLM's output contains hallucinations
+ * by comparing it against provided context.
+ *
+ * The score is the fraction of context segments that contradict the output.
+ * Lower scores are better (0 = no hallucinations, 1 = all contexts contradict the output).
+ */
+export declare class HallucinationScorer extends JudgevalScorer {
+    private model;
+    private using_native_model;
+    private _verdicts;
+    /**
+     * Create a new HallucinationScorer
+     *
+     * @param threshold - Success threshold (default: 0.5)
+     * @param model - Model to use for evaluation (default: DefaultJudge)
+     * @param include_reason - Whether to include a reason for the score (default: true)
+     * @param async_mode - Whether to use async mode (default: false)
+     * @param strict_mode - Whether to use strict mode (default: false)
+     * @param verbose_mode - Whether to include verbose logs (default: false)
+     */
+    constructor(threshold?: number, model?: string | Judge | undefined, include_reason?: boolean, async_mode?: boolean, strict_mode?: boolean, verbose_mode?: boolean);
+    /**
+     * Generate verdicts for each context
+     */
+    private _aGenerateVerdicts;
+    /**
+     * Generate verdicts for each context (synchronous)
+     */
+    private _generateVerdicts;
+    /**
+     * Generate a reason for the score
+     */
+    private _aGenerateReason;
+    /**
+     * Generate a reason for the score (synchronous)
+     */
+    private _generateReason;
+    /**
+     * Calculate the hallucination score
+     */
+    private _computeScore;
+    /**
+     * Create verbose logs for debugging
+     */
+    private _createVerboseLogs;
+    /**
+     * Check if example has required parameters
+     */
+    private _checkExampleParams;
+    /**
+     * Score an example synchronously
+     */
+    syncScoreExample(example: Example): ScorerData;
+    /**
+     * Score an example asynchronously
+     */
+    scoreExample(example: Example): Promise<ScorerData>;
+    /**
+     * Get the name of the scorer
+     */
+    get name(): string;
+}

package/dist/types/scorers/metrics/hallucination/index.d.ts

@@ -0,0 +1,3 @@
+export { HallucinationScorer } from './hallucination.js';
+export { HallucinationTemplate, HallucinationVerdictSchema, VerdictsSchema, ReasonSchema } from './prompts.js';
+export type { HallucinationVerdict } from './prompts.js';