judgeval 0.2.9 → 0.3.0

package/dist/types/evaluation-run.d.ts

@@ -1,44 +0,0 @@
-import { Example } from './data/example.js';
-import { APIJudgmentScorer, JudgevalScorer } from './scorers/base-scorer.js';
-import { Rule } from './rules.js';
-/**
- * Stores example and evaluation scorers together for running an eval task
- */
-export interface EvaluationRunOptions {
-    logResults?: boolean;
-    organizationId?: string;
-    projectName?: string;
-    evalName?: string;
-    examples: Example[];
-    scorers: Array<APIJudgmentScorer | JudgevalScorer>;
-    model: string | string[] | any;
-    aggregator?: string;
-    metadata?: Record<string, any>;
-    judgmentApiKey?: string;
-    override?: boolean;
-    rules?: Rule[];
-}
-export declare class EvaluationRun {
-    logResults: boolean;
-    organizationId?: string;
-    projectName?: string;
-    evalName?: string;
-    examples: Example[];
-    scorers: Array<APIJudgmentScorer | JudgevalScorer>;
-    model: string | string[] | any;
-    aggregator?: string;
-    metadata?: Record<string, any>;
-    judgmentApiKey?: string;
-    override?: boolean;
-    rules?: Rule[];
-    constructor(options: EvaluationRunOptions);
-    /**
-     * Validate the evaluation run configuration
-     */
-    private validate;
-    /**
-     * Convert the evaluation run to a plain object
-     *
-     */
-    toJSON(): Record<string, any>;
-}

package/dist/types/exporters/otel-exporter.d.ts

@@ -1,16 +0,0 @@
-import { ExportResult } from '@opentelemetry/core';
-import { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-base";
-interface OtelExporterConfig {
-    apiKey: string;
-    organizationId: string;
-    serviceName?: string;
-}
-export declare class JudgevalExporter implements SpanExporter {
-    private serviceName;
-    private apiKey;
-    private organizationId;
-    constructor(config: OtelExporterConfig);
-    export(spans: ReadableSpan[], resultCallback: (result: ExportResult) => void): Promise<void>;
-    shutdown(): Promise<void>;
-}
-export {};

package/dist/types/index.d.ts

@@ -1,11 +0,0 @@
-export { Example, ExampleBuilder, ExampleOptions } from './data/example.js';
-export { ScoringResult, ScoringResultBuilder, ScorerData, ScoringResultOptions } from './data/result.js';
-export { Tracer, SpanType, wrap, TraceClient } from './common/tracer.js';
-export { Scorer, APIJudgmentScorer, JudgevalScorer, ScorerWrapper } from './scorers/base-scorer.js';
-export { AnswerCorrectnessScorer, AnswerRelevancyScorer, ComparisonScorer, ContextualPrecisionScorer, ContextualRecallScorer, ContextualRelevancyScorer, ExecutionOrderScorer, FaithfulnessScorer, GroundednessScorer, HallucinationScorer, InstructionAdherenceScorer, JsonCorrectnessScorer, SummarizationScorer } from './scorers/api-scorer.js';
-export { AlertStatus, Condition, NotificationConfig, Rule, AlertResult, RulesEngine } from './rules.js';
-export { EvaluationRun, EvaluationRunOptions } from './evaluation-run.js';
-export { runEval, assertTest, JudgmentAPIError, sendToRabbitMQ, executeApiEval, mergeResults, checkMissingScorerData, checkEvalRunNameExists, logEvaluationResults, checkExamples } from './run-evaluation.js';
-export { JudgmentClient } from './judgment-client.js';
-export * from './constants.js';
-export * from './clients.js';

package/dist/types/judges/index.d.ts

@@ -1,50 +0,0 @@
-/**
- * Interface for judge models that can generate text
- */
-export interface Judge {
-    /**
-     * Generate text synchronously
-     */
-    generate(prompt: string): string;
-    /**
-     * Generate text asynchronously
-     */
-    aGenerate(prompt: string): Promise<string>;
-    /**
-     * Get the name of the model
-     */
-    getModelName(): string;
-}
-/**
- * Default judge implementation using OpenAI API
- */
-export declare class DefaultJudge implements Judge {
-    private modelName;
-    private apiKey?;
-    private user?;
-    constructor(modelName?: string, apiKey?: string, user?: string);
-    generate(prompt: string): string;
-    aGenerate(prompt: string): Promise<string>;
-    getModelName(): string;
-}
-/**
- * Together AI judge implementation
- */
-export declare class TogetherJudge implements Judge {
-    private modelName;
-    private apiKey?;
-    constructor(modelName?: string, apiKey?: string);
-    generate(prompt: string): string;
-    aGenerate(prompt: string): Promise<string>;
-    getModelName(): string;
-}
-/**
- * Create a judge instance
- * @param model Model name or Judge instance
- * @param user Optional user identifier
- * @returns Judge instance and whether it's a native model
- */
-export declare function createJudge(model?: string | Judge, user?: string): {
-    judge: Judge;
-    usingNativeModel: boolean;
-};

package/dist/types/judgment-client.d.ts

@@ -1,149 +0,0 @@
-import { Example } from './data/example.js';
-import { ScoringResult } from './data/result.js';
-import { APIJudgmentScorer, JudgevalScorer, ScorerWrapper } from './scorers/base-scorer.js';
-import { Rule } from './rules.js';
-/**
- * Singleton implementation for JudgmentClient
- */
-export declare class JudgmentClient {
-    private static instance;
-    private judgmentApiKey;
-    private organizationId;
-    /**
-     * Get the singleton instance of JudgmentClient
-     */
-    static getInstance(judgmentApiKey?: string, organizationId?: string): JudgmentClient;
-    /**
-     * Constructor for JudgmentClient
-     * @param judgmentApiKey The Judgment API key
-     * @param organizationId The organization ID
-     */
-    constructor(judgmentApiKey?: string, organizationId?: string);
-    /**
-     * Run an evaluation asynchronously
-     */
-    aRunEvaluation(examples: Example[], scorers: Array<ScorerWrapper | JudgevalScorer | APIJudgmentScorer>, model: string | string[] | any, aggregator?: string, metadata?: Record<string, any>, logResults?: boolean, projectName?: string, evalRunName?: string, override?: boolean, useJudgment?: boolean, ignoreErrors?: boolean, rules?: Rule[]): Promise<ScoringResult[]>;
-    /**
-     * Run an evaluation
-     */
-    runEvaluation(examples: Example[], scorers: Array<ScorerWrapper | JudgevalScorer | APIJudgmentScorer>, model: string | string[] | any, aggregator?: string, metadata?: Record<string, any>, logResults?: boolean, projectName?: string, evalRunName?: string, override?: boolean, useJudgment?: boolean, ignoreErrors?: boolean, asyncExecution?: boolean, rules?: Rule[]): Promise<ScoringResult[]>;
-    /**
-     * Run an evaluation with a simplified interface (recommended)
-     * @param config Configuration object for the evaluation
-     * @returns Promise<ScoringResult[]> The evaluation results
-     */
-    evaluate(config: {
-        examples: Example[];
-        scorers: Array<ScorerWrapper | JudgevalScorer | APIJudgmentScorer>;
-        model?: string | string[] | any;
-        aggregator?: string;
-        metadata?: Record<string, any>;
-        projectName?: string;
-        evalName?: string;
-        logResults?: boolean;
-        useJudgment?: boolean;
-        ignoreErrors?: boolean;
-        asyncExecution?: boolean;
-        rules?: Rule[];
-        override?: boolean;
-    }): Promise<ScoringResult[]>;
-    /**
-     * Evaluate a dataset
-     */
-    evaluateDataset(dataset: any, // Keep type loose for stub
-    scorers: Array<ScorerWrapper | JudgevalScorer>, model: string | string[] | any, aggregator?: string, metadata?: Record<string, any>, projectName?: string, evalRunName?: string, logResults?: boolean, useJudgment?: boolean, rules?: Rule[]): Promise<ScoringResult[]>;
-    /**
-     * Pull evaluation results from the server
-     *
-     * @param projectName Name of the project
-     * @param evalRunName Name of the evaluation run
-     * @returns Array of evaluation result objects with the same format as the Python SDK
-     */
-    pullEval(projectName: string, evalRunName: string): Promise<any[]>;
-    /**
-     * Retrieves evaluation results with retry mechanism
-     * @param projectName Name of the project
-     * @param evalRunName Name of the evaluation run
-     * @param options Configuration options for retries
-     * @returns The evaluation results or null if not available after all retries
-     */
-    pullEvalWithRetry(projectName: string, evalRunName: string, options?: {
-        maxRetries?: number;
-        initialDelayMs?: number;
-        maxDelayMs?: number;
-        backoffFactor?: number;
-    }): Promise<any>;
-    /**
-     * Export evaluation results to a file format
-     * @param projectName Name of the project
-     * @param evalRunName Name of the evaluation run
-     * @param format Export format ('json' or 'csv')
-     * @returns The exported data as a string
-     */
-    exportEvalResults(projectName: string, evalRunName: string, format?: 'json' | 'csv'): Promise<string>;
-    /**
-     * Delete an evaluation from the server
-     */
-    deleteEval(projectName: string, evalRunNames: string[]): Promise<boolean>;
-    /**
-     * Delete all evaluations from the server for a given project
-     */
-    deleteProjectEvals(projectName: string): Promise<boolean>;
-    /**
-     * Create a project on the server
-     */
-    createProject(projectName: string): Promise<boolean>;
-    /**
-     * Delete a project from the server
-     */
-    deleteProject(projectName: string): Promise<boolean>;
-    /**
-     * Validate that the user API key is valid
-     */
-    private validateApiKey;
-    /**
-     * Assert a test by running the evaluation and checking the results for success
-     */
-    assertTest(examples: Example[], scorers: Array<APIJudgmentScorer | JudgevalScorer>, // Type matches Python's intent
-    model: string | string[] | any, aggregator?: string, metadata?: Record<string, any>, logResults?: boolean, projectName?: string, evalRunName?: string, override?: boolean, rules?: Rule[]): Promise<void>;
-    /**
-     * Pull the results of an evaluation run. Matches `pullEval` logic but returns only the ScoringResult array.
-     * @param projectName The name of the project
-     * @param evalRunName The name of the evaluation run
-     * @returns Array of ScoringResult objects
-     */
-    pullEvalResults(projectName: string, evalRunName: string): Promise<ScoringResult[]>;
-    /**
-     * Check the status of an evaluation run using the fetch endpoint.
-     * This is a heuristic approach as the endpoint might return full results or status info.
-     * @param projectName The name of the project
-     * @param evalRunName The name of the evaluation run
-     * @returns An object representing the status { status: string, progress: number, message: string }
-     */
-    checkEvalStatus(projectName: string, evalRunName: string): Promise<{
-        status: string;
-        progress: number;
-        message: string;
-        error?: string;
-    }>;
-    /**
-     * Wait for an async evaluation to complete and return the results
-     * @param projectName The name of the project
-     * @param evalRunName The name of the evaluation run
-     * @param options Optional configuration for polling: intervalMs, maxAttempts, showProgress
-     * @returns The evaluation results as ScoringResult[] or empty array on timeout/failure.
-     */
-    waitForEvaluation(projectName: string, evalRunName: string, options?: {
-        intervalMs?: number;
-        maxAttempts?: number;
-        showProgress?: boolean;
-    }): Promise<ScoringResult[]>;
-    /**
-     * Create a simple ASCII progress bar
-     * @param percent The percentage to display (0-100)
-     * @returns A string representing the progress bar
-     */
-    private _createProgressBar;
-    private getAuthHeaders;
-    private handleApiError;
-}

package/dist/types/rules.d.ts

@@ -1,120 +0,0 @@
-import { Scorer } from './scorers/base-scorer.js';
-/**
- * Status of an alert evaluation
- */
-export declare enum AlertStatus {
-    TRIGGERED = "triggered",
-    NOT_TRIGGERED = "not_triggered"
-}
-/**
- * A single metric condition
- */
-export declare class Condition {
-    metric: Scorer;
-    constructor(metric: Scorer);
-    /**
-     * Get the name of the metric for lookups in scores dictionary
-     */
-    get metricName(): string;
-    /**
-     * Get the threshold from the metric
-     */
-    get threshold(): number;
-    /**
-     * Evaluate the condition against a value
-     * Returns true if the condition passes, false otherwise
-     */
-    evaluate(value: number): boolean;
-    /**
-     * Convert the condition to a plain object
-     */
-    toJSON(): Record<string, any>;
-}
-/**
- * Configuration for notifications when a rule is triggered
- */
-export declare class NotificationConfig {
-    enabled: boolean;
-    communicationMethods: string[];
-    emailAddresses?: string[];
-    sendAt?: number;
-    constructor(enabled?: boolean, communicationMethods?: string[], emailAddresses?: string[], sendAt?: number);
-    /**
-     * Convert the notification config to a plain object
-     */
-    toJSON(): Record<string, any>;
-}
-/**
- * Configuration for a single rule
- */
-export declare class Rule {
-    ruleId: string;
-    name: string;
-    description?: string;
-    conditions: Condition[];
-    combine_type: 'all' | 'any';
-    notification?: NotificationConfig;
-    constructor(name: string, conditions: Condition[], combine_type?: 'all' | 'any', description?: string, notification?: NotificationConfig, ruleId?: string);
-    /**
-     * Convert the rule to a plain object
-     */
-    toJSON(): Record<string, any>;
-}
-/**
- * Result of evaluating a rule
- */
-export declare class AlertResult {
-    status: AlertStatus;
-    ruleId?: string;
-    ruleName: string;
-    conditionsResult: Array<{
-        metric: string;
-        value: number;
-        threshold: number;
-        passed: boolean;
-    }>;
-    metadata: Record<string, any>;
-    notification?: NotificationConfig;
-    constructor(status: AlertStatus, ruleName: string, conditionsResult: Array<{
-        metric: string;
-        value: number;
-        threshold: number;
-        passed: boolean;
-    }>, metadata?: Record<string, any>, ruleId?: string, notification?: NotificationConfig);
-    /**
-     * Get example_id from metadata for backward compatibility
-     */
-    get exampleId(): string | undefined;
-    /**
-     * Get timestamp from metadata for backward compatibility
-     */
-    get timestamp(): string | undefined;
-    /**
-     * Convert the alert result to a plain object
-     */
-    toJSON(): Record<string, any>;
-}
-/**
- * Engine for creating and evaluating rules against metrics
- */
-export declare class RulesEngine {
-    rules: Record<string, Rule>;
-    constructor(rules: Record<string, Rule>);
-    /**
-     * Configure notification settings for a specific rule
-     */
-    configureNotification(ruleId: string, enabled?: boolean, communicationMethods?: string[], emailAddresses?: string[], sendAt?: number): void;
-    /**
-     * Configure notification settings for all rules
-     */
-    configureAllNotifications(enabled?: boolean, communicationMethods?: string[], emailAddresses?: string[], sendAt?: number): void;
-    /**
-     * Evaluate all rules against a set of scores
-     * Returns mapping of rule IDs to their alert results
-     */
-    evaluateRules(scores: Record<string, number>, exampleMetadata?: Record<string, any>): Record<string, AlertResult>;
-    /**
-     * Evaluate all rules against multiple examples in parallel
-     */
-    evaluateRulesParallel(exampleScores: Record<string, Record<string, number>>, exampleMetadata: Record<string, Record<string, any>>, maxConcurrent?: number): Promise<Record<string, Record<string, AlertResult>>>;
-}

package/dist/types/run-evaluation.d.ts

@@ -1,78 +0,0 @@
-import { Example } from './data/example.js';
-import { EvaluationRun } from './evaluation-run.js';
-import { ScoringResult } from './data/result.js';
-import { APIJudgmentScorer } from './scorers/base-scorer.js';
-/**
- * Custom error for Judgment API errors
- */
-export declare class JudgmentAPIError extends Error {
-    constructor(message: string);
-}
-/**
- * Validates an API response to ensure it has the expected format
- * Throws a JudgmentAPIError if the response is invalid
- */
-export declare function validateApiResponse(response: any): void;
-/**
- * Sends an evaluation run to the RabbitMQ evaluation queue
- */
-export declare function sendToRabbitMQ(evaluationRun: EvaluationRun): Promise<any>;
-/**
- * Checks the status of an async evaluation
- * @param evaluationRun The evaluation run to check
- * @returns The status of the evaluation
- */
-export declare function checkEvaluationStatus(evaluationRun: EvaluationRun): Promise<any>;
-/**
- * Polls the status of an async evaluation until it's complete
- * @param evaluationRun The evaluation run to poll
- * @param intervalMs The interval between polls in milliseconds
- * @param maxAttempts The maximum number of polling attempts
- * @param onProgress Optional callback for progress updates
- * @returns The evaluation results
- */
-export declare function pollEvaluationStatus(evaluationRun: EvaluationRun, intervalMs?: number, maxAttempts?: number, onProgress?: (status: any) => void): Promise<ScoringResult[]>;
-/**
- * Executes an evaluation of a list of Examples using one or more JudgmentScorers via the Judgment API
- * @param evaluationRun The evaluation run object containing the examples, scorers, and metadata
- * @returns The results of the evaluation
- */
-export declare function executeApiEval(evaluationRun: EvaluationRun): Promise<any[]>;
-/**
- * Checks if an evaluation run name already exists for a given project
- */
-export declare function checkEvalRunNameExists(evalName: string, projectName: string, judgmentApiKey: string, organizationId: string): Promise<void>;
-export declare let hasLoggedUrl: boolean;
-/**
- * Logs evaluation results to the Judgment API database.
- * @param results The results to log
- * @param projectName The project name
- * @param evalName The evaluation run name
- * @param apiKey The API key for the Judgment API
- * @param organizationId The organization ID
- * @returns A URL to view the results in the Judgment UI
- */
-export declare function logEvaluationResults(results: ScoringResult[], projectName: string, evalName: string, apiKey: string | undefined, organizationId: string): Promise<string>;
-/**
- * When executing scorers that come from both the Judgment API and local scorers, we're left with
- * results for each type of scorer. This function merges the results from the API and local evaluations,
- * grouped by example.
- */
-export declare function mergeResults(apiResults: ScoringResult[], localResults: ScoringResult[]): ScoringResult[];
-/**
- * Checks if any ScoringResult objects are missing scorersData
- */
-export declare function checkMissingScorerData(results: ScoringResult[]): ScoringResult[];
-/**
- * Checks if the example contains the necessary parameters for the scorer
- */
-export declare function checkExamples(examples: Example[], scorers: APIJudgmentScorer[]): void;
-/**
- * Executes an evaluation of Examples using one or more Scorers
- */
-export declare function runEval(evaluationRun: EvaluationRun, override?: boolean, ignoreErrors?: boolean, asyncExecution?: boolean): Promise<ScoringResult[]>;
-/**
- * Collects all failed scorers from the scoring results
- * Raises exceptions for any failed test cases
- */
-export declare function assertTest(scoringResults: ScoringResult[]): void;

package/dist/types/scorers/api-scorer.d.ts

@@ -1,84 +0,0 @@
-import { APIJudgmentScorer } from './base-scorer.js';
-import { Example } from '../data/example.js';
-import { ScorerData } from '../data/result.js';
-/**
- * Implementation of API-based scorers
- */
-export declare class AnswerCorrectnessScorer extends APIJudgmentScorer {
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
-    a_score_example(example: Example): Promise<ScorerData>;
-}
-export declare class AnswerRelevancyScorer extends APIJudgmentScorer {
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
-    a_score_example(example: Example): Promise<ScorerData>;
-}
-export declare class ComparisonScorer extends APIJudgmentScorer {
-    criteria: string[];
-    description: string;
-    constructor(threshold?: number, criteria?: string[], description?: string, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
-    toJSON(): Record<string, any>;
-    a_score_example(example: Example): Promise<ScorerData>;
-}
-export declare class ContextualPrecisionScorer extends APIJudgmentScorer {
-    requiredFields: string[];
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
-    a_score_example(example: Example): Promise<ScorerData>;
-}
-export declare class ContextualRecallScorer extends APIJudgmentScorer {
-    requiredFields: string[];
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
-    a_score_example(example: Example): Promise<ScorerData>;
-}
-export declare class ContextualRelevancyScorer extends APIJudgmentScorer {
-    requiredFields: string[];
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
-    a_score_example(example: Example): Promise<ScorerData>;
-}
-export declare class ExecutionOrderScorer extends APIJudgmentScorer {
-    strictMode: boolean;
-    expectedTools?: string[];
-    constructor(threshold?: number, expectedTools?: string[], additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
-    toJSON(): Record<string, any>;
-    a_score_example(example: Example): Promise<ScorerData>;
-}
-export declare class FaithfulnessScorer extends APIJudgmentScorer {
-    requiredFields: string[];
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
-    a_score_example(example: Example): Promise<ScorerData>;
-}
-export declare class GroundednessScorer extends APIJudgmentScorer {
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
-    a_score_example(example: Example): Promise<ScorerData>;
-}
-export declare class HallucinationScorer extends APIJudgmentScorer {
-    requiredFields: string[];
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
-    a_score_example(example: Example): Promise<ScorerData>;
-}
-export declare class InstructionAdherenceScorer extends APIJudgmentScorer {
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
-    a_score_example(example: Example): Promise<ScorerData>;
-}
-export declare class JsonCorrectnessScorer extends APIJudgmentScorer {
-    jsonSchema?: Record<string, any>;
-    constructor(threshold?: number, jsonSchema?: Record<string, any>, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
-    toJSON(): Record<string, any>;
-    a_score_example(example: Example): Promise<ScorerData>;
-}
-export declare class SummarizationScorer extends APIJudgmentScorer {
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
-    a_score_example(example: Example): Promise<ScorerData>;
-}
-export declare class Text2SQLScorer extends APIJudgmentScorer {
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
-    a_score_example(example: Example): Promise<ScorerData>;
-}
-export declare class ScorerWrapper {
-    private scorer;
-    constructor(scorer: APIJudgmentScorer);
-    get scoreType(): string;
-    get threshold(): number;
-    get additional_metadata(): Record<string, any> | undefined;
-    toJSON(): Record<string, any>;
-    static fromType(type: string, threshold: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean): APIJudgmentScorer;
-}