judgeval 0.1.33

package/dist/judgment-client.d.ts

@@ -0,0 +1,179 @@
+import { Example } from './data/example';
+import { ScoringResult } from './data/result';
+import { APIJudgmentScorer, JudgevalScorer, ScorerWrapper } from './scorers/base-scorer';
+import { Rule } from './rules';
+/**
+ * Singleton implementation for JudgmentClient
+ */
+export declare class JudgmentClient {
+    private static instance;
+    private judgmentApiKey;
+    private organizationId;
+    /**
+     * Get the singleton instance of JudgmentClient
+     */
+    static getInstance(judgmentApiKey?: string, organizationId?: string): JudgmentClient;
+    /**
+     * Constructor for JudgmentClient
+     * @param judgmentApiKey The Judgment API key
+     * @param organizationId The organization ID
+     */
+    constructor(judgmentApiKey?: string, organizationId?: string);
+    /**
+     * Run an evaluation asynchronously
+     */
+    aRunEvaluation(examples: Example[], scorers: Array<ScorerWrapper | JudgevalScorer | APIJudgmentScorer>, model: string | string[] | any, aggregator?: string, metadata?: Record<string, any>, logResults?: boolean, projectName?: string, evalRunName?: string, override?: boolean, useJudgment?: boolean, ignoreErrors?: boolean, rules?: Rule[]): Promise<ScoringResult[]>;
+    /**
+     * Run an evaluation
+     */
+    runEvaluation(examples: Example[], scorers: Array<ScorerWrapper | JudgevalScorer | APIJudgmentScorer>, model: string | string[] | any, aggregator?: string, metadata?: Record<string, any>, logResults?: boolean, projectName?: string, evalRunName?: string, override?: boolean, useJudgment?: boolean, ignoreErrors?: boolean, asyncExecution?: boolean, rules?: Rule[]): Promise<ScoringResult[]>;
+    /**
+     * Run an evaluation with a simplified interface (recommended)
+     * @param config Configuration object for the evaluation
+     * @returns Promise<ScoringResult[]> The evaluation results
+     */
+    evaluate(config: {
+        examples: Example[];
+        scorers: Array<ScorerWrapper | JudgevalScorer | APIJudgmentScorer>;
+        model?: string | string[] | any;
+        aggregator?: string;
+        metadata?: Record<string, any>;
+        projectName?: string;
+        evalName?: string;
+        logResults?: boolean;
+        useJudgment?: boolean;
+        ignoreErrors?: boolean;
+        asyncExecution?: boolean;
+        rules?: Rule[];
+        override?: boolean;
+    }): Promise<ScoringResult[]>;
+    /**
+     * Evaluate a dataset
+     */
+    evaluateDataset(dataset: any, // EvalDataset would be implemented separately
+    scorers: Array<ScorerWrapper | JudgevalScorer>, model: string | string[] | any, aggregator?: string, metadata?: Record<string, any>, projectName?: string, evalRunName?: string, logResults?: boolean, useJudgment?: boolean, rules?: Rule[]): Promise<ScoringResult[]>;
+    /**
+     * Create a dataset
+     */
+    createDataset(): any;
+    /**
+     * Push a dataset to the Judgment platform
+     */
+    pushDataset(alias: string, dataset: any, projectName: string, overwrite?: boolean): Promise<boolean>;
+    /**
+     * Pull a dataset from the Judgment platform
+     */
+    pullDataset(alias: string, projectName: string): Promise<any>;
+    /**
+     * Delete a dataset from the Judgment platform
+     */
+    deleteDataset(alias: string, projectName: string): Promise<boolean>;
+    /**
+     * Pull project dataset stats from the Judgment platform
+     */
+    pullProjectDatasetStats(projectName: string): Promise<Record<string, any>>;
+    /**
+     * Insert examples into a dataset on the Judgment platform
+     */
+    insertDataset(alias: string, examples: Example[], projectName: string): Promise<boolean>;
+    /**
+     * Pull evaluation results from the server
+     * @param projectName Name of the project
+     * @param evalRunName Name of the evaluation run
+     * @returns Array containing one object with 'id' and 'results' (list of ScoringResult)
+     */
+    pullEval(projectName: string, evalRunName: string): Promise<Array<Record<string, any | ScoringResult[]>>>;
+    /**
+     * Get evaluation run results (alias for pullEval with a more intuitive name)
+     * @param projectName Name of the project
+     * @param evalRunName Name of the evaluation run
+     * @returns Array containing one object with 'id' and 'results' (list of ScoringResult)
+     */
+    getEvalRun(projectName: string, evalRunName: string): Promise<Array<Record<string, any | ScoringResult[]>>>;
+    /**
+     * List all evaluation runs for a project
+     * @param projectName Name of the project
+     * @param limit Maximum number of evaluation runs to return (default: 100)
+     * @param offset Offset for pagination (default: 0)
+     * @returns List of evaluation run metadata
+     */
+    listEvalRuns(projectName: string, limit?: number, offset?: number): Promise<Array<Record<string, any>>>;
+    /**
+     * Get evaluation run statistics
+     * @param projectName Name of the project
+     * @param evalRunName Name of the evaluation run
+     * @returns Statistics for the evaluation run
+     */
+    getEvalRunStats(projectName: string, evalRunName: string): Promise<Record<string, any>>;
+    /**
+     * Export evaluation results to a file format
+     * @param projectName Name of the project
+     * @param evalRunName Name of the evaluation run
+     * @param format Export format ('json' or 'csv')
+     * @returns The exported data as a string
+     */
+    exportEvalResults(projectName: string, evalRunName: string, format?: 'json' | 'csv'): Promise<string>;
+    /**
+     * Delete an evaluation from the server
+     */
+    deleteEval(projectName: string, evalRunNames: string[]): Promise<boolean>;
+    /**
+     * Delete all evaluations from the server for a given project
+     */
+    deleteProjectEvals(projectName: string): Promise<boolean>;
+    /**
+     * Create a project on the server
+     */
+    createProject(projectName: string): Promise<boolean>;
+    /**
+     * Delete a project from the server
+     */
+    deleteProject(projectName: string): Promise<boolean>;
+    /**
+     * Validate that the user API key is valid
+     */
+    private validateApiKey;
+    /**
+     * Assert a test by running the evaluation and checking the results for success
+     */
+    assertTest(examples: Example[], scorers: Array<APIJudgmentScorer | JudgevalScorer>, // Type matches Python's intent
+    model: string | string[] | any, aggregator?: string, metadata?: Record<string, any>, logResults?: boolean, projectName?: string, evalRunName?: string, override?: boolean, rules?: Rule[]): Promise<void>;
+    /**
+     * Pull the results of an evaluation run. Matches `pullEval` logic but returns only the ScoringResult array.
+     * @param projectName The name of the project
+     * @param evalRunName The name of the evaluation run
+     * @returns The results of the evaluation run as ScoringResult[] or empty array on error/no results.
+     */
+    pullEvalResults(projectName: string, evalRunName: string): Promise<ScoringResult[]>;
+    /**
+     * Check the status of an evaluation run using the fetch endpoint.
+     * This is a heuristic approach as the endpoint might return full results or status info.
+     * @param projectName The name of the project
+     * @param evalRunName The name of the evaluation run
+     * @returns An object representing the status { status: string, progress: number, message: string }
+     */
+    checkEvalStatus(projectName: string, evalRunName: string): Promise<{
+        status: string;
+        progress: number;
+        message: string;
+        error?: string;
+    }>;
+    /**
+     * Wait for an async evaluation to complete and return the results
+     * @param projectName The name of the project
+     * @param evalRunName The name of the evaluation run
+     * @param options Optional configuration for polling: intervalMs, maxAttempts, showProgress
+     * @returns The evaluation results as ScoringResult[] or empty array on timeout/failure.
+     */
+    waitForEvaluation(projectName: string, evalRunName: string, options?: {
+        intervalMs?: number;
+        maxAttempts?: number;
+        showProgress?: boolean;
+    }): Promise<ScoringResult[]>;
+    /**
+     * Create a simple ASCII progress bar
+     * @param percent The percentage to display (0-100)
+     * @returns A string representing the progress bar
+     */
+    private _createProgressBar;
+}