judgeval 0.2.0 → 0.2.2

package/dist/types/common/tracer.d.ts

@@ -26,13 +26,15 @@ interface TraceEntry {
     function: string;
     span_id: string;
     depth: number;
-    timestamp: number;
+    created_at: number;
     duration?: number;
     output?: any;
     inputs?: Record<string, any>;
     span_type: SpanType;
     parent_span_id?: string;
     evaluation_runs?: any[];
+    trace_id?: string;
+    message?: string;
 }
 interface TraceSavePayload {
     trace_id: string;
@@ -49,8 +51,7 @@ interface TraceSavePayload {
         total_cost_usd: number;
     };
     entries: CondensedSpanEntry[];
-    rules?: Record<string, Rule>;
-    empty_save: boolean;
+    evaluation_runs: any[];
     overwrite: boolean;
     parent_trace_id?: string | null;
     parent_name?: string | null;
@@ -59,15 +60,24 @@ interface CondensedSpanEntry {
     span_id: string;
     function: string;
     depth: number;
-    timestamp: number;
+    created_at: string;
     parent_span_id?: string | null;
     span_type: SpanType;
     inputs: Record<string, any> | null;
     output: any | null;
-    evaluation_runs: any[];
     duration: number | null;
+    trace_id?: string;
     children?: CondensedSpanEntry[];
 }
+interface TokenCostResponse {
+    model: string;
+    prompt_tokens: number;
+    completion_tokens: number;
+    total_tokens: number;
+    prompt_tokens_cost_usd: number;
+    completion_tokens_cost_usd: number;
+    total_cost_usd: number;
+}
 /**
  * Client for interacting with Judgment trace API endpoints.
  */
@@ -77,10 +87,20 @@ declare class TraceManagerClient {
     constructor(apiKey: string, organizationId: string);
     private _fetch;
     fetchTrace(traceId: string): Promise<any>;
-    saveTrace(traceData: TraceSavePayload, emptySave: boolean): Promise<any>;
+    saveTrace(traceData: TraceSavePayload): Promise<any>;
     deleteTrace(traceId: string): Promise<any>;
     deleteTraces(traceIds: string[]): Promise<any>;
     addTraceToEvalQueue(traceData: TraceSavePayload): Promise<any>;
+    /**
+     * Calculate token costs directly using the API endpoint.
+     * This is more accurate than client-side calculation as it uses the most up-to-date pricing.
+     *
+     * @param model The model name (e.g. 'gpt-4', 'claude-3-opus-20240229')
+     * @param promptTokens Number of tokens in the prompt/input
+     * @param completionTokens Number of tokens in the completion/output
+     * @returns Object containing token counts and calculated costs in USD
+     */
+    calculateTokenCosts(model: string, promptTokens: number, completionTokens: number): Promise<TokenCostResponse | null>;
 }
 /**
  * Represents an ongoing trace context.
@@ -100,6 +120,7 @@ declare class TraceClient {
     private apiKey;
     private organizationId;
     private originalName;
+    private _spanDepths;
     constructor(config: {
         tracer: Tracer;
         traceId?: string;
@@ -154,14 +175,6 @@ declare class TraceClient {
         model?: string;
         logResults?: boolean;
     }): Promise<void>;
-    /**
-     * Private helper to add an evaluation entry to the trace.
-     * This mirrors the structure of Python's add_eval_run.
-     *
-     * @param evalRunPayload The constructed payload for the evaluation.
-     * @param startTime The start time (in seconds) of the evaluation process.
-     */
-    private _addEvalRun;
     getOriginalName(): string;
 }
 /**

package/dist/types/constants.d.ts

@@ -21,15 +21,15 @@ export declare enum APIScorer {
 }
 export declare const UNBOUNDED_SCORERS: Set<APIScorer>;
 export declare const ROOT_API = "https://api.judgmentlabs.ai";
-export declare const JUDGMENT_EVAL_API_URL = "https://api.judgmentlabs.ai/evaluate/";
 export declare const JUDGMENT_DATASETS_PUSH_API_URL = "https://api.judgmentlabs.ai/datasets/push/";
 export declare const JUDGMENT_DATASETS_PULL_API_URL = "https://api.judgmentlabs.ai/datasets/pull/";
 export declare const JUDGMENT_DATASETS_DELETE_API_URL = "https://api.judgmentlabs.ai/datasets/delete/";
 export declare const JUDGMENT_DATASETS_EXPORT_JSONL_API_URL = "https://api.judgmentlabs.ai/datasets/export_jsonl/";
 export declare const JUDGMENT_DATASETS_PROJECT_STATS_API_URL = "https://api.judgmentlabs.ai/datasets/fetch_stats_by_project/";
 export declare const JUDGMENT_DATASETS_INSERT_API_URL = "https://api.judgmentlabs.ai/datasets/insert_examples/";
-export declare const JUDGMENT_EVAL_LOG_API_URL = "https://api.judgmentlabs.ai/log_eval_results/";
 export declare const JUDGMENT_EVAL_FETCH_API_URL = "https://api.judgmentlabs.ai/fetch_eval_results/";
+export declare const JUDGMENT_EVAL_API_URL = "https://api.judgmentlabs.ai/evaluate/";
+export declare const JUDGMENT_EVAL_LOG_API_URL = "https://api.judgmentlabs.ai/log_eval_results/";
 export declare const JUDGMENT_EVAL_DELETE_API_URL = "https://api.judgmentlabs.ai/delete_eval_results_by_project_and_run_names/";
 export declare const JUDGMENT_EVAL_DELETE_PROJECT_API_URL = "https://api.judgmentlabs.ai/delete_eval_results_by_project/";
 export declare const JUDGMENT_PROJECT_DELETE_API_URL = "https://api.judgmentlabs.ai/projects/delete/";
@@ -37,8 +37,8 @@ export declare const JUDGMENT_PROJECT_CREATE_API_URL = "https://api.judgmentlabs
 export declare const JUDGMENT_TRACES_FETCH_API_URL = "https://api.judgmentlabs.ai/traces/fetch/";
 export declare const JUDGMENT_TRACES_SAVE_API_URL = "https://api.judgmentlabs.ai/traces/save/";
 export declare const JUDGMENT_TRACES_DELETE_API_URL = "https://api.judgmentlabs.ai/traces/delete/";
-export declare const JUDGMENT_TRACES_ADD_TO_EVAL_QUEUE_API_URL = "https://api.judgmentlabs.ai/traces/add_to_trace_eval_queue/";
-export declare const JUDGMENT_WEBSOCKET_URL: string;
+export declare const JUDGMENT_TRACES_ADD_TO_EVAL_QUEUE_API_URL = "https://api.judgmentlabs.ai/traces/add_to_eval_queue/";
+export declare const JUDGMENT_CALCULATE_TOKEN_COSTS_API_URL = "https://api.judgmentlabs.ai/calculate-token-costs";
 export declare const JUDGMENT_ADD_TO_RUN_EVAL_QUEUE_API_URL = "https://api.judgmentlabs.ai/add_to_run_eval_queue/";
 export declare const RABBITMQ_HOST: string;
 export declare const RABBITMQ_PORT: number;

package/dist/types/data/datasets/eval-dataset-client.d.ts

@@ -0,0 +1,39 @@
+import { Example } from '../example.js';
+import { EvalDataset } from './eval-dataset.js';
+import { AxiosResponse } from 'axios';
+export declare class EvalDatasetClient {
+    private judgmentApiKey;
+    private organizationId;
+    constructor(judgmentApiKey: string, organizationId: string);
+    createDataset(examples?: Example[]): EvalDataset;
+    /**
+     * Pushes the dataset to the Judgment platform.
+     * @returns True if successful, false otherwise.
+     */
+    pushDataset(dataset: EvalDataset, alias: string, projectName: string, overwrite?: boolean): Promise<boolean>;
+    /**
+     * Pulls the dataset from the Judgment platform.
+     */
+    pullDataset(alias: string, projectName: string): Promise<EvalDataset>;
+    /**
+     * Deletes the dataset from the Judgment platform.
+     * @returns True if successful, false otherwise.
+     */
+    deleteDataset(alias: string, projectName: string): Promise<boolean>;
+    /**
+     * Pulls dataset statistics for a project from the Judgment platform.
+     */
+    pullProjectDatasetStats(projectName: string): Promise<Record<string, any>>;
+    /**
+     * Inserts new examples into an existing dataset on the Judgment platform.
+     * @returns True if successful, false otherwise.
+     */
+    insertDataset(alias: string, examples: Example[], projectName: string): Promise<boolean>;
+    /**
+     * Exports a dataset in JSONL format from the Judgment platform.
+     * @returns AxiosResponse containing the stream if successful.
+     */
+    exportJsonl(alias: string, projectName: string): Promise<AxiosResponse>;
+    private getAuthHeaders;
+    private handleApiError;
+}

package/dist/types/data/datasets/eval-dataset.d.ts

@@ -0,0 +1,45 @@
+import { Example, ExampleOptions } from '../example.js';
+type SaveFileType = 'json' | 'csv' | 'yaml';
+export declare class EvalDataset {
+    examples: Example[];
+    private _alias;
+    private _id;
+    constructor(examples?: Example[]);
+    addExample(e: Example): void;
+    get length(): number;
+    get alias(): string | null;
+    set alias(value: string | null);
+    get id(): string | null;
+    set id(value: string | null);
+    /**
+     * Adds examples from a JSON file.
+     * Assumes the JSON file has a top-level key "examples" containing an array of example objects.
+     * @param filePath Path to the JSON file.
+     */
+    addFromJson(filePath: string): void;
+    /**
+     * Adds examples from a YAML file.
+     * Assumes the YAML file has a top-level key "examples" containing an array of example objects.
+     * @param filePath Path to the YAML file.
+     */
+    addFromYaml(filePath: string): void;
+    /**
+     * Adds examples from a CSV file.
+     * @param filePath Path to the CSV file.
+     * @param headerMapping Dictionary mapping Example headers (keys) to custom headers in the CSV (values).
+     * @param primaryDelimiter Main delimiter used in CSV file. Defaults to ",".
+     * @param secondaryDelimiter Secondary delimiter for list fields (context, retrieval_context, etc.). Defaults to ";".
+     */
+    addFromCsv(filePath: string, headerMapping: {
+        [key in keyof ExampleOptions]?: string;
+    }, primaryDelimiter?: string, secondaryDelimiter?: string): void;
+    /**
+     * Saves the dataset as a file.
+     * @param fileType The file type to save as ('json', 'csv', 'yaml').
+     * @param dirPath The directory path to save the file to.
+     * @param saveName Optional: The name of the file (without extension). Defaults to a timestamp.
+     * @param secondaryDelimiter Optional: The delimiter used for joining list fields in CSV output. Defaults to ";".
+     */
+    saveAs(fileType: SaveFileType, dirPath: string, saveName?: string, secondaryDelimiter?: string): void;
+}
+export {};

package/dist/types/data/example.d.ts

@@ -3,29 +3,35 @@
  */
 export interface ExampleOptions {
     input: string;
-    actualOutput?: string;
-    expectedOutput?: string;
+    actualOutput?: string | string[];
+    expectedOutput?: string | string[];
     context?: string[];
     retrievalContext?: string[];
     additionalMetadata?: Record<string, any>;
-    toolsCalled?: any[];
-    expectedTools?: any[];
+    toolsCalled?: string[];
+    expectedTools?: string[];
+    name?: string;
     exampleId?: string;
     exampleIndex?: number;
     timestamp?: string;
+    traceId?: string;
+    example?: boolean;
 }
 export declare class Example {
     input: string;
-    actualOutput?: string;
-    expectedOutput?: string;
+    actualOutput?: string | string[];
+    expectedOutput?: string | string[];
     context?: string[];
     retrievalContext?: string[];
     additionalMetadata?: Record<string, any>;
-    toolsCalled?: any[];
-    expectedTools?: any[];
+    toolsCalled?: string[];
+    expectedTools?: string[];
+    name?: string;
     exampleId: string;
     exampleIndex?: number;
     timestamp?: string;
+    traceId?: string;
+    example?: boolean;
     constructor(options: ExampleOptions);
     /**
      * Generate a UUID for the example ID
@@ -52,19 +58,25 @@ export declare class ExampleBuilder {
     private _additionalMetadata?;
     private _toolsCalled?;
     private _expectedTools?;
+    private _name?;
     private _exampleId?;
     private _exampleIndex?;
     private _timestamp?;
+    private _traceId?;
+    private _example?;
     input(input: string): ExampleBuilder;
-    actualOutput(actualOutput: string): ExampleBuilder;
-    expectedOutput(expectedOutput: string): ExampleBuilder;
+    actualOutput(actualOutput: string | string[]): ExampleBuilder;
+    expectedOutput(expectedOutput: string | string[]): ExampleBuilder;
     context(context: string[]): ExampleBuilder;
     retrievalContext(retrievalContext: string[]): ExampleBuilder;
     additionalMetadata(additionalMetadata: Record<string, any>): ExampleBuilder;
-    toolsCalled(toolsCalled: any[]): ExampleBuilder;
-    expectedTools(expectedTools: any[]): ExampleBuilder;
+    toolsCalled(toolsCalled: string[]): ExampleBuilder;
+    expectedTools(expectedTools: string[]): ExampleBuilder;
+    name(name: string): ExampleBuilder;
     exampleId(exampleId: string): ExampleBuilder;
     exampleIndex(exampleIndex: number): ExampleBuilder;
     timestamp(timestamp: string): ExampleBuilder;
+    traceId(traceId: string): ExampleBuilder;
+    example(example: boolean): ExampleBuilder;
     build(): Example;
 }

package/dist/types/e2etests/eval-operations.test.d.ts

@@ -0,0 +1,5 @@
+/**
+ * E2E tests for evaluation operations in the JudgmentClient.
+ * Migrated from the Python SDK's test_eval_operations.py
+ */
+export {};

package/dist/types/e2etests/judgee-traces.test.d.ts

@@ -0,0 +1,5 @@
+/**
+ * E2E tests for judgee traces operations in the Tracer API.
+ * Migrated from the Python SDK's test_judgee_traces_update.py
+ */
+export {};

package/dist/types/index.d.ts

@@ -3,7 +3,6 @@ export { ScoringResult, ScoringResultBuilder, ScorerData, ScoringResultOptions }
 export { Tracer, SpanType, wrap, TraceClient } from './common/tracer.js';
 export { Scorer, APIJudgmentScorer, JudgevalScorer, ScorerWrapper } from './scorers/base-scorer.js';
 export { AnswerCorrectnessScorer, AnswerRelevancyScorer, ComparisonScorer, ContextualPrecisionScorer, ContextualRecallScorer, ContextualRelevancyScorer, ExecutionOrderScorer, FaithfulnessScorer, GroundednessScorer, HallucinationScorer, InstructionAdherenceScorer, JsonCorrectnessScorer, SummarizationScorer } from './scorers/api-scorer.js';
-export { ExactMatchScorer } from './scorers/exact-match-scorer.js';
 export { AlertStatus, Condition, NotificationConfig, Rule, AlertResult, RulesEngine } from './rules.js';
 export { EvaluationRun, EvaluationRunOptions } from './evaluation-run.js';
 export { runEval, assertTest, JudgmentAPIError, sendToRabbitMQ, executeApiEval, mergeResults, checkMissingScorerData, checkEvalRunNameExists, logEvaluationResults, checkExamples } from './run-evaluation.js';

package/dist/types/judgment-client.d.ts

@@ -50,32 +50,8 @@ export declare class JudgmentClient {
     /**
      * Evaluate a dataset
      */
-    evaluateDataset(dataset: any, // EvalDataset would be implemented separately
+    evaluateDataset(dataset: any, // Keep type loose for stub
     scorers: Array<ScorerWrapper | JudgevalScorer>, model: string | string[] | any, aggregator?: string, metadata?: Record<string, any>, projectName?: string, evalRunName?: string, logResults?: boolean, useJudgment?: boolean, rules?: Rule[]): Promise<ScoringResult[]>;
-    /**
-     * Create a dataset
-     */
-    createDataset(): any;
-    /**
-     * Push a dataset to the Judgment platform
-     */
-    pushDataset(alias: string, dataset: any, projectName: string, overwrite?: boolean): Promise<boolean>;
-    /**
-     * Pull a dataset from the Judgment platform
-     */
-    pullDataset(alias: string, projectName: string): Promise<any>;
-    /**
-     * Delete a dataset from the Judgment platform
-     */
-    deleteDataset(alias: string, projectName: string): Promise<boolean>;
-    /**
-     * Pull project dataset stats from the Judgment platform
-     */
-    pullProjectDatasetStats(projectName: string): Promise<Record<string, any>>;
-    /**
-     * Insert examples into a dataset on the Judgment platform
-     */
-    insertDataset(alias: string, examples: Example[], projectName: string): Promise<boolean>;
     /**
      * Pull evaluation results from the server
      * @param projectName Name of the project
@@ -83,28 +59,6 @@ export declare class JudgmentClient {
      * @returns Array containing one object with 'id' and 'results' (list of ScoringResult)
      */
     pullEval(projectName: string, evalRunName: string): Promise<Array<Record<string, any | ScoringResult[]>>>;
-    /**
-     * Get evaluation run results (alias for pullEval with a more intuitive name)
-     * @param projectName Name of the project
-     * @param evalRunName Name of the evaluation run
-     * @returns Array containing one object with 'id' and 'results' (list of ScoringResult)
-     */
-    getEvalRun(projectName: string, evalRunName: string): Promise<Array<Record<string, any | ScoringResult[]>>>;
-    /**
-     * List all evaluation runs for a project
-     * @param projectName Name of the project
-     * @param limit Maximum number of evaluation runs to return (default: 100)
-     * @param offset Offset for pagination (default: 0)
-     * @returns List of evaluation run metadata
-     */
-    listEvalRuns(projectName: string, limit?: number, offset?: number): Promise<Array<Record<string, any>>>;
-    /**
-     * Get evaluation run statistics
-     * @param projectName Name of the project
-     * @param evalRunName Name of the evaluation run
-     * @returns Statistics for the evaluation run
-     */
-    getEvalRunStats(projectName: string, evalRunName: string): Promise<Record<string, any>>;
     /**
      * Export evaluation results to a file format
      * @param projectName Name of the project
@@ -176,4 +130,6 @@ export declare class JudgmentClient {
      * @returns A string representing the progress bar
      */
     private _createProgressBar;
+    private getAuthHeaders;
+    private handleApiError;
 }

package/dist/types/scorers/api-scorer.d.ts

@@ -5,67 +5,67 @@ import { ScorerData } from '../data/result.js';
  * Implementation of API-based scorers
  */
 export declare class AnswerCorrectnessScorer extends APIJudgmentScorer {
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
     a_score_example(example: Example): Promise<ScorerData>;
 }
 export declare class AnswerRelevancyScorer extends APIJudgmentScorer {
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
     a_score_example(example: Example): Promise<ScorerData>;
 }
 export declare class ComparisonScorer extends APIJudgmentScorer {
     criteria: string[];
     description: string;
-    constructor(threshold?: number, criteria?: string[], description?: string, additional_metadata?: Record<string, any>, verbose?: boolean);
+    constructor(threshold?: number, criteria?: string[], description?: string, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
     toJSON(): Record<string, any>;
     a_score_example(example: Example): Promise<ScorerData>;
 }
 export declare class ContextualPrecisionScorer extends APIJudgmentScorer {
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
     a_score_example(example: Example): Promise<ScorerData>;
 }
 export declare class ContextualRecallScorer extends APIJudgmentScorer {
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
     a_score_example(example: Example): Promise<ScorerData>;
 }
 export declare class ContextualRelevancyScorer extends APIJudgmentScorer {
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
     a_score_example(example: Example): Promise<ScorerData>;
 }
 export declare class ExecutionOrderScorer extends APIJudgmentScorer {
     strictMode: boolean;
     expectedTools?: string[];
-    constructor(threshold?: number, strictMode?: boolean, expectedTools?: string[], additional_metadata?: Record<string, any>, verbose?: boolean);
+    constructor(threshold?: number, expectedTools?: string[], additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
     toJSON(): Record<string, any>;
     a_score_example(example: Example): Promise<ScorerData>;
 }
 export declare class FaithfulnessScorer extends APIJudgmentScorer {
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
     a_score_example(example: Example): Promise<ScorerData>;
 }
 export declare class GroundednessScorer extends APIJudgmentScorer {
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
     a_score_example(example: Example): Promise<ScorerData>;
 }
 export declare class HallucinationScorer extends APIJudgmentScorer {
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
     a_score_example(example: Example): Promise<ScorerData>;
 }
 export declare class InstructionAdherenceScorer extends APIJudgmentScorer {
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
     a_score_example(example: Example): Promise<ScorerData>;
 }
 export declare class JsonCorrectnessScorer extends APIJudgmentScorer {
     jsonSchema?: Record<string, any>;
-    constructor(threshold?: number, jsonSchema?: Record<string, any>, additional_metadata?: Record<string, any>, verbose?: boolean);
+    constructor(threshold?: number, jsonSchema?: Record<string, any>, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
     toJSON(): Record<string, any>;
     a_score_example(example: Example): Promise<ScorerData>;
 }
 export declare class SummarizationScorer extends APIJudgmentScorer {
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
     a_score_example(example: Example): Promise<ScorerData>;
 }
 export declare class Text2SQLScorer extends APIJudgmentScorer {
-    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
     a_score_example(example: Example): Promise<ScorerData>;
 }
 export declare class ScorerWrapper {
@@ -75,5 +75,5 @@ export declare class ScorerWrapper {
     get threshold(): number;
     get additional_metadata(): Record<string, any> | undefined;
     toJSON(): Record<string, any>;
-    static fromType(type: string, threshold: number, additional_metadata?: Record<string, any>, verbose?: boolean): APIJudgmentScorer;
+    static fromType(type: string, threshold: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean): APIJudgmentScorer;
 }

package/dist/types/scorers/base-scorer.d.ts

@@ -8,8 +8,18 @@ export interface Scorer {
     scoreType: string;
     threshold: number;
     score?: number;
+    score_breakdown?: Record<string, any>;
+    reason?: string;
+    success?: boolean;
+    evaluation_model?: string;
+    strict_mode: boolean;
+    async_mode: boolean;
+    verbose_mode: boolean;
+    include_reason: boolean;
+    error?: string;
+    evaluation_cost?: number;
+    verbose_logs?: string;
     additional_metadata?: Record<string, any>;
-    verbose: boolean;
     validateThreshold(): void;
     toJSON(): Record<string, any>;
     successCheck(): boolean;
@@ -22,9 +32,13 @@ export declare abstract class APIJudgmentScorer implements Scorer {
     get scoreType(): string;
     readonly threshold: number;
     score?: number;
+    score_breakdown?: Record<string, any>;
     additional_metadata?: Record<string, any>;
-    verbose: boolean;
-    constructor(type: string, threshold: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    strict_mode: boolean;
+    async_mode: boolean;
+    verbose_mode: boolean;
+    include_reason: boolean;
+    constructor(type: string, threshold: number, additional_metadata?: Record<string, any>, strict_mode?: boolean, async_mode?: boolean, verbose_mode?: boolean, include_reason?: boolean);
     /**
      * Check if the score meets the threshold
      */
@@ -47,27 +61,46 @@ export declare abstract class JudgevalScorer implements Scorer {
     scoreType: string;
     threshold: number;
     score?: number;
+    score_breakdown?: Record<string, any>;
+    reason?: string;
+    success?: boolean;
+    evaluation_model?: string;
+    strict_mode: boolean;
+    async_mode: boolean;
+    verbose_mode: boolean;
+    include_reason: boolean;
+    error?: string;
+    evaluation_cost?: number;
+    verbose_logs?: string;
     additional_metadata?: Record<string, any>;
-    verbose: boolean;
-    constructor(type: string, threshold: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    constructor(type: string, threshold: number, additional_metadata?: Record<string, any>, include_reason?: boolean, async_mode?: boolean, strict_mode?: boolean, verbose_mode?: boolean);
     /**
      * Check if the score meets the threshold
      */
     successCheck(): boolean;
+    /**
+     * Internal method to check success
+     * This is equivalent to Python's _success_check method
+     */
+    protected _successCheck(): boolean;
     /**
      * Validate that the threshold is within the allowed range
      */
     validateThreshold(): void;
+    /**
+     * Convert the scorer to a plain object
+     */
+    toJSON(): Record<string, any>;
     /**
      * Score an example
-     * @param example The example to score
-     * @returns A ScorerData object with the score
+     * This must be implemented by subclasses
      */
     abstract scoreExample(example: Example): Promise<ScorerData>;
     /**
-     * Convert the scorer to a plain object
+     * Get the name of the scorer
+     * This is equivalent to Python's __name__ property
      */
-    toJSON(): Record<string, any>;
+    get name(): string;
 }
 /**
  * Wrapper for scorers to allow dynamic loading of implementations
@@ -77,8 +110,18 @@ export declare class ScorerWrapper implements Scorer {
     scoreType: string;
     threshold: number;
     score?: number;
+    score_breakdown?: Record<string, any>;
+    reason?: string;
+    success?: boolean;
+    evaluation_model?: string;
+    strict_mode: boolean;
+    async_mode: boolean;
+    verbose_mode: boolean;
+    include_reason: boolean;
+    error?: string;
+    evaluation_cost?: number;
+    verbose_logs?: string;
     additional_metadata?: Record<string, any>;
-    verbose: boolean;
     scorer: any;
     constructor(scorer: any);
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "judgeval",
-  "version": "0.2.0",
+  "version": "0.2.2",
   "description": "Judgment SDK for TypeScript/JavaScript",
   "main": "./dist/cjs/index.js",
   "module": "./dist/esm/index.js",
@@ -49,6 +49,7 @@
     "@types/node": "^20.12.12",
     "@typescript-eslint/eslint-plugin": "^7.10.0",
     "@typescript-eslint/parser": "^7.10.0",
+    "cross-env": "^7.0.3",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-prettier": "^5.1.3",