judgeval 0.1.33

package/dist/scorers/api-scorer.d.ts

@@ -0,0 +1,79 @@
+import { APIJudgmentScorer } from './base-scorer';
+import { Example } from '../data/example';
+import { ScorerData } from '../data/result';
+/**
+ * Implementation of API-based scorers
+ */
+export declare class AnswerCorrectnessScorer extends APIJudgmentScorer {
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    a_score_example(example: Example): Promise<ScorerData>;
+}
+export declare class AnswerRelevancyScorer extends APIJudgmentScorer {
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    a_score_example(example: Example): Promise<ScorerData>;
+}
+export declare class ComparisonScorer extends APIJudgmentScorer {
+    criteria: string[];
+    description: string;
+    constructor(threshold?: number, criteria?: string[], description?: string, additional_metadata?: Record<string, any>, verbose?: boolean);
+    toJSON(): Record<string, any>;
+    a_score_example(example: Example): Promise<ScorerData>;
+}
+export declare class ContextualPrecisionScorer extends APIJudgmentScorer {
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    a_score_example(example: Example): Promise<ScorerData>;
+}
+export declare class ContextualRecallScorer extends APIJudgmentScorer {
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    a_score_example(example: Example): Promise<ScorerData>;
+}
+export declare class ContextualRelevancyScorer extends APIJudgmentScorer {
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    a_score_example(example: Example): Promise<ScorerData>;
+}
+export declare class ExecutionOrderScorer extends APIJudgmentScorer {
+    strictMode: boolean;
+    expectedTools?: string[];
+    constructor(threshold?: number, strictMode?: boolean, expectedTools?: string[], additional_metadata?: Record<string, any>, verbose?: boolean);
+    toJSON(): Record<string, any>;
+    a_score_example(example: Example): Promise<ScorerData>;
+}
+export declare class FaithfulnessScorer extends APIJudgmentScorer {
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    a_score_example(example: Example): Promise<ScorerData>;
+}
+export declare class GroundednessScorer extends APIJudgmentScorer {
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    a_score_example(example: Example): Promise<ScorerData>;
+}
+export declare class HallucinationScorer extends APIJudgmentScorer {
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    a_score_example(example: Example): Promise<ScorerData>;
+}
+export declare class InstructionAdherenceScorer extends APIJudgmentScorer {
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    a_score_example(example: Example): Promise<ScorerData>;
+}
+export declare class JsonCorrectnessScorer extends APIJudgmentScorer {
+    jsonSchema?: Record<string, any>;
+    constructor(threshold?: number, jsonSchema?: Record<string, any>, additional_metadata?: Record<string, any>, verbose?: boolean);
+    toJSON(): Record<string, any>;
+    a_score_example(example: Example): Promise<ScorerData>;
+}
+export declare class SummarizationScorer extends APIJudgmentScorer {
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    a_score_example(example: Example): Promise<ScorerData>;
+}
+export declare class Text2SQLScorer extends APIJudgmentScorer {
+    constructor(threshold?: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    a_score_example(example: Example): Promise<ScorerData>;
+}
+export declare class ScorerWrapper {
+    private scorer;
+    constructor(scorer: APIJudgmentScorer);
+    get scoreType(): string;
+    get threshold(): number;
+    get additional_metadata(): Record<string, any> | undefined;
+    toJSON(): Record<string, any>;
+    static fromType(type: string, threshold: number, additional_metadata?: Record<string, any>, verbose?: boolean): APIJudgmentScorer;
+}

package/dist/scorers/api-scorer.js

@@ -0,0 +1,291 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ScorerWrapper = exports.Text2SQLScorer = exports.SummarizationScorer = exports.JsonCorrectnessScorer = exports.InstructionAdherenceScorer = exports.HallucinationScorer = exports.GroundednessScorer = exports.FaithfulnessScorer = exports.ExecutionOrderScorer = exports.ContextualRelevancyScorer = exports.ContextualRecallScorer = exports.ContextualPrecisionScorer = exports.ComparisonScorer = exports.AnswerRelevancyScorer = exports.AnswerCorrectnessScorer = void 0;
+const base_scorer_1 = require("./base-scorer");
+/**
+ * Implementation of API-based scorers
+ */
+class AnswerCorrectnessScorer extends base_scorer_1.APIJudgmentScorer {
+    constructor(threshold = 0.7, additional_metadata, verbose = false) {
+        super('answer_correctness', threshold, additional_metadata, verbose);
+        this.validateThreshold();
+    }
+    a_score_example(example) {
+        return __awaiter(this, void 0, void 0, function* () {
+            throw new Error('API scorers are evaluated on the server side');
+        });
+    }
+}
+exports.AnswerCorrectnessScorer = AnswerCorrectnessScorer;
+class AnswerRelevancyScorer extends base_scorer_1.APIJudgmentScorer {
+    constructor(threshold = 0.7, additional_metadata, verbose = false) {
+        super('answer_relevancy', threshold, additional_metadata, verbose);
+        this.validateThreshold();
+    }
+    a_score_example(example) {
+        return __awaiter(this, void 0, void 0, function* () {
+            throw new Error('API scorers are evaluated on the server side');
+        });
+    }
+}
+exports.AnswerRelevancyScorer = AnswerRelevancyScorer;
+class ComparisonScorer extends base_scorer_1.APIJudgmentScorer {
+    constructor(threshold = 0.5, criteria = ['Accuracy', 'Helpfulness', 'Relevance'], description = 'Compare the outputs based on the given criteria', additional_metadata, verbose = false) {
+        super('comparison', threshold, additional_metadata, verbose);
+        this.criteria = criteria;
+        this.description = description;
+        // Comparison is an unbounded scorer, only validate that threshold >= 0
+        if (threshold < 0) {
+            throw new Error(`Threshold for comparison must be greater than or equal to 0, got: ${threshold}`);
+        }
+    }
+    toJSON() {
+        return {
+            score_type: 'comparison',
+            threshold: this.threshold,
+            criteria: this.criteria,
+            description: this.description,
+            additional_metadata: this.additional_metadata,
+            verbose: this.verbose
+        };
+    }
+    a_score_example(example) {
+        return __awaiter(this, void 0, void 0, function* () {
+            throw new Error('API scorers are evaluated on the server side');
+        });
+    }
+}
+exports.ComparisonScorer = ComparisonScorer;
+class ContextualPrecisionScorer extends base_scorer_1.APIJudgmentScorer {
+    constructor(threshold = 0.7, additional_metadata, verbose = false) {
+        super('contextual_precision', threshold, additional_metadata, verbose);
+        this.validateThreshold();
+    }
+    a_score_example(example) {
+        return __awaiter(this, void 0, void 0, function* () {
+            throw new Error('API scorers are evaluated on the server side');
+        });
+    }
+}
+exports.ContextualPrecisionScorer = ContextualPrecisionScorer;
+class ContextualRecallScorer extends base_scorer_1.APIJudgmentScorer {
+    constructor(threshold = 0.7, additional_metadata, verbose = false) {
+        super('contextual_recall', threshold, additional_metadata, verbose);
+        this.validateThreshold();
+    }
+    a_score_example(example) {
+        return __awaiter(this, void 0, void 0, function* () {
+            throw new Error('API scorers are evaluated on the server side');
+        });
+    }
+}
+exports.ContextualRecallScorer = ContextualRecallScorer;
+class ContextualRelevancyScorer extends base_scorer_1.APIJudgmentScorer {
+    constructor(threshold = 0.7, additional_metadata, verbose = false) {
+        super('contextual_relevancy', threshold, additional_metadata, verbose);
+        this.validateThreshold();
+    }
+    a_score_example(example) {
+        return __awaiter(this, void 0, void 0, function* () {
+            throw new Error('API scorers are evaluated on the server side');
+        });
+    }
+}
+exports.ContextualRelevancyScorer = ContextualRelevancyScorer;
+class ExecutionOrderScorer extends base_scorer_1.APIJudgmentScorer {
+    constructor(threshold = 1.0, strictMode = true, expectedTools, additional_metadata, verbose = false) {
+        super('execution_order', threshold, additional_metadata, verbose);
+        this.strictMode = strictMode;
+        this.expectedTools = expectedTools;
+        this.validateThreshold();
+    }
+    toJSON() {
+        return {
+            score_type: 'execution_order',
+            threshold: this.threshold,
+            strict_mode: this.strictMode,
+            expected_tools: this.expectedTools,
+            additional_metadata: this.additional_metadata,
+            verbose: this.verbose
+        };
+    }
+    a_score_example(example) {
+        return __awaiter(this, void 0, void 0, function* () {
+            throw new Error('API scorers are evaluated on the server side');
+        });
+    }
+}
+exports.ExecutionOrderScorer = ExecutionOrderScorer;
+class FaithfulnessScorer extends base_scorer_1.APIJudgmentScorer {
+    constructor(threshold = 0.7, additional_metadata, verbose = false) {
+        super('faithfulness', threshold, additional_metadata, verbose);
+        this.validateThreshold();
+    }
+    a_score_example(example) {
+        return __awaiter(this, void 0, void 0, function* () {
+            throw new Error('API scorers are evaluated on the server side');
+        });
+    }
+}
+exports.FaithfulnessScorer = FaithfulnessScorer;
+class GroundednessScorer extends base_scorer_1.APIJudgmentScorer {
+    constructor(threshold = 0.7, additional_metadata, verbose = false) {
+        super('groundedness', threshold, additional_metadata, verbose);
+        this.validateThreshold();
+    }
+    a_score_example(example) {
+        return __awaiter(this, void 0, void 0, function* () {
+            throw new Error('API scorers are evaluated on the server side');
+        });
+    }
+}
+exports.GroundednessScorer = GroundednessScorer;
+class HallucinationScorer extends base_scorer_1.APIJudgmentScorer {
+    constructor(threshold = 0.7, additional_metadata, verbose = false) {
+        super('hallucination', threshold, additional_metadata, verbose);
+        this.validateThreshold();
+    }
+    a_score_example(example) {
+        return __awaiter(this, void 0, void 0, function* () {
+            throw new Error('API scorers are evaluated on the server side');
+        });
+    }
+}
+exports.HallucinationScorer = HallucinationScorer;
+class InstructionAdherenceScorer extends base_scorer_1.APIJudgmentScorer {
+    constructor(threshold = 0.7, additional_metadata, verbose = false) {
+        super('instruction_adherence', threshold, additional_metadata, verbose);
+        this.validateThreshold();
+    }
+    a_score_example(example) {
+        return __awaiter(this, void 0, void 0, function* () {
+            throw new Error('API scorers are evaluated on the server side');
+        });
+    }
+}
+exports.InstructionAdherenceScorer = InstructionAdherenceScorer;
+class JsonCorrectnessScorer extends base_scorer_1.APIJudgmentScorer {
+    constructor(threshold = 0.7, jsonSchema, additional_metadata, verbose = false) {
+        super('json_correctness', threshold, additional_metadata, verbose);
+        this.jsonSchema = jsonSchema;
+        this.validateThreshold();
+    }
+    toJSON() {
+        return {
+            score_type: 'json_correctness',
+            threshold: this.threshold,
+            json_schema: this.jsonSchema,
+            additional_metadata: this.additional_metadata,
+            verbose: this.verbose
+        };
+    }
+    a_score_example(example) {
+        return __awaiter(this, void 0, void 0, function* () {
+            throw new Error('API scorers are evaluated on the server side');
+        });
+    }
+}
+exports.JsonCorrectnessScorer = JsonCorrectnessScorer;
+class SummarizationScorer extends base_scorer_1.APIJudgmentScorer {
+    constructor(threshold = 0.7, additional_metadata, verbose = false) {
+        super('summarization', threshold, additional_metadata, verbose);
+        this.validateThreshold();
+    }
+    a_score_example(example) {
+        return __awaiter(this, void 0, void 0, function* () {
+            throw new Error('API scorers are evaluated on the server side');
+        });
+    }
+}
+exports.SummarizationScorer = SummarizationScorer;
+class Text2SQLScorer extends base_scorer_1.APIJudgmentScorer {
+    constructor(threshold = 0.7, additional_metadata, verbose = false) {
+        super('text2sql', threshold, additional_metadata, verbose);
+        this.validateThreshold();
+    }
+    a_score_example(example) {
+        return __awaiter(this, void 0, void 0, function* () {
+            throw new Error('API scorers are evaluated on the server side');
+        });
+    }
+}
+exports.Text2SQLScorer = Text2SQLScorer;
+// Create a ScorerWrapper class to dynamically load the appropriate implementation
+class ScorerWrapper {
+    constructor(scorer) {
+        this.scorer = scorer;
+    }
+    get scoreType() {
+        return this.scorer.scoreType;
+    }
+    get threshold() {
+        return this.scorer.threshold;
+    }
+    get additional_metadata() {
+        return this.scorer.additional_metadata;
+    }
+    toJSON() {
+        return this.scorer.toJSON();
+    }
+    static fromType(type, threshold, additional_metadata, verbose = false) {
+        switch (type.toLowerCase()) {
+            case 'answer_correctness':
+                return new AnswerCorrectnessScorer(threshold, additional_metadata, verbose);
+            case 'answer_relevancy':
+                return new AnswerRelevancyScorer(threshold, additional_metadata, verbose);
+            case 'comparison':
+                // For comparison, extract criteria and description from metadata if available
+                const criteria = (additional_metadata === null || additional_metadata === void 0 ? void 0 : additional_metadata.criteria) || ['Accuracy', 'Helpfulness', 'Relevance'];
+                const description = (additional_metadata === null || additional_metadata === void 0 ? void 0 : additional_metadata.description) || 'Compare the outputs based on the given criteria';
+                const comparisonMetadata = Object.assign({}, additional_metadata);
+                comparisonMetadata === null || comparisonMetadata === void 0 ? true : delete comparisonMetadata.criteria;
+                comparisonMetadata === null || comparisonMetadata === void 0 ? true : delete comparisonMetadata.description;
+                return new ComparisonScorer(threshold, criteria, description, comparisonMetadata, verbose);
+            case 'contextual_precision':
+                return new ContextualPrecisionScorer(threshold, additional_metadata, verbose);
+            case 'contextual_recall':
+                return new ContextualRecallScorer(threshold, additional_metadata, verbose);
+            case 'contextual_relevancy':
+                return new ContextualRelevancyScorer(threshold, additional_metadata, verbose);
+            case 'execution_order':
+                // For execution order, extract strict_mode and expected_tools from metadata if available
+                const strictMode = (additional_metadata === null || additional_metadata === void 0 ? void 0 : additional_metadata.strict_mode) || true;
+                const expectedTools = additional_metadata === null || additional_metadata === void 0 ? void 0 : additional_metadata.expected_tools;
+                const executionOrderMetadata = Object.assign({}, additional_metadata);
+                executionOrderMetadata === null || executionOrderMetadata === void 0 ? true : delete executionOrderMetadata.strict_mode;
+                executionOrderMetadata === null || executionOrderMetadata === void 0 ? true : delete executionOrderMetadata.expected_tools;
+                return new ExecutionOrderScorer(threshold, strictMode, expectedTools, executionOrderMetadata, verbose);
+            case 'faithfulness':
+                return new FaithfulnessScorer(threshold, additional_metadata, verbose);
+            case 'groundedness':
+                return new GroundednessScorer(threshold, additional_metadata, verbose);
+            case 'hallucination':
+                return new HallucinationScorer(threshold, additional_metadata, verbose);
+            case 'instruction_adherence':
+                return new InstructionAdherenceScorer(threshold, additional_metadata, verbose);
+            case 'json_correctness':
+                // For JSON correctness, extract json_schema from metadata if available
+                const jsonSchema = additional_metadata === null || additional_metadata === void 0 ? void 0 : additional_metadata.json_schema;
+                const jsonMetadata = Object.assign({}, additional_metadata);
+                jsonMetadata === null || jsonMetadata === void 0 ? true : delete jsonMetadata.json_schema;
+                return new JsonCorrectnessScorer(threshold, jsonSchema, jsonMetadata, verbose);
+            case 'summarization':
+                return new SummarizationScorer(threshold, additional_metadata, verbose);
+            case 'text2sql':
+                return new Text2SQLScorer(threshold, additional_metadata, verbose);
+            default:
+                throw new Error(`Unknown scorer type: ${type}`);
+        }
+    }
+}
+exports.ScorerWrapper = ScorerWrapper;
+//# sourceMappingURL=api-scorer.js.map

package/dist/scorers/api-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"api-scorer.js","sourceRoot":"","sources":["../../src/scorers/api-scorer.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,+CAAkD;AAKlD;;GAEG;AACH,MAAa,uBAAwB,SAAQ,+BAAiB;IAC5D,YAAY,YAAoB,GAAG,EAAE,mBAAyC,EAAE,UAAmB,KAAK;QACtG,KAAK,CAAC,oBAAoB,EAAE,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QACrE,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAEK,eAAe,CAAC,OAAgB;;YACpC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;KAAA;CACF;AATD,0DASC;AAED,MAAa,qBAAsB,SAAQ,+BAAiB;IAC1D,YAAY,YAAoB,GAAG,EAAE,mBAAyC,EAAE,UAAmB,KAAK;QACtG,KAAK,CAAC,kBAAkB,EAAE,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QACnE,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAEK,eAAe,CAAC,OAAgB;;YACpC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;KAAA;CACF;AATD,sDASC;AAED,MAAa,gBAAiB,SAAQ,+BAAiB;IAIrD,YACE,YAAoB,GAAG,EACvB,WAAqB,CAAC,UAAU,EAAE,aAAa,EAAE,WAAW,CAAC,EAC7D,cAAsB,iDAAiD,EACvE,mBAAyC,EACzC,UAAmB,KAAK;QAExB,KAAK,CAAC,YAAY,EAAE,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QAC7D,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,uEAAuE;QACvE,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,qEAAqE,SAAS,EAAE,CAAC,CAAC;QACpG,CAAC;IACH,CAAC;IAED,MAAM;QACJ,OAAO;YACL,UAAU,EAAE,YAAY;YACxB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC;IACJ,CAAC;IAEK,eAAe,CAAC,OAAgB;;YACpC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;KAAA;CACF;AAlCD,4CAkCC;AAED,MAAa,yBAA0B,SAAQ,+BAAiB;IAC9D,YAAY,YAAoB,GAAG,EAAE,mBAAyC,EAAE,UAAmB,KAAK;QACtG,KAAK,CAAC,sBAAsB,EAAE,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QACvE,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAEK,eAAe,CAAC,OAAgB;;YACpC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;KAAA;CACF;AATD,8DASC;AAED,MAAa,sBAAuB,SAAQ,+BAAiB;IAC3D,YAAY,YAAoB,GAAG,EAAE,mBAAyC,EAAE,UAAmB,KAAK;QACtG,KAAK,CAAC,mBAAmB,EAAE,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QACpE,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAEK,eAAe,CAAC,OAAgB;;YACpC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;KAAA;CACF;AATD,wDASC;AAED,MAAa,yBAA0B,SAAQ,+BAAiB;IAC9D,YAAY,YAAoB,GAAG,EAAE,mBAAyC,EAAE,UAAmB,KAAK;QACtG,KAAK,CAAC,sBAAsB,EAAE,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QACvE,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAEK,eAAe,CAAC,OAAgB;;YACpC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;KAAA;CACF;AATD,8DASC;AAED,MAAa,oBAAqB,SAAQ,+BAAiB;IAIzD,YAAY,YAAoB,GAAG,EAAE,aAAsB,IAAI,EAAE,aAAwB,EAAE,mBAAyC,EAAE,UAAmB,KAAK;QAC5J,KAAK,CAAC,iBAAiB,EAAE,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QAClE,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAED,MAAM;QACJ,OAAO;YACL,UAAU,EAAE,iBAAiB;YAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,WAAW,EAAE,IAAI,CAAC,UAAU;YAC5B,cAAc,EAAE,IAAI,CAAC,aAAa;YAClC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC;IACJ,CAAC;IAEK,eAAe,CAAC,OAAgB;;YACpC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;KAAA;CACF;AAzBD,oDAyBC;AAED,MAAa,kBAAmB,SAAQ,+BAAiB;IACvD,YAAY,YAAoB,GAAG,EAAE,mBAAyC,EAAE,UAAmB,KAAK;QACtG,KAAK,CAAC,cAAc,EAAE,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QAC/D,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAEK,eAAe,CAAC,OAAgB;;YACpC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;KAAA;CACF;AATD,gDASC;AAED,MAAa,kBAAmB,SAAQ,+BAAiB;IACvD,YAAY,YAAoB,GAAG,EAAE,mBAAyC,EAAE,UAAmB,KAAK;QACtG,KAAK,CAAC,cAAc,EAAE,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QAC/D,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAEK,eAAe,CAAC,OAAgB;;YACpC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;KAAA;CACF;AATD,gDASC;AAED,MAAa,mBAAoB,SAAQ,+BAAiB;IACxD,YAAY,YAAoB,GAAG,EAAE,mBAAyC,EAAE,UAAmB,KAAK;QACtG,KAAK,CAAC,eAAe,EAAE,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QAChE,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAEK,eAAe,CAAC,OAAgB;;YACpC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;KAAA;CACF;AATD,kDASC;AAED,MAAa,0BAA2B,SAAQ,+BAAiB;IAC/D,YAAY,YAAoB,GAAG,EAAE,mBAAyC,EAAE,UAAmB,KAAK;QACtG,KAAK,CAAC,uBAAuB,EAAE,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QACxE,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAEK,eAAe,CAAC,OAAgB;;YACpC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;KAAA;CACF;AATD,gEASC;AAED,MAAa,qBAAsB,SAAQ,+BAAiB;IAG1D,YACE,YAAoB,GAAG,EACvB,UAAgC,EAChC,mBAAyC,EACzC,UAAmB,KAAK;QAExB,KAAK,CAAC,kBAAkB,EAAE,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QACnE,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAED,MAAM;QACJ,OAAO;YACL,UAAU,EAAE,kBAAkB;YAC9B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,WAAW,EAAE,IAAI,CAAC,UAAU;YAC5B,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC;IACJ,CAAC;IAEK,eAAe,CAAC,OAAgB;;YACpC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;KAAA;CACF;AA3BD,sDA2BC;AAED,MAAa,mBAAoB,SAAQ,+BAAiB;IACxD,YAAY,YAAoB,GAAG,EAAE,mBAAyC,EAAE,UAAmB,KAAK;QACtG,KAAK,CAAC,eAAe,EAAE,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QAChE,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAEK,eAAe,CAAC,OAAgB;;YACpC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;KAAA;CACF;AATD,kDASC;AAED,MAAa,cAAe,SAAQ,+BAAiB;IACnD,YAAY,YAAoB,GAAG,EAAE,mBAAyC,EAAE,UAAmB,KAAK;QACtG,KAAK,CAAC,UAAU,EAAE,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QAC3D,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAEK,eAAe,CAAC,OAAgB;;YACpC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;KAAA;CACF;AATD,wCASC;AAED,kFAAkF;AAClF,MAAa,aAAa;IAGxB,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;IAC/B,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;IAC/B,CAAC;IAED,IAAI,mBAAmB;QACrB,OAAO,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC;IACzC,CAAC;IAED,MAAM;QACJ,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;IAC9B,CAAC;IAED,MAAM,CAAC,QAAQ,CAAC,IAAY,EAAE,SAAiB,EAAE,mBAAyC,EAAE,UAAmB,KAAK;QAClH,QAAQ,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YAC3B,KAAK,oBAAoB;gBACvB,OAAO,IAAI,uBAAuB,CAAC,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;YAC9E,KAAK,kBAAkB;gBACrB,OAAO,IAAI,qBAAqB,CAAC,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;YAC5E,KAAK,YAAY;gBACf,8EAA8E;gBAC9E,MAAM,QAAQ,GAAG,CAAA,mBAAmB,aAAnB,mBAAmB,uBAAnB,mBAAmB,CAAE,QAAoB,KAAI,CAAC,UAAU,EAAE,aAAa,EAAE,WAAW,CAAC,CAAC;gBACvG,MAAM,WAAW,GAAG,CAAA,mBAAmB,aAAnB,mBAAmB,uBAAnB,mBAAmB,CAAE,WAAqB,KAAI,iDAAiD,CAAC;gBACpH,MAAM,kBAAkB,qBAAQ,mBAAmB,CAAE,CAAC;gBAC/C,kBAAkB,aAAlB,kBAAkB,4BAAlB,kBAAkB,CAAE,QAAQ,CAAC;gBAC7B,kBAAkB,aAAlB,kBAAkB,4BAAlB,kBAAkB,CAAE,WAAW,CAAC;gBACvC,OAAO,IAAI,gBAAgB,CAAC,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,kBAAkB,EAAE,OAAO,CAAC,CAAC;YAC7F,KAAK,sBAAsB;gBACzB,OAAO,IAAI,yBAAyB,CAAC,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;YAChF,KAAK,mBAAmB;gBACtB,OAAO,IAAI,sBAAsB,CAAC,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;YAC7E,KAAK,sBAAsB;gBACzB,OAAO,IAAI,yBAAyB,CAAC,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;YAChF,KAAK,iBAAiB;gBACpB,yFAAyF;gBACzF,MAAM,UAAU,GAAG,CAAA,mBAAmB,aAAnB,mBAAmB,uBAAnB,mBAAmB,CAAE,WAAsB,KAAI,IAAI,CAAC;gBACvE,MAAM,aAAa,GAAG,mBAAmB,aAAnB,mBAAmB,uBAAnB,mBAAmB,CAAE,cAA0B,CAAC;gBACtE,MAAM,sBAAsB,qBAAQ,mBAAmB,CAAE,CAAC;gBACnD,sBAAsB,aAAtB,sBAAsB,4BAAtB,sBAAsB,CAAE,WAAW,CAAC;gBACpC,sBAAsB,aAAtB,sBAAsB,4BAAtB,sBAAsB,CAAE,cAAc,CAAC;gBAC9C,OAAO,IAAI,oBAAoB,CAAC,SAAS,EAAE,UAAU,EAAE,aAAa,EAAE,sBAAsB,EAAE,OAAO,CAAC,CAAC;YACzG,KAAK,cAAc;gBACjB,OAAO,IAAI,kBAAkB,CAAC,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;YACzE,KAAK,cAAc;gBACjB,OAAO,IAAI,kBAAkB,CAAC,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;YACzE,KAAK,eAAe;gBAClB,OAAO,IAAI,mBAAmB,CAAC,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;YAC1E,KAAK,uBAAuB;gBAC1B,OAAO,IAAI,0BAA0B,CAAC,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;YACjF,KAAK,kBAAkB;gBACrB,uEAAuE;gBACvE,MAAM,UAAU,GAAG,mBAAmB,aAAnB,mBAAmB,uBAAnB,mBAAmB,CAAE,WAAW,CAAC;gBACpD,MAAM,YAAY,qBAAQ,mBAAmB,CAAE,CAAC;gBACzC,YAAY,aAAZ,YAAY,4BAAZ,YAAY,CAAE,WAAW,CAAC;gBACjC,OAAO,IAAI,qBAAqB,CAAC,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;YACjF,KAAK,eAAe;gBAClB,OAAO,IAAI,mBAAmB,CAAC,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;YAC1E,KAAK,UAAU;gBACb,OAAO,IAAI,cAAc,CAAC,SAAS,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;YACrE;gBACE,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;CACF;AAzED,sCAyEC"}

package/dist/scorers/base-scorer.d.ts

@@ -0,0 +1,100 @@
+import { Example } from '../data/example';
+import { ScorerData } from '../data/result';
+/**
+ * Interface for all judgment scorers
+ */
+export interface Scorer {
+    type: string;
+    scoreType: string;
+    threshold: number;
+    score?: number;
+    additional_metadata?: Record<string, any>;
+    verbose: boolean;
+    validateThreshold(): void;
+    toJSON(): Record<string, any>;
+    successCheck(): boolean;
+}
+/**
+ * Base class for API judgment scorers
+ */
+export declare abstract class APIJudgmentScorer implements Scorer {
+    readonly type: string;
+    get scoreType(): string;
+    readonly threshold: number;
+    score?: number;
+    additional_metadata?: Record<string, any>;
+    verbose: boolean;
+    constructor(type: string, threshold: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    /**
+     * Check if the score meets the threshold
+     */
+    successCheck(): boolean;
+    /**
+     * Validate that the threshold is within the allowed range
+     */
+    validateThreshold(): void;
+    /**
+     * Convert the scorer to a plain object
+     */
+    toJSON(): Record<string, any>;
+    a_score_example(example: Example): Promise<ScorerData>;
+}
+/**
+ * Base class for local judgment scorers
+ */
+export declare abstract class JudgevalScorer implements Scorer {
+    type: string;
+    scoreType: string;
+    threshold: number;
+    score?: number;
+    additional_metadata?: Record<string, any>;
+    verbose: boolean;
+    constructor(type: string, threshold: number, additional_metadata?: Record<string, any>, verbose?: boolean);
+    /**
+     * Check if the score meets the threshold
+     */
+    successCheck(): boolean;
+    /**
+     * Validate that the threshold is within the allowed range
+     */
+    validateThreshold(): void;
+    /**
+     * Score an example
+     * @param example The example to score
+     * @returns A ScorerData object with the score
+     */
+    abstract scoreExample(example: Example): Promise<ScorerData>;
+    /**
+     * Convert the scorer to a plain object
+     */
+    toJSON(): Record<string, any>;
+}
+/**
+ * Wrapper for scorers to allow dynamic loading of implementations
+ */
+export declare class ScorerWrapper implements Scorer {
+    type: string;
+    scoreType: string;
+    threshold: number;
+    score?: number;
+    additional_metadata?: Record<string, any>;
+    verbose: boolean;
+    scorer: any;
+    constructor(scorer: any);
+    /**
+     * Check if the score meets the threshold
+     */
+    successCheck(): boolean;
+    /**
+     * Load the appropriate implementation based on the use_judgment flag
+     */
+    loadImplementation(useJudgment?: boolean): APIJudgmentScorer | JudgevalScorer;
+    /**
+     * Validate that the threshold is within the allowed range
+     */
+    validateThreshold(): void;
+    /**
+     * Convert the scorer to a plain object
+     */
+    toJSON(): Record<string, any>;
+}

package/dist/scorers/base-scorer.js

@@ -0,0 +1,190 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ScorerWrapper = exports.JudgevalScorer = exports.APIJudgmentScorer = void 0;
+const constants_1 = require("../constants");
+/**
+ * Base class for API judgment scorers
+ */
+class APIJudgmentScorer {
+    get scoreType() { return this.type; } // For backward compatibility
+    constructor(type, threshold, additional_metadata, verbose = false) {
+        this.type = type;
+        this.threshold = threshold;
+        this.additional_metadata = additional_metadata;
+        this.verbose = verbose;
+    }
+    /**
+     * Check if the score meets the threshold
+     */
+    successCheck() {
+        if (this.score === undefined) {
+            return false;
+        }
+        return this.score >= this.threshold;
+    }
+    /**
+     * Validate that the threshold is within the allowed range
+     */
+    validateThreshold() {
+        // Check if this is an unbounded scorer
+        const isUnbounded = Array.from(constants_1.UNBOUNDED_SCORERS).some(scorer => scorer.toLowerCase() === this.type.toLowerCase());
+        if (isUnbounded) {
+            if (this.threshold < 0) {
+                throw new Error(`Threshold for ${this.type} must be greater than or equal to 0, got: ${this.threshold}`);
+            }
+        }
+        else {
+            if (this.threshold < 0 || this.threshold > 1) {
+                throw new Error(`Threshold for ${this.type} must be between 0 and 1, got: ${this.threshold}`);
+            }
+        }
+    }
+    /**
+     * Convert the scorer to a plain object
+     */
+    toJSON() {
+        const result = {
+            score_type: this.type,
+            threshold: this.threshold,
+            score: this.score,
+            additional_metadata: this.additional_metadata,
+            verbose: this.verbose,
+        };
+        return result;
+    }
+    a_score_example(example) {
+        return __awaiter(this, void 0, void 0, function* () {
+            throw new Error('API scorers are evaluated on the server side');
+        });
+    }
+}
+exports.APIJudgmentScorer = APIJudgmentScorer;
+/**
+ * Base class for local judgment scorers
+ */
+class JudgevalScorer {
+    constructor(type, threshold, additional_metadata, verbose = false) {
+        this.type = type;
+        this.scoreType = type; // For backward compatibility
+        this.threshold = threshold;
+        this.additional_metadata = additional_metadata;
+        this.verbose = verbose;
+    }
+    /**
+     * Check if the score meets the threshold
+     */
+    successCheck() {
+        if (this.score === undefined) {
+            return false;
+        }
+        return this.score >= this.threshold;
+    }
+    /**
+     * Validate that the threshold is within the allowed range
+     */
+    validateThreshold() {
+        // Check if this is an unbounded scorer
+        const isUnbounded = Array.from(constants_1.UNBOUNDED_SCORERS).some(scorer => scorer.toLowerCase() === this.type.toLowerCase());
+        if (isUnbounded) {
+            if (this.threshold < 0) {
+                throw new Error(`Threshold for ${this.type} must be greater than or equal to 0, got: ${this.threshold}`);
+            }
+        }
+        else {
+            if (this.threshold < 0 || this.threshold > 1) {
+                throw new Error(`Threshold for ${this.type} must be between 0 and 1, got: ${this.threshold}`);
+            }
+        }
+    }
+    /**
+     * Convert the scorer to a plain object
+     */
+    toJSON() {
+        return {
+            score_type: this.type,
+            threshold: this.threshold,
+            score: this.score,
+            additional_metadata: this.additional_metadata,
+            verbose: this.verbose,
+        };
+    }
+}
+exports.JudgevalScorer = JudgevalScorer;
+/**
+ * Wrapper for scorers to allow dynamic loading of implementations
+ */
+class ScorerWrapper {
+    constructor(scorer) {
+        this.scorer = scorer;
+        this.type = scorer.type;
+        this.scoreType = scorer.scoreType || scorer.score_type; // For backward compatibility
+        this.threshold = scorer.threshold;
+        this.score = scorer.score;
+        this.additional_metadata = scorer.additional_metadata;
+        this.verbose = scorer.verbose;
+    }
+    /**
+     * Check if the score meets the threshold
+     */
+    successCheck() {
+        if (this.score === undefined) {
+            return false;
+        }
+        return this.score >= this.threshold;
+    }
+    /**
+     * Load the appropriate implementation based on the use_judgment flag
+     */
+    loadImplementation(useJudgment = true) {
+        // This would be implemented based on the specific scorer types
+        // For now, we'll just return the scorer as is
+        if (useJudgment) {
+            // Return API implementation
+            return this.scorer;
+        }
+        else {
+            // Return local implementation
+            return this.scorer;
+        }
+    }
+    /**
+     * Validate that the threshold is within the allowed range
+     */
+    validateThreshold() {
+        // Check if this is an unbounded scorer
+        const isUnbounded = Array.from(constants_1.UNBOUNDED_SCORERS).some(scorer => scorer.toLowerCase() === this.type.toLowerCase());
+        if (isUnbounded) {
+            if (this.threshold < 0) {
+                throw new Error(`Threshold for ${this.type} must be greater than or equal to 0, got: ${this.threshold}`);
+            }
+        }
+        else {
+            if (this.threshold < 0 || this.threshold > 1) {
+                throw new Error(`Threshold for ${this.type} must be between 0 and 1, got: ${this.threshold}`);
+            }
+        }
+    }
+    /**
+     * Convert the scorer to a plain object
+     */
+    toJSON() {
+        return {
+            score_type: this.type,
+            threshold: this.threshold,
+            score: this.score,
+            additional_metadata: this.additional_metadata,
+            verbose: this.verbose,
+        };
+    }
+}
+exports.ScorerWrapper = ScorerWrapper;
+//# sourceMappingURL=base-scorer.js.map