judgeval 0.2.9 → 0.3.0

package/dist/scorers/api_scorers/faithfulness-scorer.js

@@ -0,0 +1,20 @@
+import { ExampleParams } from "../../data";
+import { APIScorerType, createAPIScorer } from "../api-scorer";
+const FAITHFULNESS_REQUIRED_PARAMS = [
+    ExampleParams.INPUT,
+    ExampleParams.ACTUAL_OUTPUT,
+    ExampleParams.RETRIEVAL_CONTEXT,
+];
+export function createFaithfulnessScorer(scorerArgs) {
+    const scorer = createAPIScorer(APIScorerType.FAITHFULNESS, FAITHFULNESS_REQUIRED_PARAMS);
+    if (scorerArgs) {
+        if (scorerArgs.threshold !== undefined) {
+            scorer.setThreshold(scorerArgs.threshold);
+        }
+        if (scorerArgs.model) {
+            scorer.addModel(scorerArgs.model);
+        }
+    }
+    return scorer;
+}
+//# sourceMappingURL=faithfulness-scorer.js.map

package/dist/scorers/api_scorers/faithfulness-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"faithfulness-scorer.js","sourceRoot":"","sources":["../../../src/scorers/api_scorers/faithfulness-scorer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAa,aAAa,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAE1E,MAAM,4BAA4B,GAAG;IACnC,aAAa,CAAC,KAAK;IACnB,aAAa,CAAC,aAAa;IAC3B,aAAa,CAAC,iBAAiB;CACvB,CAAC;AAYX,MAAM,UAAU,wBAAwB,CACtC,UAAmC;IAEnC,MAAM,MAAM,GAAG,eAAe,CAC5B,aAAa,CAAC,YAAY,EAC1B,4BAA4B,CAC7B,CAAC;IAEF,IAAI,UAAU,EAAE,CAAC;QACf,IAAI,UAAU,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACvC,MAAM,CAAC,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;YACrB,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/scorers/api_scorers/index.d.ts

@@ -0,0 +1,2 @@
+export { BasePromptScorer, fetchPromptScorer, JudgmentAPIError, PromptScorer, pushPromptScorer, scorerExists, TracePromptScorer, } from "./prompt_scorer";
+//# sourceMappingURL=index.d.ts.map

package/dist/scorers/api_scorers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/scorers/api_scorers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACZ,iBAAiB,GAClB,MAAM,iBAAiB,CAAC"}

package/dist/scorers/api_scorers/index.js

@@ -0,0 +1,2 @@
+export { BasePromptScorer, fetchPromptScorer, JudgmentAPIError, PromptScorer, pushPromptScorer, scorerExists, TracePromptScorer, } from "./prompt_scorer";
+//# sourceMappingURL=index.js.map

package/dist/scorers/api_scorers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scorers/api_scorers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACZ,iBAAiB,GAClB,MAAM,iBAAiB,CAAC"}

package/dist/scorers/api_scorers/instruction-adherence-scorer.d.ts

@@ -0,0 +1,11 @@
+import { ExampleParams } from "../../data";
+import { APIScorer, APIScorerType } from "../api-scorer";
+declare const INSTRUCTION_ADHERENCE_REQUIRED_PARAMS: readonly [ExampleParams.INPUT, ExampleParams.ACTUAL_OUTPUT];
+export type InstructionAdherenceScorer = APIScorer<APIScorerType.INSTRUCTION_ADHERENCE, typeof INSTRUCTION_ADHERENCE_REQUIRED_PARAMS>;
+export type InstructionAdherenceScorerArgs = {
+    threshold?: number;
+    model?: string;
+};
+export declare function createInstructionAdherenceScorer(scorerArgs?: InstructionAdherenceScorerArgs): InstructionAdherenceScorer;
+export {};
+//# sourceMappingURL=instruction-adherence-scorer.d.ts.map

package/dist/scorers/api_scorers/instruction-adherence-scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"instruction-adherence-scorer.d.ts","sourceRoot":"","sources":["../../../src/scorers/api_scorers/instruction-adherence-scorer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,aAAa,EAAmB,MAAM,eAAe,CAAC;AAE1E,QAAA,MAAM,qCAAqC,6DAGjC,CAAC;AAEX,MAAM,MAAM,0BAA0B,GAAG,SAAS,CAChD,aAAa,CAAC,qBAAqB,EACnC,OAAO,qCAAqC,CAC7C,CAAC;AAEF,MAAM,MAAM,8BAA8B,GAAG;IAC3C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,wBAAgB,gCAAgC,CAC9C,UAAU,CAAC,EAAE,8BAA8B,GAC1C,0BAA0B,CAkB5B"}

package/dist/scorers/api_scorers/instruction-adherence-scorer.js

@@ -0,0 +1,20 @@
+import { ExampleParams } from "../../data";
+import { APIScorerType, createAPIScorer } from "../api-scorer";
+const INSTRUCTION_ADHERENCE_REQUIRED_PARAMS = [
+    ExampleParams.INPUT,
+    ExampleParams.ACTUAL_OUTPUT,
+];
+export function createInstructionAdherenceScorer(scorerArgs) {
+    const scorer = createAPIScorer(APIScorerType.INSTRUCTION_ADHERENCE, INSTRUCTION_ADHERENCE_REQUIRED_PARAMS);
+    scorer.name = "Instruction Adherence";
+    if (scorerArgs) {
+        if (scorerArgs.threshold !== undefined) {
+            scorer.setThreshold(scorerArgs.threshold);
+        }
+        if (scorerArgs.model) {
+            scorer.addModel(scorerArgs.model);
+        }
+    }
+    return scorer;
+}
+//# sourceMappingURL=instruction-adherence-scorer.js.map

package/dist/scorers/api_scorers/instruction-adherence-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"instruction-adherence-scorer.js","sourceRoot":"","sources":["../../../src/scorers/api_scorers/instruction-adherence-scorer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAa,aAAa,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAE1E,MAAM,qCAAqC,GAAG;IAC5C,aAAa,CAAC,KAAK;IACnB,aAAa,CAAC,aAAa;CACnB,CAAC;AAYX,MAAM,UAAU,gCAAgC,CAC9C,UAA2C;IAE3C,MAAM,MAAM,GAAG,eAAe,CAC5B,aAAa,CAAC,qBAAqB,EACnC,qCAAqC,CACtC,CAAC;IAEF,MAAM,CAAC,IAAI,GAAG,uBAAuB,CAAC;IAEtC,IAAI,UAAU,EAAE,CAAC;QACf,IAAI,UAAU,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACvC,MAAM,CAAC,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;YACrB,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/scorers/api_scorers/prompt_scorer/base-prompt-scorer.d.ts

@@ -0,0 +1,48 @@
+import { ScorerConfig } from "../../../internal/api/models";
+import { APIScorer, APIScorerType } from "../../api-scorer";
+export declare abstract class BasePromptScorer implements APIScorer<APIScorerType, readonly string[]> {
+    scoreType: APIScorerType;
+    name: string;
+    prompt: string;
+    threshold: number;
+    options?: Record<string, number> | null;
+    judgmentApiKey: string;
+    organizationId: string;
+    requiredParams: readonly string[];
+    score_type: string;
+    class_name: string;
+    model?: string;
+    score?: number;
+    error?: string | null;
+    strict_mode?: boolean;
+    constructor(scoreType: APIScorerType, name: string, prompt: string, threshold: number, requiredParams: readonly string[], options?: Record<string, number> | null, judgmentApiKey?: string, organizationId?: string);
+    score_breakdown?: Record<string, any> | null | undefined;
+    reason?: string | null | undefined;
+    using_native_model?: boolean | null | undefined;
+    success?: boolean | null | undefined;
+    model_client?: any;
+    additional_metadata?: Record<string, any> | null | undefined;
+    user?: string | null | undefined;
+    server_hosted?: boolean | undefined;
+    static get<T extends BasePromptScorer>(this: new (...args: any[]) => T, name: string, judgmentApiKey?: string, organizationId?: string): Promise<T>;
+    static create<T extends BasePromptScorer>(this: new (...args: any[]) => T, name: string, prompt: string, threshold?: number, options?: Record<string, number> | null, judgmentApiKey?: string, organizationId?: string): Promise<T>;
+    updateThreshold(threshold: number): Promise<void>;
+    setPrompt(prompt: string): Promise<void>;
+    setOptions(options: Record<string, number> | null): Promise<void>;
+    appendToPrompt(promptAddition: string): Promise<void>;
+    getThreshold(): number;
+    getPrompt(): string;
+    getOptions(): Record<string, number> | null;
+    getName(): string;
+    getConfig(): Record<string, any>;
+    pushPromptScorer(): Promise<void>;
+    toString(): string;
+    addModel(model: string): void;
+    successCheck(): boolean;
+    getRequiredParams(): string[];
+    setThreshold(threshold: number): void;
+    getScoreType(): APIScorerType;
+    setRequiredParams(params: readonly string[]): void;
+    toTransport(): ScorerConfig;
+}
+//# sourceMappingURL=base-prompt-scorer.d.ts.map

package/dist/scorers/api_scorers/prompt_scorer/base-prompt-scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-prompt-scorer.d.ts","sourceRoot":"","sources":["../../../../src/scorers/api_scorers/prompt_scorer/base-prompt-scorer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAQ5D,8BAAsB,gBACpB,YAAW,SAAS,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAE/C,SAAS,EAAE,aAAa,CAAC;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IACxC,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;gBAG3B,SAAS,EAAE,aAAa,EACxB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,cAAc,EAAE,SAAS,MAAM,EAAE,EACjC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,EACvC,cAAc,GAAE,MAA+B,EAC/C,cAAc,GAAE,MAA8B;IAiBhD,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACzD,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACnC,kBAAkB,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;IAChD,OAAO,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;IACrC,YAAY,CAAC,EAAE,GAAG,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAC7D,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjC,aAAa,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;WAEvB,GAAG,CAAC,CAAC,SAAS,gBAAgB,EACzC,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,EAC/B,IAAI,EAAE,MAAM,EACZ,cAAc,GAAE,MAA+B,EAC/C,cAAc,GAAE,MAA8B,GAC7C,OAAO,CAAC,CAAC,CAAC;WAkCA,MAAM,CAAC,CAAC,SAAS,gBAAgB,EAC5C,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,EAC/B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,SAAS,GAAE,MAAY,EACvB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,EACvC,cAAc,GAAE,MAA+B,EAC/C,cAAc,GAAE,MAA8B,GAC7C,OAAO,CAAC,CAAC,CAAC;IAoCP,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKjD,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAKjE,cAAc,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAK3D,YAAY,IAAI,MAAM;IAItB,SAAS,IAAI,MAAM;IAInB,UAAU,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;IAI3C,OAAO,IAAI,MAAM;IAIjB,SAAS,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;IAS1B,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IAWvC,QAAQ,IAAI,MAAM;IAMlB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAI7B,YAAY,IAAI,OAAO;IAYvB,iBAAiB,IAAI,MAAM,EAAE;IAO7B,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAOrC,YAAY,IAAI,aAAa;IAI7B,iBAAiB,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI;IAIlD,WAAW,IAAI,YAAY;CAa5B"}

package/dist/scorers/api_scorers/prompt_scorer/base-prompt-scorer.js

@@ -0,0 +1,133 @@
+import { JUDGMENT_API_KEY, JUDGMENT_ORG_ID } from "../../../env";
+import { APIScorerType } from "../../api-scorer";
+import { fetchPromptScorer, JudgmentAPIError, pushPromptScorer, scorerExists, } from "./prompt-scorer-utils";
+export class BasePromptScorer {
+    constructor(scoreType, name, prompt, threshold, requiredParams, options, judgmentApiKey = JUDGMENT_API_KEY || "", organizationId = JUDGMENT_ORG_ID || "") {
+        this.scoreType = scoreType;
+        this.name = name;
+        this.prompt = prompt;
+        this.threshold = threshold;
+        this.requiredParams = requiredParams;
+        this.options = options;
+        this.judgmentApiKey = judgmentApiKey;
+        this.organizationId = organizationId;
+        this.score_type = scoreType;
+        this.class_name = "BasePromptScorer";
+        this.model = undefined;
+        this.score = undefined;
+        this.error = null;
+        this.strict_mode = false;
+    }
+    static async get(name, judgmentApiKey = JUDGMENT_API_KEY || "", organizationId = JUDGMENT_ORG_ID || "") {
+        const config = await fetchPromptScorer(name, judgmentApiKey, organizationId);
+        const isTrace = config.is_trace === true;
+        const expectedIsTrace = this.prototype.scoreType === APIScorerType.TRACE_PROMPT_SCORER;
+        if (isTrace !== expectedIsTrace) {
+            throw new JudgmentAPIError(400, `Scorer with name ${name} is not a ${this.name}`);
+        }
+        const scoreType = isTrace
+            ? APIScorerType.TRACE_PROMPT_SCORER
+            : APIScorerType.PROMPT_SCORER;
+        return new this(scoreType, config.name, // Use config.name instead of name parameter
+        config.prompt, config.threshold, [], config.options, judgmentApiKey, organizationId);
+    }
+    static async create(name, prompt, threshold = 0.5, options, judgmentApiKey = JUDGMENT_API_KEY || "", organizationId = JUDGMENT_ORG_ID || "") {
+        if (await scorerExists(name, judgmentApiKey, organizationId)) {
+            throw new JudgmentAPIError(400, `Scorer with name ${name} already exists. Either use the existing scorer with the get() method or use a new name.`);
+        }
+        const isTrace = this.prototype.scoreType === APIScorerType.TRACE_PROMPT_SCORER;
+        const scoreType = isTrace
+            ? APIScorerType.TRACE_PROMPT_SCORER
+            : APIScorerType.PROMPT_SCORER;
+        await pushPromptScorer(name, prompt, threshold, options, judgmentApiKey, organizationId, isTrace);
+        return new this(scoreType, name, prompt, threshold, [], options, judgmentApiKey, organizationId);
+    }
+    async updateThreshold(threshold) {
+        this.setThreshold(threshold);
+        await this.pushPromptScorer();
+    }
+    async setPrompt(prompt) {
+        this.prompt = prompt;
+        await this.pushPromptScorer();
+    }
+    async setOptions(options) {
+        this.options = options;
+        await this.pushPromptScorer();
+    }
+    async appendToPrompt(promptAddition) {
+        this.prompt += promptAddition;
+        await this.pushPromptScorer();
+    }
+    getThreshold() {
+        return this.threshold;
+    }
+    getPrompt() {
+        return this.prompt;
+    }
+    getOptions() {
+        return this.options ? { ...this.options } : null;
+    }
+    getName() {
+        return this.name;
+    }
+    getConfig() {
+        return {
+            name: this.name,
+            prompt: this.prompt,
+            threshold: this.threshold,
+            options: this.options,
+        };
+    }
+    async pushPromptScorer() {
+        await pushPromptScorer(this.name, this.prompt, this.threshold, this.options, this.judgmentApiKey, this.organizationId);
+    }
+    toString() {
+        return `${this.constructor.name}(name=${this.name}, prompt=${this.prompt}, threshold=${this.threshold}, options=${JSON.stringify(this.options)})`;
+    }
+    addModel(model) {
+        this.model = model;
+    }
+    successCheck() {
+        if (this.error != null) {
+            return false;
+        }
+        if (this.score == null) {
+            return false;
+        }
+        const threshold = this.threshold ?? 0.5;
+        const score = this.score;
+        return threshold != null && score != null && score >= threshold;
+    }
+    getRequiredParams() {
+        if (Array.isArray(this.requiredParams)) {
+            return [...this.requiredParams];
+        }
+        return [];
+    }
+    setThreshold(threshold) {
+        if (threshold < 0 || threshold > 1) {
+            throw new Error(`Threshold must be between 0 and 1, got: ${threshold}`);
+        }
+        this.threshold = threshold;
+    }
+    getScoreType() {
+        return this.scoreType;
+    }
+    setRequiredParams(params) {
+        this.requiredParams = params;
+    }
+    toTransport() {
+        return {
+            score_type: this.getScoreType(),
+            name: this.getName(),
+            threshold: this.getThreshold(),
+            strict_mode: this.strict_mode ?? false,
+            required_params: this.getRequiredParams(),
+            kwargs: {
+                prompt: this.getPrompt(),
+                ...(this.getOptions() ? { options: this.getOptions() } : {}),
+            },
+        };
+    }
+}
+//# sourceMappingURL=base-prompt-scorer.js.map

package/dist/scorers/api_scorers/prompt_scorer/base-prompt-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-prompt-scorer.js","sourceRoot":"","sources":["../../../../src/scorers/api_scorers/prompt_scorer/base-prompt-scorer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAEjE,OAAO,EAAa,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAC5D,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,GACb,MAAM,uBAAuB,CAAC;AAE/B,MAAM,OAAgB,gBAAgB;IAkBpC,YACE,SAAwB,EACxB,IAAY,EACZ,MAAc,EACd,SAAiB,EACjB,cAAiC,EACjC,OAAuC,EACvC,iBAAyB,gBAAgB,IAAI,EAAE,EAC/C,iBAAyB,eAAe,IAAI,EAAE;QAE9C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,kBAAkB,CAAC;QACrC,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC3B,CAAC;IAUD,MAAM,CAAC,KAAK,CAAC,GAAG,CAEd,IAAY,EACZ,iBAAyB,gBAAgB,IAAI,EAAE,EAC/C,iBAAyB,eAAe,IAAI,EAAE;QAE9C,MAAM,MAAM,GAAG,MAAM,iBAAiB,CACpC,IAAI,EACJ,cAAc,EACd,cAAc,CACf,CAAC;QAEF,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,KAAK,IAAI,CAAC;QACzC,MAAM,eAAe,GACnB,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,aAAa,CAAC,mBAAmB,CAAC;QAEjE,IAAI,OAAO,KAAK,eAAe,EAAE,CAAC;YAChC,MAAM,IAAI,gBAAgB,CACxB,GAAG,EACH,oBAAoB,IAAI,aAAa,IAAI,CAAC,IAAI,EAAE,CACjD,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,OAAO;YACvB,CAAC,CAAC,aAAa,CAAC,mBAAmB;YACnC,CAAC,CAAC,aAAa,CAAC,aAAa,CAAC;QAEhC,OAAO,IAAI,IAAI,CACb,SAAS,EACT,MAAM,CAAC,IAAI,EAAE,4CAA4C;QACzD,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,SAAS,EAChB,EAAE,EACF,MAAM,CAAC,OAAO,EACd,cAAc,EACd,cAAc,CACf,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,MAAM,CAEjB,IAAY,EACZ,MAAc,EACd,YAAoB,GAAG,EACvB,OAAuC,EACvC,iBAAyB,gBAAgB,IAAI,EAAE,EAC/C,iBAAyB,eAAe,IAAI,EAAE;QAE9C,IAAI,MAAM,YAAY,CAAC,IAAI,EAAE,cAAc,EAAE,cAAc,CAAC,EAAE,CAAC;YAC7D,MAAM,IAAI,gBAAgB,CACxB,GAAG,EACH,oBAAoB,IAAI,0FAA0F,CACnH,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GACX,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,aAAa,CAAC,mBAAmB,CAAC;QACjE,MAAM,SAAS,GAAG,OAAO;YACvB,CAAC,CAAC,aAAa,CAAC,mBAAmB;YACnC,CAAC,CAAC,aAAa,CAAC,aAAa,CAAC;QAEhC,MAAM,gBAAgB,CACpB,IAAI,EACJ,MAAM,EACN,SAAS,EACT,OAAO,EACP,cAAc,EACd,cAAc,EACd,OAAO,CACR,CAAC;QAEF,OAAO,IAAI,IAAI,CACb,SAAS,EACT,IAAI,EACJ,MAAM,EACN,SAAS,EACT,EAAE,EACF,OAAO,EACP,cAAc,EACd,cAAc,CACf,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,SAAiB;QACrC,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAC7B,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAc;QAC5B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAAsC;QACrD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,cAAsB;QACzC,IAAI,CAAC,MAAM,IAAI,cAAc,CAAC;QAC9B,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAChC,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,SAAS;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACnD,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,SAAS;QACP,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,gBAAgB;QACpB,MAAM,gBAAgB,CACpB,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,cAAc,CACpB,CAAC;IACJ,CAAC;IAED,QAAQ;QACN,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,SAAS,IAAI,CAAC,IAAI,YAC/C,IAAI,CAAC,MACP,eAAe,IAAI,CAAC,SAAS,aAAa,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;IAC5E,CAAC;IAED,QAAQ,CAAC,KAAa;QACpB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,YAAY;QACV,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;YACvB,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;YACvB,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,OAAO,SAAS,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,SAAS,CAAC;IAClE,CAAC;IAED,iBAAiB;QACf,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;YACvC,OAAO,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC;QAClC,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,YAAY,CAAC,SAAiB;QAC5B,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,2CAA2C,SAAS,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,iBAAiB,CAAC,MAAyB;QACzC,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC;IAC/B,CAAC;IAED,WAAW;QACT,OAAO;YACL,UAAU,EAAE,IAAI,CAAC,YAAY,EAAE;YAC/B,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE;YACpB,SAAS,EAAE,IAAI,CAAC,YAAY,EAAE;YAC9B,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,KAAK;YACtC,eAAe,EAAE,IAAI,CAAC,iBAAiB,EAAE;YACzC,MAAM,EAAE;gBACN,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE;gBACxB,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC7D;SACF,CAAC;IACJ,CAAC;CACF"}

package/dist/scorers/api_scorers/prompt_scorer/index.d.ts

@@ -0,0 +1,5 @@
+export * from "./base-prompt-scorer";
+export * from "./prompt-scorer";
+export * from "./prompt-scorer-utils";
+export * from "./trace-prompt-scorer";
+//# sourceMappingURL=index.d.ts.map

package/dist/scorers/api_scorers/prompt_scorer/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scorers/api_scorers/prompt_scorer/index.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAC;AACrC,cAAc,iBAAiB,CAAC;AAChC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC"}

package/dist/scorers/api_scorers/prompt_scorer/index.js

@@ -0,0 +1,5 @@
+export * from "./base-prompt-scorer";
+export * from "./prompt-scorer";
+export * from "./prompt-scorer-utils";
+export * from "./trace-prompt-scorer";
+//# sourceMappingURL=index.js.map

package/dist/scorers/api_scorers/prompt_scorer/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/scorers/api_scorers/prompt_scorer/index.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAC;AACrC,cAAc,iBAAiB,CAAC;AAChC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC"}

package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer-utils.d.ts

@@ -0,0 +1,14 @@
+export declare class JudgmentAPIError extends Error {
+    statusCode: number;
+    constructor(statusCode: number, message: string);
+}
+export declare function pushPromptScorer(name: string, prompt: string, threshold: number, options?: Record<string, number> | null, judgmentApiKey?: string, organizationId?: string, isTrace?: boolean): Promise<string>;
+export declare function fetchPromptScorer(name: string, judgmentApiKey?: string, organizationId?: string): Promise<{
+    name: string;
+    prompt: string;
+    threshold: number;
+    options?: Record<string, number> | null;
+    is_trace?: boolean | null;
+}>;
+export declare function scorerExists(name: string, judgmentApiKey?: string, organizationId?: string): Promise<boolean>;
+//# sourceMappingURL=prompt-scorer-utils.d.ts.map

package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt-scorer-utils.d.ts","sourceRoot":"","sources":["../../../../src/scorers/api_scorers/prompt_scorer/prompt-scorer-utils.ts"],"names":[],"mappings":"AAOA,qBAAa,gBAAiB,SAAQ,KAAK;IAEhC,UAAU,EAAE,MAAM;gBAAlB,UAAU,EAAE,MAAM,EACzB,OAAO,EAAE,MAAM;CAKlB;AAED,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,EACvC,cAAc,GAAE,MAA+B,EAC/C,cAAc,GAAE,MAA8B,EAC9C,OAAO,CAAC,EAAE,OAAO,GAChB,OAAO,CAAC,MAAM,CAAC,CAkBjB;AAED,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,MAAM,EACZ,cAAc,GAAE,MAA+B,EAC/C,cAAc,GAAE,MAA8B;;;;;;GAiB/C;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EACZ,cAAc,GAAE,MAA+B,EAC/C,cAAc,GAAE,MAA8B,GAC7C,OAAO,CAAC,OAAO,CAAC,CAYlB"}

package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer-utils.js

@@ -0,0 +1,44 @@
+import { JUDGMENT_API_KEY, JUDGMENT_API_URL, JUDGMENT_ORG_ID, } from "../../../env";
+import { JudgmentApiClient } from "../../../internal/api";
+export class JudgmentAPIError extends Error {
+    constructor(statusCode, message) {
+        super(message);
+        this.statusCode = statusCode;
+        this.name = "JudgmentAPIError";
+    }
+}
+export async function pushPromptScorer(name, prompt, threshold, options, judgmentApiKey = JUDGMENT_API_KEY || "", organizationId = JUDGMENT_ORG_ID || "", isTrace) {
+    if (!JUDGMENT_API_URL || !judgmentApiKey || !organizationId) {
+        throw new Error("Missing required API credentials");
+    }
+    const client = new JudgmentApiClient(JUDGMENT_API_URL, judgmentApiKey, organizationId);
+    const response = await client.saveScorer({
+        name,
+        prompt,
+        threshold,
+        options,
+        is_trace: isTrace,
+    });
+    return response.name;
+}
+export async function fetchPromptScorer(name, judgmentApiKey = JUDGMENT_API_KEY || "", organizationId = JUDGMENT_ORG_ID || "") {
+    if (!JUDGMENT_API_URL || !judgmentApiKey || !organizationId) {
+        throw new Error("Missing required API credentials");
+    }
+    const client = new JudgmentApiClient(JUDGMENT_API_URL, judgmentApiKey, organizationId);
+    const response = await client.fetchScorers({ names: [name] });
+    if (response.scorers.length === 0) {
+        throw new JudgmentAPIError(404, `Scorer with name ${name} not found`);
+    }
+    const { created_at, updated_at, ...config } = response.scorers[0];
+    return config;
+}
+export async function scorerExists(name, judgmentApiKey = JUDGMENT_API_KEY || "", organizationId = JUDGMENT_ORG_ID || "") {
+    if (!JUDGMENT_API_URL || !judgmentApiKey || !organizationId) {
+        throw new Error("Missing required API credentials");
+    }
+    const client = new JudgmentApiClient(JUDGMENT_API_URL, judgmentApiKey, organizationId);
+    const response = await client.scorerExists({ name });
+    return response.exists;
+}
+//# sourceMappingURL=prompt-scorer-utils.js.map

package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt-scorer-utils.js","sourceRoot":"","sources":["../../../../src/scorers/api_scorers/prompt_scorer/prompt-scorer-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,GAChB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAE1D,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACzC,YACS,UAAkB,EACzB,OAAe;QAEf,KAAK,CAAC,OAAO,CAAC,CAAC;QAHR,eAAU,GAAV,UAAU,CAAQ;QAIzB,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,MAAc,EACd,SAAiB,EACjB,OAAuC,EACvC,iBAAyB,gBAAgB,IAAI,EAAE,EAC/C,iBAAyB,eAAe,IAAI,EAAE,EAC9C,OAAiB;IAEjB,IAAI,CAAC,gBAAgB,IAAI,CAAC,cAAc,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,iBAAiB,CAClC,gBAAgB,EAChB,cAAc,EACd,cAAc,CACf,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC;QACvC,IAAI;QACJ,MAAM;QACN,SAAS;QACT,OAAO;QACP,QAAQ,EAAE,OAAO;KAClB,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,IAAI,CAAC;AACvB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,iBAAyB,gBAAgB,IAAI,EAAE,EAC/C,iBAAyB,eAAe,IAAI,EAAE;IAE9C,IAAI,CAAC,gBAAgB,IAAI,CAAC,cAAc,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,iBAAiB,CAClC,gBAAgB,EAChB,cAAc,EACd,cAAc,CACf,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC9D,IAAI,QAAQ,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,gBAAgB,CAAC,GAAG,EAAE,oBAAoB,IAAI,YAAY,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,MAAM,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAClE,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAY,EACZ,iBAAyB,gBAAgB,IAAI,EAAE,EAC/C,iBAAyB,eAAe,IAAI,EAAE;IAE9C,IAAI,CAAC,gBAAgB,IAAI,CAAC,cAAc,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,iBAAiB,CAClC,gBAAgB,EAChB,cAAc,EACd,cAAc,CACf,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;IACrD,OAAO,QAAQ,CAAC,MAAM,CAAC;AACzB,CAAC"}

package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer.d.ts

@@ -0,0 +1,7 @@
+import { APIScorerType } from "../../api-scorer";
+import { BasePromptScorer } from "./base-prompt-scorer";
+export { fetchPromptScorer, JudgmentAPIError, pushPromptScorer, scorerExists, } from "./prompt-scorer-utils";
+export declare class PromptScorer extends BasePromptScorer {
+    constructor(scoreType: APIScorerType, name: string, prompt: string, threshold: number, requiredParams?: readonly string[], options?: Record<string, number> | null, judgmentApiKey?: string, organizationId?: string);
+}
+//# sourceMappingURL=prompt-scorer.d.ts.map

package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt-scorer.d.ts","sourceRoot":"","sources":["../../../../src/scorers/api_scorers/prompt_scorer/prompt-scorer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,GACb,MAAM,uBAAuB,CAAC;AAE/B,qBAAa,YAAa,SAAQ,gBAAgB;gBAE9C,SAAS,EAAE,aAAa,EACxB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,cAAc,GAAE,SAAS,MAAM,EAAO,EACtC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,EACvC,cAAc,GAAE,MAA+B,EAC/C,cAAc,GAAE,MAA8B;CAajD"}

package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer.js

@@ -0,0 +1,9 @@
+import { JUDGMENT_API_KEY, JUDGMENT_ORG_ID } from "../../../env";
+import { BasePromptScorer } from "./base-prompt-scorer";
+export { fetchPromptScorer, JudgmentAPIError, pushPromptScorer, scorerExists, } from "./prompt-scorer-utils";
+export class PromptScorer extends BasePromptScorer {
+    constructor(scoreType, name, prompt, threshold, requiredParams = [], options, judgmentApiKey = JUDGMENT_API_KEY || "", organizationId = JUDGMENT_ORG_ID || "") {
+        super(scoreType, name, prompt, threshold, requiredParams, options, judgmentApiKey, organizationId);
+    }
+}
+//# sourceMappingURL=prompt-scorer.js.map

package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt-scorer.js","sourceRoot":"","sources":["../../../../src/scorers/api_scorers/prompt_scorer/prompt-scorer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAEjE,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,GACb,MAAM,uBAAuB,CAAC;AAE/B,MAAM,OAAO,YAAa,SAAQ,gBAAgB;IAChD,YACE,SAAwB,EACxB,IAAY,EACZ,MAAc,EACd,SAAiB,EACjB,iBAAoC,EAAE,EACtC,OAAuC,EACvC,iBAAyB,gBAAgB,IAAI,EAAE,EAC/C,iBAAyB,eAAe,IAAI,EAAE;QAE9C,KAAK,CACH,SAAS,EACT,IAAI,EACJ,MAAM,EACN,SAAS,EACT,cAAc,EACd,OAAO,EACP,cAAc,EACd,cAAc,CACf,CAAC;IACJ,CAAC;CACF"}

package/dist/scorers/api_scorers/prompt_scorer/trace-prompt-scorer.d.ts

@@ -0,0 +1,6 @@
+import { APIScorerType } from "../../api-scorer";
+import { BasePromptScorer } from "./base-prompt-scorer";
+export declare class TracePromptScorer extends BasePromptScorer {
+    constructor(scoreType: APIScorerType, name: string, prompt: string, threshold: number, requiredParams?: readonly string[], options?: Record<string, number> | null, judgmentApiKey?: string, organizationId?: string);
+}
+//# sourceMappingURL=trace-prompt-scorer.d.ts.map

package/dist/scorers/api_scorers/prompt_scorer/trace-prompt-scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"trace-prompt-scorer.d.ts","sourceRoot":"","sources":["../../../../src/scorers/api_scorers/prompt_scorer/trace-prompt-scorer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAExD,qBAAa,iBAAkB,SAAQ,gBAAgB;gBAEnD,SAAS,EAAE,aAAa,EACxB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,cAAc,GAAE,SAAS,MAAM,EAAO,EACtC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,EACvC,cAAc,GAAE,MAA+B,EAC/C,cAAc,GAAE,MAA8B;CAajD"}

package/dist/scorers/api_scorers/prompt_scorer/trace-prompt-scorer.js

@@ -0,0 +1,8 @@
+import { JUDGMENT_API_KEY, JUDGMENT_ORG_ID } from "../../../env";
+import { BasePromptScorer } from "./base-prompt-scorer";
+export class TracePromptScorer extends BasePromptScorer {
+    constructor(scoreType, name, prompt, threshold, requiredParams = [], options, judgmentApiKey = JUDGMENT_API_KEY || "", organizationId = JUDGMENT_ORG_ID || "") {
+        super(scoreType, name, prompt, threshold, requiredParams, options, judgmentApiKey, organizationId);
+    }
+}
+//# sourceMappingURL=trace-prompt-scorer.js.map

package/dist/scorers/api_scorers/prompt_scorer/trace-prompt-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"trace-prompt-scorer.js","sourceRoot":"","sources":["../../../../src/scorers/api_scorers/prompt_scorer/trace-prompt-scorer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAEjE,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAExD,MAAM,OAAO,iBAAkB,SAAQ,gBAAgB;IACrD,YACE,SAAwB,EACxB,IAAY,EACZ,MAAc,EACd,SAAiB,EACjB,iBAAoC,EAAE,EACtC,OAAuC,EACvC,iBAAyB,gBAAgB,IAAI,EAAE,EAC/C,iBAAyB,eAAe,IAAI,EAAE;QAE9C,KAAK,CACH,SAAS,EACT,IAAI,EACJ,MAAM,EACN,SAAS,EACT,cAAc,EACd,OAAO,EACP,cAAc,EACd,cAAc,CACf,CAAC;IACJ,CAAC;CACF"}

package/dist/scorers/base-scorer.d.ts

@@ -0,0 +1,10 @@
+import { ScorerConfig } from "../internal/api/models";
+import { BaseScorer as BaseScorerModel } from "../internal/api/models/BaseScorer";
+export type BaseScorer = BaseScorerModel & {
+    addModel: (model: string) => void;
+    successCheck: () => boolean;
+    getRequiredParams: () => string[];
+    toTransport: () => ScorerConfig;
+};
+export declare function createBaseScorer(): BaseScorer;
+//# sourceMappingURL=base-scorer.d.ts.map

package/dist/scorers/base-scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-scorer.d.ts","sourceRoot":"","sources":["../../src/scorers/base-scorer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,UAAU,IAAI,eAAe,EAAE,MAAM,mCAAmC,CAAC;AAElF,MAAM,MAAM,UAAU,GAAG,eAAe,GAAG;IACzC,QAAQ,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC,YAAY,EAAE,MAAM,OAAO,CAAC;IAC5B,iBAAiB,EAAE,MAAM,MAAM,EAAE,CAAC;IAClC,WAAW,EAAE,MAAM,YAAY,CAAC;CACjC,CAAC;AAEF,wBAAgB,gBAAgB,IAAI,UAAU,CAyC7C"}

package/dist/scorers/base-scorer.js

@@ -0,0 +1,41 @@
+export function createBaseScorer() {
+    const scorer = {
+        score_type: "",
+        class_name: "",
+        name: "",
+        addModel: (model) => {
+            scorer.model = model;
+        },
+        successCheck: () => {
+            if (scorer.error != null) {
+                return false;
+            }
+            if (scorer.score == null) {
+                return false;
+            }
+            const threshold = scorer.threshold ?? 0.5;
+            const score = scorer.score;
+            return threshold != null && score != null && score >= threshold;
+        },
+        getRequiredParams: () => {
+            return [];
+        },
+        toTransport: () => {
+            return {
+                score_type: scorer.score_type,
+                name: scorer.name,
+                threshold: scorer.threshold ?? 0.5,
+                strict_mode: scorer.strict_mode ?? false,
+                required_params: [],
+                kwargs: {},
+            };
+        },
+    };
+    scorer.class_name = "BaseScorer";
+    scorer.name = scorer.class_name;
+    if (scorer.strict_mode === true) {
+        scorer.threshold = 1.0;
+    }
+    return scorer;
+}
+//# sourceMappingURL=base-scorer.js.map

package/dist/scorers/base-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-scorer.js","sourceRoot":"","sources":["../../src/scorers/base-scorer.ts"],"names":[],"mappings":"AAUA,MAAM,UAAU,gBAAgB;IAC9B,MAAM,MAAM,GAAe;QACzB,UAAU,EAAE,EAAE;QACd,UAAU,EAAE,EAAE;QACd,IAAI,EAAE,EAAE;QACR,QAAQ,EAAE,CAAC,KAAa,EAAE,EAAE;YAC1B,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QACvB,CAAC;QACD,YAAY,EAAE,GAAG,EAAE;YACjB,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;gBACzB,OAAO,KAAK,CAAC;YACf,CAAC;YACD,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;gBACzB,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,GAAG,CAAC;YAC1C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;YAC3B,OAAO,SAAS,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,SAAS,CAAC;QAClE,CAAC;QACD,iBAAiB,EAAE,GAAG,EAAE;YACtB,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,WAAW,EAAE,GAAG,EAAE;YAChB,OAAO;gBACL,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,GAAG;gBAClC,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,KAAK;gBACxC,eAAe,EAAE,EAAE;gBACnB,MAAM,EAAE,EAAE;aACX,CAAC;QACJ,CAAC;KACF,CAAC;IAEF,MAAM,CAAC,UAAU,GAAG,YAAY,CAAC;IACjC,MAAM,CAAC,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC;IAChC,IAAI,MAAM,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;QAChC,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC;IACzB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/scorers/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./api-scorer";
+export * from "./api_scorers";
+export * from "./base-scorer";
+//# sourceMappingURL=index.d.ts.map

package/dist/scorers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/scorers/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC"}

package/dist/scorers/index.js

@@ -0,0 +1,4 @@
+export * from "./api-scorer";
+export * from "./api_scorers";
+export * from "./base-scorer";
+//# sourceMappingURL=index.js.map

package/dist/scorers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/scorers/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC"}

package/dist/tracer/OpenTelemetryKeys.d.ts

@@ -0,0 +1,12 @@
+export declare class OpenTelemetryKeys {
+    static readonly AttributeKeys: {
+        readonly JUDGMENT_SPAN_KIND: "judgment.span_kind";
+        readonly JUDGMENT_INPUT: "judgment.input";
+        readonly JUDGMENT_OUTPUT: "judgment.output";
+        readonly PENDING_TRACE_EVAL: "judgment.pending_trace_eval";
+    };
+    static readonly ResourceKeys: {
+        readonly JUDGMENT_PROJECT_ID: "judgment.project_id";
+    };
+}
+//# sourceMappingURL=OpenTelemetryKeys.d.ts.map

package/dist/tracer/OpenTelemetryKeys.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"OpenTelemetryKeys.d.ts","sourceRoot":"","sources":["../../src/tracer/OpenTelemetryKeys.ts"],"names":[],"mappings":"AAAA,qBAAa,iBAAiB;IAC5B,gBAAuB,aAAa;;;;;MAKzB;IAEX,gBAAuB,YAAY;;MAExB;CACZ"}

package/dist/tracer/OpenTelemetryKeys.js

@@ -0,0 +1,12 @@
+export class OpenTelemetryKeys {
+}
+OpenTelemetryKeys.AttributeKeys = {
+    JUDGMENT_SPAN_KIND: "judgment.span_kind",
+    JUDGMENT_INPUT: "judgment.input",
+    JUDGMENT_OUTPUT: "judgment.output",
+    PENDING_TRACE_EVAL: "judgment.pending_trace_eval",
+};
+OpenTelemetryKeys.ResourceKeys = {
+    JUDGMENT_PROJECT_ID: "judgment.project_id",
+};
+//# sourceMappingURL=OpenTelemetryKeys.js.map

package/dist/tracer/OpenTelemetryKeys.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"OpenTelemetryKeys.js","sourceRoot":"","sources":["../../src/tracer/OpenTelemetryKeys.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,iBAAiB;;AACL,+BAAa,GAAG;IACrC,kBAAkB,EAAE,oBAAoB;IACxC,cAAc,EAAE,gBAAgB;IAChC,eAAe,EAAE,iBAAiB;IAClC,kBAAkB,EAAE,6BAA6B;CACzC,CAAC;AAEY,8BAAY,GAAG;IACpC,mBAAmB,EAAE,qBAAqB;CAClC,CAAC"}

package/dist/tracer/Tracer.d.ts

@@ -0,0 +1,47 @@
+import { JudgmentApiClient } from "../internal/api";
+import { Example as ExampleModel } from "../internal/api/models";
+import { BaseScorer } from "../scorers/base-scorer";
+import { JudgmentSpanExporter, NoOpSpanExporter } from "./exporters";
+import { TracerConfiguration } from "./TracerConfiguration";
+export type Serializer = (obj: unknown) => string;
+export declare class Tracer {
+    private readonly configuration;
+    private readonly apiClient;
+    private readonly serializer;
+    private projectId;
+    private projectIdPromise;
+    getConfiguration(): TracerConfiguration;
+    getProjectId(): string | null;
+    getSerializer(): Serializer;
+    constructor(configuration: TracerConfiguration, apiClient: JudgmentApiClient, serializer: Serializer);
+    private resolveProjectId;
+    static builder(): TracerBuilder;
+    static createDefault(projectName: string): Tracer;
+    static createWithConfiguration(configuration: TracerConfiguration): Tracer;
+    getSpanExporter(): Promise<JudgmentSpanExporter | NoOpSpanExporter>;
+    setSpanKind(kind: string | null): void;
+    setAttribute(key: string, value: unknown, type?: any): void;
+    setLLMSpan(): void;
+    setToolSpan(): void;
+    setGeneralSpan(): void;
+    setAttributes(attributes: Record<string, unknown>): void;
+    setInput(input: unknown, type?: any): void;
+    setOutput(output: unknown, type?: any): void;
+    asyncEvaluate(scorer: BaseScorer, example: ExampleModel, model?: string): void;
+    asyncTraceEvaluate(scorer: BaseScorer, model?: string): void;
+    private createTraceEvaluationRun;
+    private createJudgmentSpanExporter;
+    private createEvaluationRun;
+    private enqueueEvaluation;
+}
+export declare class TracerBuilder {
+    private config?;
+    private _apiClient?;
+    private _serializer;
+    static builder(): TracerBuilder;
+    configuration(configuration: TracerConfiguration): this;
+    apiClient(apiClient: JudgmentApiClient): this;
+    serializer(serializer: Serializer): this;
+    build(): Tracer;
+}
+//# sourceMappingURL=Tracer.d.ts.map

package/dist/tracer/Tracer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Tracer.d.ts","sourceRoot":"","sources":["../../src/tracer/Tracer.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAEL,OAAO,IAAI,YAAY,EAExB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAErE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,OAAO,KAAK,MAAM,CAAC;AAElD,qBAAa,MAAM;IACjB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAsB;IACpD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAoB;IAC9C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IACxC,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,gBAAgB,CAAyB;IAE1C,gBAAgB,IAAI,mBAAmB;IAIvC,YAAY,IAAI,MAAM,GAAG,IAAI;IAI7B,aAAa,IAAI,UAAU;gBAKhC,aAAa,EAAE,mBAAmB,EAClC,SAAS,EAAE,iBAAiB,EAC5B,UAAU,EAAE,UAAU;YAQV,gBAAgB;WA4BhB,OAAO,IAAI,aAAa;WAIxB,aAAa,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM;WAM1C,uBAAuB,CACnC,aAAa,EAAE,mBAAmB,GACjC,MAAM;IAII,eAAe,IAAI,OAAO,CACrC,oBAAoB,GAAG,gBAAgB,CACxC;IAeM,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI;IActC,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,GAAG,GAAG,IAAI;IAS3D,UAAU,IAAI,IAAI;IAIlB,WAAW,IAAI,IAAI;IAInB,cAAc,IAAI,IAAI;IAItB,aAAa,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAcxD,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,GAAG,GAAG,IAAI;IAI1C,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,GAAG,GAAG,IAAI;IAI5C,aAAa,CAClB,MAAM,EAAE,UAAU,EAClB,OAAO,EAAE,YAAY,EACrB,KAAK,CAAC,EAAE,MAAM,GACb,IAAI;IAiCA,kBAAkB,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI;IA0CnE,OAAO,CAAC,wBAAwB;IAsBhC,OAAO,CAAC,0BAA0B;IAalC,OAAO,CAAC,mBAAmB;YA0Bb,iBAAiB;CAYhC;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAC,CAAsB;IACrC,OAAO,CAAC,UAAU,CAAC,CAAoB;IACvC,OAAO,CAAC,WAAW,CAA8B;WAEnC,OAAO,IAAI,aAAa;IAI/B,aAAa,CAAC,aAAa,EAAE,mBAAmB,GAAG,IAAI;IAKvD,SAAS,CAAC,SAAS,EAAE,iBAAiB,GAAG,IAAI;IAK7C,UAAU,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI;IAKxC,KAAK,IAAI,MAAM;CAevB"}