judgeval 0.2.4 → 0.2.5

package/dist/cjs/scorers/metrics/hallucination/prompts.js

@@ -0,0 +1,106 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HallucinationTemplate = exports.ReasonSchema = exports.VerdictsSchema = exports.HallucinationVerdictSchema = void 0;
+const zod_1 = require("zod");
+/**
+ * Schema for hallucination verdict
+ */
+exports.HallucinationVerdictSchema = zod_1.z.object({
+    verdict: zod_1.z.string(),
+    reason: zod_1.z.string()
+});
+/**
+ * Schema for verdicts
+ */
+exports.VerdictsSchema = zod_1.z.object({
+    verdicts: zod_1.z.array(exports.HallucinationVerdictSchema)
+});
+/**
+ * Schema for reason
+ */
+exports.ReasonSchema = zod_1.z.object({
+    reason: zod_1.z.string()
+});
+/**
+ * Templates for hallucination scorer prompts
+ */
+class HallucinationTemplate {
+    /**
+     * Generate a prompt to evaluate hallucinations in the actual output
+     */
+    static generateVerdicts(actualOutput, contexts) {
+        return `==== TASK INSTRUCTIONS ====
+You will be provided with an \`actual output\` (the response of an LLM to a particular query) and \`contexts\` (ground truth contextual information from a knowledge base).
+Your task is to take each context in contexts and determine whether the \`actual output\` factually agrees with the context.
+
+Additional notes:
+You should NOT use any prior knowledge you have in your decision making process; take each context at face value. 
+Since you will determine a verdict for EACH context, the number of 'verdicts' is EXACTLY EQUAL TO the number of contexts. 
+You should be lenient in your judgment when the actual output lacks detail with respect to the context segment; you should ONLY provide a 'no' answer if the context contradicts the actual output.
+
+==== FORMATTING INSTRUCTIONS ====
+You should return a JSON object with a key 'verdicts', which is a list of JSON objects. Each JSON object corresponds to a context in \`contexts\`, and should have 2 fields: 'verdict' and 'reason'. 
+The 'verdict' key should be EXACTLY one of 'yes' or 'no', representing whether the \`actual output\` factually agrees with the context segment. 
+The 'reason' is the justification for the verdict. If your verdict is 'no', try to provide a correction in the reason. 
+
+==== EXAMPLE ====
+Example contexts: ["Einstein won the Nobel Prize for his discovery of the photoelectric effect.", "Einstein won the Nobel Prize in 1968."]
+Example actual output: "Einstein won the Nobel Prize in 1969 for his discovery of the photoelectric effect."
+
+Example:
+{
+    "verdicts": [
+        {
+            "verdict": "yes",
+            "reason": "The actual output agrees with the provided context which states that Einstein won the Nobel Prize for his discovery of the photoelectric effect."
+        },
+        {
+            "verdict": "no",
+            "reason": "The actual output contradicts the provided context which states that Einstein won the Nobel Prize in 1968, not 1969."
+        }
+    ]  
+}
+
+==== YOUR TURN ====
+Contexts:
+${contexts.map((context, index) => `${index + 1}. ${context}`).join('\n')}
+
+Actual Output:
+${actualOutput}
+
+JSON:`;
+    }
+    /**
+     * Generate a prompt to create a reason for the hallucination score
+     */
+    static generateReason(actualOutput, contexts) {
+        return `==== TASK INSTRUCTIONS ====
+You will be provided with an \`actual output\` (the response of an LLM to a particular query) and \`contexts\` (ground truth contextual information from a knowledge base).
+Your task is to analyze whether the actual output contains any hallucinations (factual inaccuracies) when compared to the provided contexts.
+
+Please provide a clear and concise reason summarizing your analysis. Focus on any contradictions between the actual output and the contexts, or note if the output is factually consistent with the contexts.
+
+==== FORMATTING INSTRUCTIONS ====
+Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+Example JSON:
+{
+    "reason": "The output contains factual inaccuracies because..."
+}
+
+Or if no hallucinations:
+{
+    "reason": "The output is factually consistent with the provided contexts."
+}
+
+==== YOUR TURN ====
+Contexts:
+${contexts.map((context, index) => `${index + 1}. ${context}`).join('\n')}
+
+Actual Output:
+${actualOutput}
+
+JSON:`;
+    }
+}
+exports.HallucinationTemplate = HallucinationTemplate;
+//# sourceMappingURL=prompts.js.map

package/dist/cjs/scorers/metrics/hallucination/prompts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompts.js","sourceRoot":"","sources":["../../../../../src/scorers/metrics/hallucination/prompts.ts"],"names":[],"mappings":";;;AAAA,6BAAwB;AAExB;;GAEG;AACU,QAAA,0BAA0B,GAAG,OAAC,CAAC,MAAM,CAAC;IACjD,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE;IACnB,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE;CACnB,CAAC,CAAC;AAIH;;GAEG;AACU,QAAA,cAAc,GAAG,OAAC,CAAC,MAAM,CAAC;IACrC,QAAQ,EAAE,OAAC,CAAC,KAAK,CAAC,kCAA0B,CAAC;CAC9C,CAAC,CAAC;AAEH;;GAEG;AACU,QAAA,YAAY,GAAG,OAAC,CAAC,MAAM,CAAC;IACnC,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE;CACnB,CAAC,CAAC;AAEH;;GAEG;AACH,MAAa,qBAAqB;IAChC;;OAEG;IACH,MAAM,CAAC,gBAAgB,CAAC,YAAoB,EAAE,QAAkB;QAC9D,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAkCT,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,GAAG,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;;;EAGvE,YAAY;;MAER,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,cAAc,CAAC,YAAoB,EAAE,QAAkB;QAC5D,OAAO;;;;;;;;;;;;;;;;;;;;EAoBT,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,GAAG,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;;;EAGvE,YAAY;;MAER,CAAC;IACL,CAAC;CACF;AA9ED,sDA8EC"}

package/dist/cjs/scorers/metrics/instruction-adherence/index.js

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./instruction-adherence.js"), exports);
+__exportStar(require("./prompts.js"), exports);
+//# sourceMappingURL=index.js.map

package/dist/cjs/scorers/metrics/instruction-adherence/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/scorers/metrics/instruction-adherence/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6DAA2C;AAC3C,+CAA6B"}

package/dist/cjs/scorers/metrics/instruction-adherence/instruction-adherence.js

@@ -0,0 +1,382 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InstructionAdherenceScorer = void 0;
+const base_scorer_js_1 = require("../../base-scorer.js");
+const constants_js_1 = require("../../../constants.js");
+const logger_js_1 = require("../../../common/logger.js");
+const prompts_js_1 = require("./prompts.js");
+const index_js_1 = require("../../../judges/index.js");
+// Required parameters for this scorer
+const required_params = ['input', 'actualOutput'];
+/**
+ * InstructionAdherenceScorer evaluates how well an LLM follows instructions
+ * by extracting instructions from the input and checking if they are followed in the output.
+ *
+ * The score is the average of scores for each instruction (1 = followed, 0.5 = partially followed, 0 = not followed).
+ */
+class InstructionAdherenceScorer extends base_scorer_js_1.JudgevalScorer {
+    /**
+     * Create a new InstructionAdherenceScorer
+     *
+     * @param threshold - Success threshold (default: 0.5)
+     * @param model - Model to use for evaluation (default: DefaultJudge)
+     * @param include_reason - Whether to include a reason for the score (default: true)
+     * @param async_mode - Whether to use async mode (default: false)
+     * @param strict_mode - Whether to use strict mode (default: false)
+     * @param verbose_mode - Whether to include verbose logs (default: false)
+     */
+    constructor(threshold = 0.5, model = undefined, include_reason = true, async_mode = false, strict_mode = false, verbose_mode = false) {
+        super(constants_js_1.APIScorer.INSTRUCTION_ADHERENCE, strict_mode ? 1 : threshold, undefined, include_reason, async_mode, strict_mode, verbose_mode);
+        this._instructions = [];
+        this._verdicts = [];
+        const { judge, usingNativeModel } = (0, index_js_1.createJudge)(model);
+        this.model = judge;
+        this.using_native_model = usingNativeModel;
+        this.evaluation_model = this.model.getModelName();
+    }
+    /**
+     * Extract instructions from input text
+     */
+    _aGetInstructions(input) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const prompt = prompts_js_1.InstructionAdherenceTemplate.getInstructions(input);
+            if (this.using_native_model) {
+                const res = yield this.model.aGenerate(prompt);
+                try {
+                    const data = JSON.parse(res);
+                    return data.instructions || [];
+                }
+                catch (error) {
+                    throw new Error(`Failed to parse response: ${error}`);
+                }
+            }
+            else {
+                try {
+                    // Create a parser function to validate the response
+                    const parseInstructionsResponse = (response) => {
+                        const parsed = JSON.parse(response);
+                        const result = prompts_js_1.InstructionsSchema.safeParse(parsed);
+                        if (result.success) {
+                            return result.data;
+                        }
+                        throw new Error(`Invalid response format: ${result.error}`);
+                    };
+                    const res = yield this.model.aGenerate(prompt);
+                    return parseInstructionsResponse(res).instructions;
+                }
+                catch (error) {
+                    const res = yield this.model.aGenerate(prompt);
+                    try {
+                        const data = JSON.parse(res);
+                        return data.instructions || [];
+                    }
+                    catch (parseError) {
+                        throw new Error(`Failed to parse response: ${parseError}`);
+                    }
+                }
+            }
+        });
+    }
+    /**
+     * Extract instructions from input text (synchronous)
+     */
+    _getInstructions(input) {
+        const prompt = prompts_js_1.InstructionAdherenceTemplate.getInstructions(input);
+        if (this.using_native_model) {
+            const res = this.model.generate(prompt);
+            try {
+                const data = JSON.parse(res);
+                return data.instructions || [];
+            }
+            catch (error) {
+                throw new Error(`Failed to parse response: ${error}`);
+            }
+        }
+        else {
+            try {
+                // Create a parser function to validate the response
+                const parseInstructionsResponse = (response) => {
+                    const parsed = JSON.parse(response);
+                    const result = prompts_js_1.InstructionsSchema.safeParse(parsed);
+                    if (result.success) {
+                        return result.data;
+                    }
+                    throw new Error(`Invalid response format: ${result.error}`);
+                };
+                const res = this.model.generate(prompt);
+                return parseInstructionsResponse(res).instructions;
+            }
+            catch (error) {
+                const res = this.model.generate(prompt);
+                try {
+                    const data = JSON.parse(res);
+                    return data.instructions || [];
+                }
+                catch (parseError) {
+                    throw new Error(`Failed to parse response: ${parseError}`);
+                }
+            }
+        }
+    }
+    /**
+     * Generate verdicts for each instruction
+     */
+    _aGetVerdicts(instructions, actualOutput) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (instructions.length === 0) {
+                return [];
+            }
+            const prompt = prompts_js_1.InstructionAdherenceTemplate.generateVerdicts(instructions, actualOutput);
+            if (this.using_native_model) {
+                const res = yield this.model.aGenerate(prompt);
+                try {
+                    const data = JSON.parse(res);
+                    return data.verdicts || [];
+                }
+                catch (error) {
+                    throw new Error(`Failed to parse response: ${error}`);
+                }
+            }
+            else {
+                try {
+                    // Create a parser function to validate the response
+                    const parseVerdictsResponse = (response) => {
+                        const parsed = JSON.parse(response);
+                        const result = prompts_js_1.VerdictsSchema.safeParse(parsed);
+                        if (result.success) {
+                            return result.data;
+                        }
+                        throw new Error(`Invalid response format: ${result.error}`);
+                    };
+                    const res = yield this.model.aGenerate(prompt);
+                    return parseVerdictsResponse(res).verdicts;
+                }
+                catch (error) {
+                    const res = yield this.model.aGenerate(prompt);
+                    try {
+                        const data = JSON.parse(res);
+                        return data.verdicts || [];
+                    }
+                    catch (parseError) {
+                        throw new Error(`Failed to parse response: ${parseError}`);
+                    }
+                }
+            }
+        });
+    }
+    /**
+     * Generate verdicts for each instruction (synchronous)
+     */
+    _getVerdicts(instructions, actualOutput) {
+        if (instructions.length === 0) {
+            return [];
+        }
+        const prompt = prompts_js_1.InstructionAdherenceTemplate.generateVerdicts(instructions, actualOutput);
+        if (this.using_native_model) {
+            const res = this.model.generate(prompt);
+            try {
+                const data = JSON.parse(res);
+                return data.verdicts || [];
+            }
+            catch (error) {
+                throw new Error(`Failed to parse response: ${error}`);
+            }
+        }
+        else {
+            try {
+                // Create a parser function to validate the response
+                const parseVerdictsResponse = (response) => {
+                    const parsed = JSON.parse(response);
+                    const result = prompts_js_1.VerdictsSchema.safeParse(parsed);
+                    if (result.success) {
+                        return result.data;
+                    }
+                    throw new Error(`Invalid response format: ${result.error}`);
+                };
+                const res = this.model.generate(prompt);
+                return parseVerdictsResponse(res).verdicts;
+            }
+            catch (error) {
+                const res = this.model.generate(prompt);
+                try {
+                    const data = JSON.parse(res);
+                    return data.verdicts || [];
+                }
+                catch (parseError) {
+                    throw new Error(`Failed to parse response: ${parseError}`);
+                }
+            }
+        }
+    }
+    /**
+     * Calculate the instruction adherence score
+     */
+    _computeScore() {
+        if (this._verdicts.length === 0) {
+            return 1;
+        }
+        let totalScore = 0;
+        for (const verdict of this._verdicts) {
+            totalScore += verdict.score;
+        }
+        return totalScore / this._verdicts.length;
+    }
+    /**
+     * Create verbose logs for debugging
+     */
+    _createVerboseLogs() {
+        if (!this.verbose_mode) {
+            return null;
+        }
+        const steps = [
+            `Instructions:\n${JSON.stringify(this._instructions, null, 2)}`,
+            `Score: ${this.score}\nReason: ${this.reason || "No reason provided"}`
+        ];
+        return steps.join('\n\n');
+    }
+    /**
+     * Check if example has required parameters
+     */
+    _checkExampleParams(example) {
+        for (const param of required_params) {
+            if (param === 'input' && !example.input) {
+                throw new Error(`Example is missing required parameter: input`);
+            }
+            else if (param === 'actualOutput' && !example.actualOutput) {
+                throw new Error(`Example is missing required parameter: actualOutput`);
+            }
+        }
+    }
+    /**
+     * Score an example synchronously
+     */
+    syncScoreExample(example) {
+        (0, logger_js_1.info)("Starting example scoring (sync mode)");
+        try {
+            // Check required parameters
+            this._checkExampleParams(example);
+            // Process example
+            this._instructions = this._getInstructions(example.input);
+            this._verdicts = this._getVerdicts(this._instructions, example.actualOutput);
+            // Add instructions and verdicts to additional metadata
+            const additional_metadata = {
+                instructions: this._instructions,
+                verdicts: this._verdicts
+            };
+            this.score = this._computeScore();
+            this.reason = this._verdicts.length > 0 ? JSON.stringify(this._verdicts) : 'No instructions found';
+            this.success = this._successCheck();
+            const verbose_logs = this._createVerboseLogs();
+            (0, logger_js_1.info)(`Scoring completed with score: ${this.score}`);
+            // Ensure all fields match the ScorerData interface
+            return {
+                name: this.type,
+                threshold: this.threshold,
+                success: this.success,
+                score: this.score,
+                reason: this.reason || "",
+                strict_mode: this.strict_mode,
+                evaluation_model: this.evaluation_model || null,
+                error: null,
+                evaluation_cost: null,
+                verbose_logs: verbose_logs,
+                additional_metadata: additional_metadata
+            };
+        }
+        catch (error) {
+            // Handle errors
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            this.error = errorMessage;
+            return {
+                name: this.type,
+                threshold: this.threshold,
+                success: false,
+                score: 0,
+                reason: `Error during scoring: ${errorMessage}`,
+                strict_mode: this.strict_mode,
+                evaluation_model: this.evaluation_model || null,
+                error: errorMessage,
+                evaluation_cost: null,
+                verbose_logs: null,
+                additional_metadata: {}
+            };
+        }
+    }
+    /**
+     * Score an example asynchronously
+     */
+    scoreExample(example) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.async_mode) {
+                return this.syncScoreExample(example);
+            }
+            (0, logger_js_1.info)("Starting example scoring (async mode)");
+            try {
+                // Check required parameters
+                this._checkExampleParams(example);
+                // Process example
+                this._instructions = yield this._aGetInstructions(example.input);
+                this._verdicts = yield this._aGetVerdicts(this._instructions, example.actualOutput);
+                // Add instructions and verdicts to additional metadata
+                const additional_metadata = {
+                    instructions: this._instructions,
+                    verdicts: this._verdicts
+                };
+                this.score = this._computeScore();
+                this.reason = this._verdicts.length > 0 ? JSON.stringify(this._verdicts) : 'No instructions found';
+                this.success = this._successCheck();
+                const verbose_logs = this._createVerboseLogs();
+                (0, logger_js_1.info)(`Scoring completed with score: ${this.score}`);
+                // Ensure all fields match the ScorerData interface
+                return {
+                    name: this.type,
+                    threshold: this.threshold,
+                    success: this.success,
+                    score: this.score,
+                    reason: this.reason || "",
+                    strict_mode: this.strict_mode,
+                    evaluation_model: this.evaluation_model || null,
+                    error: null,
+                    evaluation_cost: null,
+                    verbose_logs: verbose_logs,
+                    additional_metadata: additional_metadata
+                };
+            }
+            catch (error) {
+                // Handle errors
+                const errorMessage = error instanceof Error ? error.message : String(error);
+                this.error = errorMessage;
+                return {
+                    name: this.type,
+                    threshold: this.threshold,
+                    success: false,
+                    score: 0,
+                    reason: `Error during scoring: ${errorMessage}`,
+                    strict_mode: this.strict_mode,
+                    evaluation_model: this.evaluation_model || null,
+                    error: errorMessage,
+                    evaluation_cost: null,
+                    verbose_logs: null,
+                    additional_metadata: {}
+                };
+            }
+        });
+    }
+    /**
+     * Get the name of the scorer
+     */
+    get name() {
+        return "Instruction Adherence";
+    }
+}
+exports.InstructionAdherenceScorer = InstructionAdherenceScorer;
+//# sourceMappingURL=instruction-adherence.js.map

package/dist/cjs/scorers/metrics/instruction-adherence/instruction-adherence.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"instruction-adherence.js","sourceRoot":"","sources":["../../../../../src/scorers/metrics/instruction-adherence/instruction-adherence.ts"],"names":[],"mappings":";;;;;;;;;;;;AAEA,yDAAsD;AACtD,wDAAkD;AAClD,yDAAmE;AACnE,6CAKsB;AACtB,uDAA8D;AAE9D,sCAAsC;AACtC,MAAM,eAAe,GAAG,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;AAElD;;;;;GAKG;AACH,MAAa,0BAA2B,SAAQ,+BAAc;IAM5D;;;;;;;;;OASG;IACH,YACE,YAAoB,GAAG,EACvB,QAAoC,SAAS,EAC7C,iBAA0B,IAAI,EAC9B,aAAsB,KAAK,EAC3B,cAAuB,KAAK,EAC5B,eAAwB,KAAK;QAE7B,KAAK,CACH,wBAAS,CAAC,qBAAqB,EAC/B,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,EAC3B,SAAS,EACT,cAAc,EACd,UAAU,EACV,WAAW,EACX,YAAY,CACb,CAAC;QA7BI,kBAAa,GAAa,EAAE,CAAC;QAC7B,cAAS,GAAkC,EAAE,CAAC;QA8BpD,MAAM,EAAE,KAAK,EAAE,gBAAgB,EAAE,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;QACvD,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,kBAAkB,GAAG,gBAAgB,CAAC;QAC3C,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;IACpD,CAAC;IAED;;OAEG;IACW,iBAAiB,CAAC,KAAa;;YAC3C,MAAM,MAAM,GAAG,yCAA4B,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAEnE,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBAC5B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC/C,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBAC7B,OAAO,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC;gBACjC,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC;oBACH,oDAAoD;oBACpD,MAAM,yBAAyB,GAAG,CAAC,QAAgB,EAA8B,EAAE;wBACjF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;wBACpC,MAAM,MAAM,GAAG,+BAAkB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;wBACpD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;4BACnB,OAAO,MAAM,CAAC,IAAI,CAAC;wBACrB,CAAC;wBACD,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;oBAC9D,CAAC,CAAC;oBAEF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;oBAC/C,OAAO,yBAAyB,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC;gBACrD,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;oBAC/C,IAAI,CAAC;wBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;wBAC7B,OAAO,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC;oBACjC,CAAC;oBAAC,OAAO,UAAU,EAAE,CAAC;wBACpB,MAAM,IAAI,KAAK,CAAC,6BAA6B,UAAU,EAAE,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;KAAA;IAED;;OAEG;IACK,gBAAgB,CAAC,KAAa;QACpC,MAAM,MAAM,GAAG,yCAA4B,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAEnE,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACxC,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC7B,OAAO,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC;YACjC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,oDAAoD;gBACpD,MAAM,yBAAyB,GAAG,CAAC,QAAgB,EAA8B,EAAE;oBACjF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;oBACpC,MAAM,MAAM,GAAG,+BAAkB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;oBACpD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;wBACnB,OAAO,MAAM,CAAC,IAAI,CAAC;oBACrB,CAAC;oBACD,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC9D,CAAC,CAAC;gBAEF,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACxC,OAAO,yBAAyB,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC;YACrD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACxC,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBAC7B,OAAO,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC;gBACjC,CAAC;gBAAC,OAAO,UAAU,EAAE,CAAC;oBACpB,MAAM,IAAI,KAAK,CAAC,6BAA6B,UAAU,EAAE,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACW,aAAa,CAAC,YAAsB,EAAE,YAAoB;;YACtE,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,MAAM,MAAM,GAAG,yCAA4B,CAAC,gBAAgB,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;YAEzF,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBAC5B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC/C,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBAC7B,OAAO,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;gBAC7B,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC;oBACH,oDAAoD;oBACpD,MAAM,qBAAqB,GAAG,CAAC,QAAgB,EAA+C,EAAE;wBAC9F,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;wBACpC,MAAM,MAAM,GAAG,2BAAc,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;wBAChD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;4BACnB,OAAO,MAAM,CAAC,IAAI,CAAC;wBACrB,CAAC;wBACD,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;oBAC9D,CAAC,CAAC;oBAEF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;oBAC/C,OAAO,qBAAqB,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;gBAC7C,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;oBAC/C,IAAI,CAAC;wBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;wBAC7B,OAAO,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;oBAC7B,CAAC;oBAAC,OAAO,UAAU,EAAE,CAAC;wBACpB,MAAM,IAAI,KAAK,CAAC,6BAA6B,UAAU,EAAE,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;KAAA;IAED;;OAEG;IACK,YAAY,CAAC,YAAsB,EAAE,YAAoB;QAC/D,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,MAAM,GAAG,yCAA4B,CAAC,gBAAgB,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;QAEzF,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACxC,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC7B,OAAO,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;YAC7B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,oDAAoD;gBACpD,MAAM,qBAAqB,GAAG,CAAC,QAAgB,EAA+C,EAAE;oBAC9F,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;oBACpC,MAAM,MAAM,GAAG,2BAAc,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;oBAChD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;wBACnB,OAAO,MAAM,CAAC,IAAI,CAAC;oBACrB,CAAC;oBACD,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC9D,CAAC,CAAC;gBAEF,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACxC,OAAO,qBAAqB,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YAC7C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACxC,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBAC7B,OAAO,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;gBAC7B,CAAC;gBAAC,OAAO,UAAU,EAAE,CAAC;oBACpB,MAAM,IAAI,KAAK,CAAC,6BAA6B,UAAU,EAAE,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa;QACnB,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,OAAO,CAAC,CAAC;QACX,CAAC;QAED,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrC,UAAU,IAAI,OAAO,CAAC,KAAK,CAAC;QAC9B,CAAC;QAED,OAAO,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;IAC5C,CAAC;IAED;;OAEG;IACK,kBAAkB;QACxB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,KAAK,GAAG;YACZ,kBAAkB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;YAC/D,UAAU,IAAI,CAAC,KAAK,aAAa,IAAI,CAAC,MAAM,IAAI,oBAAoB,EAAE;SACvE,CAAC;QAEF,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAED;;OAEG;IACK,mBAAmB,CAAC,OAAgB;QAC1C,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;YACpC,IAAI,KAAK,KAAK,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACxC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;YAClE,CAAC;iBAAM,IAAI,KAAK,KAAK,cAAc,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;gBAC7D,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;YACzE,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,OAAgB;QAC/B,IAAA,gBAAI,EAAC,sCAAsC,CAAC,CAAC;QAE7C,IAAI,CAAC;YACH,4BAA4B;YAC5B,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;YAElC,kBAAkB;YAClB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,KAAM,CAAC,CAAC;YAC3D,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC,YAAsB,CAAC,CAAC;YAEvF,uDAAuD;YACvD,MAAM,mBAAmB,GAAG;gBAC1B,YAAY,EAAE,IAAI,CAAC,aAAa;gBAChC,QAAQ,EAAE,IAAI,CAAC,SAAS;aACzB,CAAC;YAEF,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC;YACnG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YACpC,MAAM,YAAY,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAE/C,IAAA,gBAAI,EAAC,iCAAiC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;YAEpD,mDAAmD;YACnD,OAAO;gBACL,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;gBACzB,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,IAAI,IAAI;gBAC/C,KAAK,EAAE,IAAI;gBACX,eAAe,EAAE,IAAI;gBACrB,YAAY,EAAE,YAAY;gBAC1B,mBAAmB,EAAE,mBAAmB;aACzC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,gBAAgB;YAChB,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC;YAE1B,OAAO;gBACL,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,yBAAyB,YAAY,EAAE;gBAC/C,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,IAAI,IAAI;gBAC/C,KAAK,EAAE,YAAY;gBACnB,eAAe,EAAE,IAAI;gBACrB,YAAY,EAAE,IAAI;gBAClB,mBAAmB,EAAE,EAAE;aACxB,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACG,YAAY,CAAC,OAAgB;;YACjC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACrB,OAAO,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YACxC,CAAC;YAED,IAAA,gBAAI,EAAC,uCAAuC,CAAC,CAAC;YAE9C,IAAI,CAAC;gBACH,4BAA4B;gBAC5B,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;gBAElC,kBAAkB;gBAClB,IAAI,CAAC,aAAa,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAM,CAAC,CAAC;gBAClE,IAAI,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC,YAAsB,CAAC,CAAC;gBAE9F,uDAAuD;gBACvD,MAAM,mBAAmB,GAAG;oBAC1B,YAAY,EAAE,IAAI,CAAC,aAAa;oBAChC,QAAQ,EAAE,IAAI,CAAC,SAAS;iBACzB,CAAC;gBAEF,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;gBAClC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC;gBACnG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;gBACpC,MAAM,YAAY,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBAE/C,IAAA,gBAAI,EAAC,iCAAiC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;gBAEpD,mDAAmD;gBACnD,OAAO;oBACL,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,OAAO,EAAE,IAAI,CAAC,OAAO;oBACrB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;oBACzB,WAAW,EAAE,IAAI,CAAC,WAAW;oBAC7B,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,IAAI,IAAI;oBAC/C,KAAK,EAAE,IAAI;oBACX,eAAe,EAAE,IAAI;oBACrB,YAAY,EAAE,YAAY;oBAC1B,mBAAmB,EAAE,mBAAmB;iBACzC,CAAC;YACJ,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,gBAAgB;gBAChB,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC5E,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC;gBAE1B,OAAO;oBACL,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,yBAAyB,YAAY,EAAE;oBAC/C,WAAW,EAAE,IAAI,CAAC,WAAW;oBAC7B,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,IAAI,IAAI;oBAC/C,KAAK,EAAE,YAAY;oBACnB,eAAe,EAAE,IAAI;oBACrB,YAAY,EAAE,IAAI;oBAClB,mBAAmB,EAAE,EAAE;iBACxB,CAAC;YACJ,CAAC;QACH,CAAC;KAAA;IAED;;OAEG;IACH,IAAI,IAAI;QACN,OAAO,uBAAuB,CAAC;IACjC,CAAC;CACF;AAnYD,gEAmYC"}

package/dist/cjs/scorers/metrics/instruction-adherence/prompts.js

@@ -0,0 +1,124 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InstructionAdherenceTemplate = exports.InstructionsSchema = exports.VerdictsSchema = exports.VerdictSchema = void 0;
+const zod_1 = require("zod");
+/**
+ * Schema for a single instruction adherence verdict
+ */
+exports.VerdictSchema = zod_1.z.object({
+    instruction: zod_1.z.string(),
+    score: zod_1.z.number(),
+    reason: zod_1.z.string()
+});
+/**
+ * Schema for a list of verdicts
+ */
+exports.VerdictsSchema = zod_1.z.object({
+    verdicts: zod_1.z.array(exports.VerdictSchema)
+});
+/**
+ * Schema for a list of instructions
+ */
+exports.InstructionsSchema = zod_1.z.object({
+    instructions: zod_1.z.array(zod_1.z.string())
+});
+/**
+ * Templates for prompts used in the InstructionAdherenceScorer
+ */
+class InstructionAdherenceTemplate {
+    /**
+     * Generate a prompt to extract instructions from input text
+     */
+    static getInstructions(input) {
+        return `You will be presented with a piece of text. Your task is to break down the text and generate a list of the instructions contained within the text.
+
+===== START OF EXAMPLES =====
+Example 1:
+Example text: Hello my name is John Doe. I like cars. Write two poems about the weather and create a joke. Also what is 5 + 5?
+
+Output:
+{
+    "instructions": ["Write two poem about the weather", "Create a joke", "What is 5 + 5?"]
+}
+===== END OF EXAMPLES =====
+
+**
+IMPORTANT: Please return your answer in valid JSON format, with the "instructions" key mapping to a list of strings. No words or explanation is needed.
+**
+
+==== START OF INPUT ====
+Text:
+${input}
+==== END OF INPUT ====
+
+==== YOUR ANSWER ====
+JSON:`;
+    }
+    /**
+     * Generate a prompt to evaluate adherence to instructions
+     */
+    static generateVerdicts(instructions, actualOutput) {
+        return `You will be presented with a list of instructions and a piece of text. For each instruction, determine if the instruction was completed in the text. There are 3 categories: either completed, partially completed, or not completed. The scores for these will be 1, 0.5, and 0 respectively.
+Go through each instruction and provide score for each instruction as well as the reasoning for that score.
+
+==== FORMATTING YOUR ANSWER ====
+Please return your answer in JSON format, with a list of JSON objects with keys "instruction", "score", and "reason". No words or explanation beyond the output JSON is needed.
+
+
+===== START OF EXAMPLES =====
+Example 1:
+instructions: ["Write two poems about the weather", "Create a joke", "What is 5 + 5?"]
+output: Poem 1: The Sun's Embrace
+The sun climbs high, a golden flame,
+It whispers warmth, it calls my name.
+The sky, a canvas, blue and clear,
+A perfect day for cars, my dear.
+
+The asphalt hums beneath the wheels,
+A symphony of speed it feels.
+The weather smiles, no clouds in sight,
+A driver's joy, pure delight.
+
+Poem 2: The Storm's Dance
+A sunlit meadow, alive with whispers of wind, where daisies dance and hope begins again. Each petal holds a promise—bright, unbruised— a symphony of light that cannot be refused.
+
+Joke
+Why dont cars ever get cold in the winter?
+Because they have radiators!
+
+Math Answer
+5 + 5 = 10
+
+YOUR JSON OUTPUT:
+{
+    "verdicts": [
+        {
+            "instruction": "Write two poem about the weather",
+            "score": 0.5,
+            "reason": "The output contained one poem about the weather, but the other poem was not about the weather."
+        },
+        {
+            "instruction": "Create a joke",
+            "score": 1,
+            "reason": "There was a joke created in the output."
+        },
+        {
+            "instruction": "What is 5 + 5?",
+            "score": 1,
+            "reason": "The answer to the math question was provided in the output."
+        }
+    ]
+}
+===== END OF EXAMPLES =====
+
+==== START OF INPUT ====
+instructions: ${JSON.stringify(instructions)}
+output: ${actualOutput}
+==== END OF INPUT ====
+
+==== YOUR ANSWER ====
+JSON:`;
+    }
+}
+exports.InstructionAdherenceTemplate = InstructionAdherenceTemplate;
+//# sourceMappingURL=prompts.js.map

package/dist/cjs/scorers/metrics/instruction-adherence/prompts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompts.js","sourceRoot":"","sources":["../../../../../src/scorers/metrics/instruction-adherence/prompts.ts"],"names":[],"mappings":";;;AAAA,6BAAwB;AAExB;;GAEG;AACU,QAAA,aAAa,GAAG,OAAC,CAAC,MAAM,CAAC;IACpC,WAAW,EAAE,OAAC,CAAC,MAAM,EAAE;IACvB,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE;IACjB,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE;CACnB,CAAC,CAAC;AAEH;;GAEG;AACU,QAAA,cAAc,GAAG,OAAC,CAAC,MAAM,CAAC;IACrC,QAAQ,EAAE,OAAC,CAAC,KAAK,CAAC,qBAAa,CAAC;CACjC,CAAC,CAAC;AAEH;;GAEG;AACU,QAAA,kBAAkB,GAAG,OAAC,CAAC,MAAM,CAAC;IACzC,YAAY,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;CAClC,CAAC,CAAC;AAYH;;GAEG;AACH,MAAa,4BAA4B;IACvC;;OAEG;IACH,MAAM,CAAC,eAAe,CAAC,KAAa;QAClC,OAAO;;;;;;;;;;;;;;;;;;EAkBT,KAAK;;;;MAID,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,gBAAgB,CAAC,YAAsB,EAAE,YAAoB;QAClE,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gBAsDK,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC;UAClC,YAAY;;;;MAIhB,CAAC;IACL,CAAC;CACF;AA/FD,oEA+FC"}