agentevals 0.0.0

package/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2025 LangChain, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1 @@
+# js

package/dist/evaluators/exact.cjs

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.exactMatch = void 0;
+const utils_js_1 = require("./utils.cjs");
+const _scorer = (params) => {
+    const { outputs, referenceOutputs } = params;
+    if (outputs === null || referenceOutputs === null) {
+        throw new Error("Exact match requires both outputs and referenceOutputs");
+    }
+    const outputsJson = JSON.stringify(outputs, null, 2);
+    const referenceOutputsJson = JSON.stringify(referenceOutputs, null, 2);
+    return outputsJson === referenceOutputsJson;
+};
+/**
+ * Performs exact matching between input and output values.
+ * @param outputs outputs to compare
+ * @param referenceOutputs Reference outputs to compare
+ * @returns
+ */
+const exactMatch = async (params) => {
+    return (0, utils_js_1._runEvaluator)("exact_match", _scorer, "exact_match", params);
+};
+exports.exactMatch = exactMatch;

package/dist/evaluators/exact.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Performs exact matching between input and output values.
+ * @param outputs outputs to compare
+ * @param referenceOutputs Reference outputs to compare
+ * @returns
+ */
+export declare const exactMatch: (params: {
+    outputs: unknown;
+    referenceOutputs: unknown;
+}) => Promise<import("langsmith/vitest").SimpleEvaluationResult>;

package/dist/evaluators/exact.js

@@ -0,0 +1,19 @@
+import { _runEvaluator } from "./utils.js";
+const _scorer = (params) => {
+    const { outputs, referenceOutputs } = params;
+    if (outputs === null || referenceOutputs === null) {
+        throw new Error("Exact match requires both outputs and referenceOutputs");
+    }
+    const outputsJson = JSON.stringify(outputs, null, 2);
+    const referenceOutputsJson = JSON.stringify(referenceOutputs, null, 2);
+    return outputsJson === referenceOutputsJson;
+};
+/**
+ * Performs exact matching between input and output values.
+ * @param outputs outputs to compare
+ * @param referenceOutputs Reference outputs to compare
+ * @returns
+ */
+export const exactMatch = async (params) => {
+    return _runEvaluator("exact_match", _scorer, "exact_match", params);
+};

package/dist/evaluators/llm.cjs

@@ -0,0 +1,284 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createLLMAsJudge = exports._createLLMAsJudgeScorer = void 0;
+const runnables_1 = require("@langchain/core/runnables");
+const prompts_1 = require("@langchain/core/prompts");
+const universal_1 = require("langchain/chat_models/universal");
+const traceable_1 = require("langsmith/traceable");
+const utils_js_1 = require("./utils.cjs");
+function _isRunnableInterface(prompt) {
+    return runnables_1.Runnable.isRunnable(prompt);
+}
+function _isBaseChatModel(x) {
+    const model = x;
+    return (x != null &&
+        typeof x === "object" &&
+        typeof model._modelType === "function" &&
+        model._modelType() === "base_chat_model");
+}
+function appendFewShotExamples({ messages, fewShotExamples, }) {
+    // Find the last user message to append examples to
+    const lastUserMessageIdx = messages
+        .slice()
+        .reverse()
+        .findIndex((msg) => msg.role === "user");
+    if (lastUserMessageIdx === -1) {
+        throw new Error("Appending few-shot examples requires a user message in the provided prompt");
+    }
+    const actualIdx = messages.length - 1 - lastUserMessageIdx;
+    messages[actualIdx].content +=
+        "\n\n" +
+            fewShotExamples
+                .map((example) => {
+                let exampleStr = `<example>\n<input>${JSON.stringify(example.inputs)}</input>\n<output>${JSON.stringify(example.outputs)}</output>`;
+                if (example.reasoning) {
+                    exampleStr += `\n<reasoning>${example.reasoning}</reasoning>`;
+                }
+                if (example.score !== undefined) {
+                    exampleStr += `\n<score>${example.score}</score>`;
+                }
+                exampleStr += "\n</example>";
+                return exampleStr;
+            })
+                .join("\n");
+    return messages;
+}
+function constructOutputSchema({ schema, continuous, choices, useReasoning, }) {
+    const jsonSchema = schema ?? {
+        type: "object",
+        additionalProperties: false,
+        strict: true,
+    };
+    let description;
+    let scoreSchema;
+    if (choices) {
+        description =
+            "A number that represents the degree to which the criteria in the prompt are met.";
+        scoreSchema = {
+            type: "number",
+            description,
+            enum: choices,
+        };
+    }
+    else if (continuous) {
+        description =
+            "A number that represents the degree to which the criteria in the prompt are met, from 0.0 to 1.0. 1.0 means the criteria are met perfectly. 0.0 means none of the criteria are met.";
+        scoreSchema = {
+            type: "number",
+            description,
+        };
+    }
+    else {
+        description =
+            "A score that is true if criteria in the prompt are met, and false otherwise.";
+        scoreSchema = {
+            type: "boolean",
+            description,
+        };
+    }
+    if (!schema) {
+        if (useReasoning) {
+            jsonSchema.properties = {
+                reasoning: {
+                    type: "string",
+                    description: "A human-readable explanation of the score. You MUST end the reasoning with a sentence that says: Thus, the score should be: SCORE_YOU_ASSIGN.",
+                },
+                score: scoreSchema,
+            };
+            jsonSchema.required = ["reasoning", "score"];
+        }
+        else {
+            jsonSchema.properties = {
+                score: scoreSchema,
+            };
+            jsonSchema.required = ["score"];
+        }
+    }
+    return [jsonSchema, description];
+}
+const _createLLMAsJudgeScorer = (params) => {
+    const { prompt, system, schema, model, continuous, choices, fewShotExamples, } = params;
+    let judge = params.judge;
+    const useReasoning = params.useReasoning ?? true;
+    const getScore = async (params) => {
+        let { inputs, outputs, referenceOutputs, ...rest } = params;
+        if (system && typeof prompt !== "string") {
+            throw new Error("`system` is only supported when `prompt` is a string template");
+        }
+        let stringifiedInputs = inputs;
+        let stringifiedOutputs = outputs;
+        let stringifiedReferenceOutputs = referenceOutputs;
+        if (inputs && typeof inputs !== "string") {
+            stringifiedInputs = JSON.stringify(inputs);
+        }
+        if (outputs && typeof outputs !== "string") {
+            stringifiedOutputs = JSON.stringify(outputs);
+        }
+        if (referenceOutputs && typeof referenceOutputs !== "string") {
+            stringifiedReferenceOutputs = JSON.stringify(referenceOutputs);
+        }
+        const stringifiedRest = Object.fromEntries(Object.entries(rest).map(([key, value]) => [
+            key,
+            typeof value === "string" ? value : JSON.stringify(value),
+        ]));
+        let messages = [];
+        if (_isRunnableInterface(prompt)) {
+            const formattedPrompt = await prompt.invoke({
+                inputs: stringifiedInputs,
+                outputs: stringifiedOutputs,
+                reference_outputs: stringifiedReferenceOutputs,
+                ...stringifiedRest,
+            });
+            messages = formattedPrompt.messages;
+        }
+        else if (typeof prompt === "string") {
+            const template = prompts_1.ChatPromptTemplate.fromTemplate(prompt);
+            const formattedPrompt = await template.invoke({
+                inputs: stringifiedInputs,
+                outputs: stringifiedOutputs,
+                reference_outputs: stringifiedReferenceOutputs,
+                ...stringifiedRest,
+            });
+            messages = formattedPrompt.messages;
+        }
+        else {
+            messages = await prompt({
+                inputs,
+                outputs,
+                reference_outputs: referenceOutputs,
+                ...rest,
+            });
+        }
+        if (system) {
+            messages = [{ role: "system", content: system }, ...messages];
+        }
+        let normalizedMessages = (0, utils_js_1._normalizeToOpenAIMessagesList)(messages);
+        if (fewShotExamples) {
+            normalizedMessages = appendFewShotExamples({
+                messages: normalizedMessages,
+                fewShotExamples,
+            });
+        }
+        const [jsonSchema, description] = constructOutputSchema({
+            schema,
+            continuous,
+            choices,
+            useReasoning,
+        });
+        if (!judge) {
+            judge = await (0, universal_1.initChatModel)(model);
+        }
+        let response;
+        if (_isBaseChatModel(judge)) {
+            response = await judge
+                .withStructuredOutput({
+                title: "score",
+                description,
+                ...jsonSchema,
+            })
+                .invoke(normalizedMessages);
+            if (schema === undefined) {
+                if (useReasoning) {
+                    return [response.score, response.reasoning];
+                }
+                return response.score;
+            }
+            else {
+                return response;
+            }
+        }
+        else {
+            if (!model) {
+                throw new Error("`model` is required for non-LangChain clients");
+            }
+            const params = {
+                messages: normalizedMessages,
+                model,
+                response_format: {
+                    type: "json_schema",
+                    json_schema: {
+                        name: "score",
+                        strict: true,
+                        schema: jsonSchema,
+                    },
+                },
+            };
+            const invokeLlm = (0, traceable_1.traceable)(judge.chat.completions.create.bind(judge.chat.completions), {
+                metadata: {
+                    ls_provider: "openai",
+                    ls_model_name: model,
+                    ls_model_type: "chat",
+                },
+                run_type: "llm",
+                name: "OpenAI Chat Completion",
+            });
+            const response = await invokeLlm(params);
+            const parsed = JSON.parse(response.choices[0].message.content);
+            if (schema === undefined) {
+                if (useReasoning) {
+                    return [parsed.score, parsed.reasoning];
+                }
+                return parsed.score;
+            }
+            return parsed;
+        }
+    };
+    return getScore;
+};
+exports._createLLMAsJudgeScorer = _createLLMAsJudgeScorer;
+/**
+ * Create an evaluator that uses an LLM to assess output quality based on specified criteria.
+ *
+ * @param params Configuration object with the following properties:
+ * @param params.prompt The evaluation prompt - can be a string template, LangChain prompt template,
+ *                     or function that returns a list of chat messages
+ * @param params.feedbackKey Key used to store the evaluation result, defaults to "score"
+ * @param params.judge The LLM used for evaluation. Can be an OpenAI client or a LangChain model.
+ *                     If using OpenAI client, must specify "model" parameter.
+ *                     If omitted, "model" will be used to instantiate a LangChain model instance.
+ * @param params.model Model identifier to use. Defaults to "openai:o3-mini".
+ *                     If "judge" is an OpenAI client, this should be a model name directly.
+ *                     If "judge" is omitted, must be a valid LangChain model identifier.
+ * @param params.system Optional system message to prepend to the prompt
+ * @param params.continuous If true, score will be a float between 0 and 1.
+ *                         If false, score will be boolean. Defaults to false.
+ * @param params.choices Optional list of specific float values the score must be chosen from
+ * @param params.useReasoning If true, includes explanation for the score in the output.
+ *                           Defaults to true.
+ * @param params.fewShotExamples Optional list of example evaluations to append to the prompt
+ *
+ * @returns A function that takes inputs, outputs, reference_outputs, and other kwargs,
+ *          formats them into a prompt, invokes the judge, and returns an evaluation result
+ *
+ * @example
+ * ```typescript
+ * import { createLLMAsJudge } from "openevals";
+ *
+ * const evaluator = createLLMAsJudge({
+ *   prompt: "Rate the quality of this response from 0 to 1: {outputs}",
+ *   continuous: true,
+ * });
+ * const result = await evaluator({
+ *   inputs: { question: "What color is the sky?" },
+ *   outputs: { response: "Blue" },
+ * });
+ * ```
+ */
+const createLLMAsJudge = ({ prompt, feedbackKey = "score", model = "openai:o3-mini", system, judge, continuous = false, choices, useReasoning = true, fewShotExamples, }) => {
+    const scorer = (0, exports._createLLMAsJudgeScorer)({
+        prompt,
+        judge,
+        model,
+        system,
+        continuous,
+        choices,
+        useReasoning,
+        fewShotExamples,
+    });
+    const _wrappedEvaluator = async (inputs) => {
+        const runName = feedbackKey !== "score" ? "llm_as_judge" : `llm_as_${feedbackKey}_judge`;
+        return (0, utils_js_1._runEvaluator)(runName, scorer, feedbackKey, inputs);
+    };
+    return _wrappedEvaluator;
+};
+exports.createLLMAsJudge = createLLMAsJudge;

package/dist/evaluators/llm.d.ts

@@ -0,0 +1,73 @@
+import { RunnableInterface } from "@langchain/core/runnables";
+import { BaseChatModel } from "@langchain/core/language_models/chat_models";
+import { ChatCompletionMessage, FewShotExample, ModelClient, SingleResultScorerReturnType } from "./types.js";
+export declare const _createLLMAsJudgeScorer: (params: {
+    prompt: string | RunnableInterface<any, any, import("@langchain/core/runnables").RunnableConfig<Record<string, any>>> | ((...args: unknown[]) => ChatCompletionMessage[] | Promise<ChatCompletionMessage[]>);
+    system?: string | undefined;
+    schema?: Record<string, unknown> | undefined;
+    judge?: ModelClient | BaseChatModel<import("@langchain/core/language_models/chat_models").BaseChatModelCallOptions, import("@langchain/core/messages").AIMessageChunk> | undefined;
+    model?: string | undefined;
+    continuous?: boolean | undefined;
+    choices?: number[] | undefined;
+    useReasoning?: boolean | undefined;
+    fewShotExamples?: FewShotExample[] | undefined;
+}) => (params: {
+    [key: string]: unknown;
+    inputs: unknown;
+    outputs: unknown;
+    referenceOutputs?: unknown;
+}) => Promise<SingleResultScorerReturnType>;
+/**
+ * Create an evaluator that uses an LLM to assess output quality based on specified criteria.
+ *
+ * @param params Configuration object with the following properties:
+ * @param params.prompt The evaluation prompt - can be a string template, LangChain prompt template,
+ *                     or function that returns a list of chat messages
+ * @param params.feedbackKey Key used to store the evaluation result, defaults to "score"
+ * @param params.judge The LLM used for evaluation. Can be an OpenAI client or a LangChain model.
+ *                     If using OpenAI client, must specify "model" parameter.
+ *                     If omitted, "model" will be used to instantiate a LangChain model instance.
+ * @param params.model Model identifier to use. Defaults to "openai:o3-mini".
+ *                     If "judge" is an OpenAI client, this should be a model name directly.
+ *                     If "judge" is omitted, must be a valid LangChain model identifier.
+ * @param params.system Optional system message to prepend to the prompt
+ * @param params.continuous If true, score will be a float between 0 and 1.
+ *                         If false, score will be boolean. Defaults to false.
+ * @param params.choices Optional list of specific float values the score must be chosen from
+ * @param params.useReasoning If true, includes explanation for the score in the output.
+ *                           Defaults to true.
+ * @param params.fewShotExamples Optional list of example evaluations to append to the prompt
+ *
+ * @returns A function that takes inputs, outputs, reference_outputs, and other kwargs,
+ *          formats them into a prompt, invokes the judge, and returns an evaluation result
+ *
+ * @example
+ * ```typescript
+ * import { createLLMAsJudge } from "openevals";
+ *
+ * const evaluator = createLLMAsJudge({
+ *   prompt: "Rate the quality of this response from 0 to 1: {outputs}",
+ *   continuous: true,
+ * });
+ * const result = await evaluator({
+ *   inputs: { question: "What color is the sky?" },
+ *   outputs: { response: "Blue" },
+ * });
+ * ```
+ */
+export declare const createLLMAsJudge: ({ prompt, feedbackKey, model, system, judge, continuous, choices, useReasoning, fewShotExamples, }: {
+    prompt: string | RunnableInterface<any, any, import("@langchain/core/runnables").RunnableConfig<Record<string, any>>> | ((...args: unknown[]) => ChatCompletionMessage[] | Promise<ChatCompletionMessage[]>);
+    feedbackKey?: string | undefined;
+    model?: string | undefined;
+    system?: string | undefined;
+    judge?: ModelClient | BaseChatModel<import("@langchain/core/language_models/chat_models").BaseChatModelCallOptions, import("@langchain/core/messages").AIMessageChunk> | undefined;
+    continuous?: boolean | undefined;
+    choices?: number[] | undefined;
+    useReasoning?: boolean | undefined;
+    fewShotExamples?: FewShotExample[] | undefined;
+}) => (inputs: {
+    [key: string]: unknown;
+    inputs: unknown;
+    outputs: unknown;
+    referenceOutputs?: unknown;
+}) => Promise<import("langsmith/vitest").SimpleEvaluationResult>;