agentevals 0.0.0

package/dist/evaluators/trajectory/unordered.cjs

@@ -0,0 +1,33 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.trajectoryUnorderedMatch = void 0;
+const utils_js_1 = require("../utils.cjs");
+const utils_js_2 = require("./utils.cjs");
+/**
+ * Evaluate whether an input agent trajectory and called tools contains all the tools used in a reference trajectory.
+ * This accounts for some differences in an LLM's reasoning process in a case-by-case basis.
+ *
+ * @param params - The parameters for trajectory unordered match evaluation
+ * @param params.outputs - Actual trajectory the agent followed.
+ *    May be a list of OpenAI messages, a list of LangChain messages, or a dictionary containing
+ *    a "messages" key with one of the above.
+ * @param params.reference_outputs - Ideal reference trajectory the agent should have followed.
+ *    May be a list of OpenAI messages, a list of LangChain messages, or a dictionary containing
+ *    a "messages" key with one of the above.
+ * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
+ */
+async function trajectoryUnorderedMatch(params) {
+    const { outputs, referenceOutputs } = params;
+    const outputsList = (0, utils_js_1._normalizeToOpenAIMessagesList)(outputs);
+    const referenceOutputsList = (0, utils_js_1._normalizeToOpenAIMessagesList)(referenceOutputs);
+    const getScore = async () => {
+        if (outputsList == null || referenceOutputsList == null) {
+            throw new Error("Trajectory unordered match requires both outputs and reference_outputs");
+        }
+        const unorderedMatch = (0, utils_js_2._isTrajectorySuperset)(outputsList, referenceOutputsList) &&
+            (0, utils_js_2._isTrajectorySuperset)(referenceOutputsList, outputsList);
+        return unorderedMatch;
+    };
+    return (0, utils_js_1._runEvaluator)("trajectory_unordered_match", getScore, "trajectory_unordered_match", params);
+}
+exports.trajectoryUnorderedMatch = trajectoryUnorderedMatch;

package/dist/evaluators/trajectory/unordered.d.ts

@@ -0,0 +1,23 @@
+import { BaseMessage } from "@langchain/core/messages";
+import { ChatCompletionMessage, EvaluatorResult } from "../types.js";
+/**
+ * Evaluate whether an input agent trajectory and called tools contains all the tools used in a reference trajectory.
+ * This accounts for some differences in an LLM's reasoning process in a case-by-case basis.
+ *
+ * @param params - The parameters for trajectory unordered match evaluation
+ * @param params.outputs - Actual trajectory the agent followed.
+ *    May be a list of OpenAI messages, a list of LangChain messages, or a dictionary containing
+ *    a "messages" key with one of the above.
+ * @param params.reference_outputs - Ideal reference trajectory the agent should have followed.
+ *    May be a list of OpenAI messages, a list of LangChain messages, or a dictionary containing
+ *    a "messages" key with one of the above.
+ * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
+ */
+export declare function trajectoryUnorderedMatch(params: {
+    outputs: ChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage)[];
+    };
+    referenceOutputs: ChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage)[];
+    };
+}): Promise<EvaluatorResult>;

package/dist/evaluators/trajectory/unordered.js

@@ -0,0 +1,29 @@
+import { _normalizeToOpenAIMessagesList, _runEvaluator } from "../utils.js";
+import { _isTrajectorySuperset } from "./utils.js";
+/**
+ * Evaluate whether an input agent trajectory and called tools contains all the tools used in a reference trajectory.
+ * This accounts for some differences in an LLM's reasoning process in a case-by-case basis.
+ *
+ * @param params - The parameters for trajectory unordered match evaluation
+ * @param params.outputs - Actual trajectory the agent followed.
+ *    May be a list of OpenAI messages, a list of LangChain messages, or a dictionary containing
+ *    a "messages" key with one of the above.
+ * @param params.reference_outputs - Ideal reference trajectory the agent should have followed.
+ *    May be a list of OpenAI messages, a list of LangChain messages, or a dictionary containing
+ *    a "messages" key with one of the above.
+ * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
+ */
+export async function trajectoryUnorderedMatch(params) {
+    const { outputs, referenceOutputs } = params;
+    const outputsList = _normalizeToOpenAIMessagesList(outputs);
+    const referenceOutputsList = _normalizeToOpenAIMessagesList(referenceOutputs);
+    const getScore = async () => {
+        if (outputsList == null || referenceOutputsList == null) {
+            throw new Error("Trajectory unordered match requires both outputs and reference_outputs");
+        }
+        const unorderedMatch = _isTrajectorySuperset(outputsList, referenceOutputsList) &&
+            _isTrajectorySuperset(referenceOutputsList, outputsList);
+        return unorderedMatch;
+    };
+    return _runEvaluator("trajectory_unordered_match", getScore, "trajectory_unordered_match", params);
+}

package/dist/evaluators/trajectory/utils.cjs

@@ -0,0 +1,68 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports._chatCompletionMessagesToString = exports._isTrajectorySuperset = void 0;
+function _normalizeToolCall(toolCall) {
+    if ("function" in toolCall &&
+        toolCall.function != null &&
+        typeof toolCall.function === "object") {
+        return {
+            name: toolCall.function.name,
+            args: toolCall.function.arguments,
+        };
+    }
+    return toolCall;
+}
+function _extractToolCalls(messages) {
+    const toolCalls = [];
+    for (const message of messages) {
+        if (message.tool_calls) {
+            toolCalls.push(...message.tool_calls.map(_normalizeToolCall));
+        }
+    }
+    return toolCalls;
+}
+function _isTrajectorySuperset(outputs, referenceOutputs) {
+    const outputToolCalls = _extractToolCalls(outputs);
+    const referenceToolCalls = _extractToolCalls(referenceOutputs);
+    const outputToolCounts = new Map();
+    const referenceToolCounts = new Map();
+    for (const call of outputToolCalls) {
+        outputToolCounts.set(call.name, (outputToolCounts.get(call.name) ?? 0) + 1);
+    }
+    for (const call of referenceToolCalls) {
+        referenceToolCounts.set(call.name, (referenceToolCounts.get(call.name) ?? 0) + 1);
+    }
+    const allTools = new Set([
+        ...outputToolCounts.keys(),
+        ...referenceToolCounts.keys(),
+    ]);
+    for (const name of allTools) {
+        if ((outputToolCounts.get(name) ?? 0) < (referenceToolCounts.get(name) ?? 0)) {
+            return false;
+        }
+    }
+    return true;
+}
+exports._isTrajectorySuperset = _isTrajectorySuperset;
+function _chatCompletionMessagesToString(messages) {
+    function formatMessage(message) {
+        let content = message.content ?? "";
+        // Handle tool/function calls
+        if (message.tool_calls) {
+            const toolCallsStr = message.tool_calls
+                .map((call) => {
+                const func = call.function ?? {};
+                return `<tool_call>\n<name>${func.name ?? ""}</name>\n<arguments>${func.arguments ?? ""}</arguments>\n</tool_call>`;
+            })
+                .join("\n");
+            content = content ? `${content}\n${toolCallsStr}` : toolCallsStr;
+        }
+        // Handle tool call results
+        if (message.tool_call_id) {
+            content = `<tool_result>\n<id>${message.tool_call_id}</id>\n<content>${content}</content>\n</tool_result>`;
+        }
+        return `<${message.role ?? ""}>\n${content}\n</${message.role ?? ""}>`;
+    }
+    return messages.map(formatMessage).join("\n\n");
+}
+exports._chatCompletionMessagesToString = _chatCompletionMessagesToString;

package/dist/evaluators/trajectory/utils.d.ts

@@ -0,0 +1,3 @@
+import { ChatCompletionMessage } from "../types.js";
+export declare function _isTrajectorySuperset(outputs: ChatCompletionMessage[], referenceOutputs: ChatCompletionMessage[]): boolean;
+export declare function _chatCompletionMessagesToString(messages: ChatCompletionMessage[]): string;

package/dist/evaluators/trajectory/utils.js

@@ -0,0 +1,63 @@
+function _normalizeToolCall(toolCall) {
+    if ("function" in toolCall &&
+        toolCall.function != null &&
+        typeof toolCall.function === "object") {
+        return {
+            name: toolCall.function.name,
+            args: toolCall.function.arguments,
+        };
+    }
+    return toolCall;
+}
+function _extractToolCalls(messages) {
+    const toolCalls = [];
+    for (const message of messages) {
+        if (message.tool_calls) {
+            toolCalls.push(...message.tool_calls.map(_normalizeToolCall));
+        }
+    }
+    return toolCalls;
+}
+export function _isTrajectorySuperset(outputs, referenceOutputs) {
+    const outputToolCalls = _extractToolCalls(outputs);
+    const referenceToolCalls = _extractToolCalls(referenceOutputs);
+    const outputToolCounts = new Map();
+    const referenceToolCounts = new Map();
+    for (const call of outputToolCalls) {
+        outputToolCounts.set(call.name, (outputToolCounts.get(call.name) ?? 0) + 1);
+    }
+    for (const call of referenceToolCalls) {
+        referenceToolCounts.set(call.name, (referenceToolCounts.get(call.name) ?? 0) + 1);
+    }
+    const allTools = new Set([
+        ...outputToolCounts.keys(),
+        ...referenceToolCounts.keys(),
+    ]);
+    for (const name of allTools) {
+        if ((outputToolCounts.get(name) ?? 0) < (referenceToolCounts.get(name) ?? 0)) {
+            return false;
+        }
+    }
+    return true;
+}
+export function _chatCompletionMessagesToString(messages) {
+    function formatMessage(message) {
+        let content = message.content ?? "";
+        // Handle tool/function calls
+        if (message.tool_calls) {
+            const toolCallsStr = message.tool_calls
+                .map((call) => {
+                const func = call.function ?? {};
+                return `<tool_call>\n<name>${func.name ?? ""}</name>\n<arguments>${func.arguments ?? ""}</arguments>\n</tool_call>`;
+            })
+                .join("\n");
+            content = content ? `${content}\n${toolCallsStr}` : toolCallsStr;
+        }
+        // Handle tool call results
+        if (message.tool_call_id) {
+            content = `<tool_result>\n<id>${message.tool_call_id}</id>\n<content>${content}</content>\n</tool_result>`;
+        }
+        return `<${message.role ?? ""}>\n${content}\n</${message.role ?? ""}>`;
+    }
+    return messages.map(formatMessage).join("\n\n");
+}

package/dist/evaluators/types.cjs

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/evaluators/types.d.ts

@@ -0,0 +1,44 @@
+export type EvaluatorResult = {
+    key: string;
+    score: number | boolean;
+    comment?: string;
+};
+export type SimpleEvaluator = (params: {
+    inputs?: unknown;
+    outputs: unknown;
+    reference_outputs?: unknown;
+    [key: string]: unknown;
+}) => Promise<EvaluatorResult | EvaluatorResult[]> | EvaluatorResult | EvaluatorResult[];
+export type SingleResultScorerReturnType = boolean | number | [boolean | number, string];
+export type MultiResultScorerReturnType = {
+    [key: string]: boolean | number | {
+        score: boolean | number;
+        reasoning?: string;
+    };
+};
+export type ChatCompletionMessage = {
+    content: any;
+    role: string;
+    tool_calls?: Record<string, any>[];
+    tool_call_id?: string;
+};
+export type ChatCompletion = {
+    choices: {
+        message: ChatCompletionMessage;
+    }[];
+};
+export type FewShotExample = {
+    inputs: unknown;
+    outputs: unknown;
+    score: number | boolean;
+    reasoning?: string;
+};
+export interface ChatCompletionsClient {
+    create(params: Record<string, any>): Promise<ChatCompletion>;
+}
+export interface ModelChatClient {
+    completions: ChatCompletionsClient;
+}
+export interface ModelClient {
+    chat: ModelChatClient;
+}

package/dist/evaluators/types.js

@@ -0,0 +1 @@
+export {};

package/dist/evaluators/utils.cjs

@@ -0,0 +1,85 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports._runEvaluator = exports.processScore = exports._normalizeToOpenAIMessagesList = exports._convertToOpenAIMessage = void 0;
+const messages_1 = require("@langchain/core/messages");
+const openai_1 = require("@langchain/openai");
+const jestlike_1 = require("langsmith/utils/jestlike");
+const _convertToOpenAIMessage = (message) => {
+    if ((0, messages_1.isBaseMessage)(message)) {
+        return (0, openai_1._convertMessagesToOpenAIParams)([message])[0];
+    }
+    else {
+        return message;
+    }
+};
+exports._convertToOpenAIMessage = _convertToOpenAIMessage;
+const _normalizeToOpenAIMessagesList = (messages) => {
+    let messagesList;
+    if (!Array.isArray(messages)) {
+        if ("messages" in messages && Array.isArray(messages.messages)) {
+            messagesList = messages.messages;
+        }
+        else {
+            throw new Error(`If passing messages as an object, it must contain a "messages" key`);
+        }
+    }
+    else {
+        messagesList = messages;
+    }
+    return messagesList.map(exports._convertToOpenAIMessage);
+};
+exports._normalizeToOpenAIMessagesList = _normalizeToOpenAIMessagesList;
+const processScore = (_, value) => {
+    if (typeof value === "object") {
+        if (value != null && "score" in value) {
+            return [
+                value.score,
+                "reasoning" in value && typeof value.reasoning === "string"
+                    ? value.reasoning
+                    : undefined,
+            ];
+        }
+        else {
+            throw new Error(`Expected a dictionary with a "score" key, but got "${JSON.stringify(value, null, 2)}"`);
+        }
+    }
+    return [value];
+};
+exports.processScore = processScore;
+const _runEvaluator = async (runName, scorer, feedbackKey, extra) => {
+    const runScorer = async (params) => {
+        let score = await scorer(params);
+        let reasoning;
+        const results = [];
+        if (!Array.isArray(score) && typeof score === "object") {
+            for (const [key, value] of Object.entries(score)) {
+                const [keyScore, reasoning] = (0, exports.processScore)(key, value);
+                results.push({ key, score: keyScore, comment: reasoning });
+            }
+        }
+        else {
+            if (Array.isArray(score)) {
+                reasoning = score[1];
+                score = score[0];
+            }
+            results.push({ key: feedbackKey, score, comment: reasoning });
+        }
+        if (results.length === 1) {
+            return results[0];
+        }
+        else {
+            return results;
+        }
+    };
+    if ((0, jestlike_1.isInTestContext)()) {
+        const res = await (0, jestlike_1.wrapEvaluator)(runScorer)(extra ?? {}, {
+            name: runName,
+        });
+        return res;
+    }
+    else {
+        const res = await runScorer(extra ?? {});
+        return res;
+    }
+};
+exports._runEvaluator = _runEvaluator;

package/dist/evaluators/utils.d.ts

@@ -0,0 +1,13 @@
+import { BaseMessage } from "@langchain/core/messages";
+import { SimpleEvaluationResult } from "langsmith/utils/jestlike";
+import { ChatCompletionMessage, MultiResultScorerReturnType, SingleResultScorerReturnType } from "./types.js";
+export declare const _convertToOpenAIMessage: (message: BaseMessage | ChatCompletionMessage) => ChatCompletionMessage;
+export declare const _normalizeToOpenAIMessagesList: (messages: (ChatCompletionMessage | BaseMessage)[] | {
+    messages: (BaseMessage | ChatCompletionMessage)[];
+}) => ChatCompletionMessage[];
+export declare const processScore: (_: string, value: boolean | number | {
+    score: boolean | number;
+    reasoning?: string;
+}) => readonly [number | boolean, string | undefined] | readonly [number | boolean];
+export type EvaluationResultType<O> = O extends MultiResultScorerReturnType ? SimpleEvaluationResult[] : SimpleEvaluationResult;
+export declare const _runEvaluator: <T extends Record<string, unknown>, O extends SingleResultScorerReturnType | MultiResultScorerReturnType | Promise<SingleResultScorerReturnType | MultiResultScorerReturnType>>(runName: string, scorer: (params: T) => O, feedbackKey: string, extra?: T | undefined) => Promise<EvaluationResultType<O>>;

package/dist/evaluators/utils.js

@@ -0,0 +1,78 @@
+import { isBaseMessage } from "@langchain/core/messages";
+import { _convertMessagesToOpenAIParams } from "@langchain/openai";
+import { wrapEvaluator, isInTestContext, } from "langsmith/utils/jestlike";
+export const _convertToOpenAIMessage = (message) => {
+    if (isBaseMessage(message)) {
+        return _convertMessagesToOpenAIParams([message])[0];
+    }
+    else {
+        return message;
+    }
+};
+export const _normalizeToOpenAIMessagesList = (messages) => {
+    let messagesList;
+    if (!Array.isArray(messages)) {
+        if ("messages" in messages && Array.isArray(messages.messages)) {
+            messagesList = messages.messages;
+        }
+        else {
+            throw new Error(`If passing messages as an object, it must contain a "messages" key`);
+        }
+    }
+    else {
+        messagesList = messages;
+    }
+    return messagesList.map(_convertToOpenAIMessage);
+};
+export const processScore = (_, value) => {
+    if (typeof value === "object") {
+        if (value != null && "score" in value) {
+            return [
+                value.score,
+                "reasoning" in value && typeof value.reasoning === "string"
+                    ? value.reasoning
+                    : undefined,
+            ];
+        }
+        else {
+            throw new Error(`Expected a dictionary with a "score" key, but got "${JSON.stringify(value, null, 2)}"`);
+        }
+    }
+    return [value];
+};
+export const _runEvaluator = async (runName, scorer, feedbackKey, extra) => {
+    const runScorer = async (params) => {
+        let score = await scorer(params);
+        let reasoning;
+        const results = [];
+        if (!Array.isArray(score) && typeof score === "object") {
+            for (const [key, value] of Object.entries(score)) {
+                const [keyScore, reasoning] = processScore(key, value);
+                results.push({ key, score: keyScore, comment: reasoning });
+            }
+        }
+        else {
+            if (Array.isArray(score)) {
+                reasoning = score[1];
+                score = score[0];
+            }
+            results.push({ key: feedbackKey, score, comment: reasoning });
+        }
+        if (results.length === 1) {
+            return results[0];
+        }
+        else {
+            return results;
+        }
+    };
+    if (isInTestContext()) {
+        const res = await wrapEvaluator(runScorer)(extra ?? {}, {
+            name: runName,
+        });
+        return res;
+    }
+    else {
+        const res = await runScorer(extra ?? {});
+        return res;
+    }
+};

package/dist/index.cjs

@@ -0,0 +1,43 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CONCISENESS_PROMPT = exports.CORRECTNESS_PROMPT = exports.HALLUCINATION_PROMPT = exports.DEFAULT_PROMPT = exports.createTrajectoryLLMAsJudge = exports.trajectoryUnorderedMatch = exports.trajectorySuperset = exports.trajectorySubset = exports.trajectoryStrictMatch = exports.createLLMAsJudge = exports.levenshteinDistance = exports.createEmbeddingSimilarityEvaluator = exports.exactMatch = void 0;
+var exact_js_1 = require("./evaluators/exact.cjs");
+Object.defineProperty(exports, "exactMatch", { enumerable: true, get: function () { return exact_js_1.exactMatch; } });
+var embedding_similarity_js_1 = require("./evaluators/string/embedding_similarity.cjs");
+Object.defineProperty(exports, "createEmbeddingSimilarityEvaluator", { enumerable: true, get: function () { return embedding_similarity_js_1.createEmbeddingSimilarityEvaluator; } });
+var levenshtein_js_1 = require("./evaluators/string/levenshtein.cjs");
+Object.defineProperty(exports, "levenshteinDistance", { enumerable: true, get: function () { return levenshtein_js_1.levenshteinDistance; } });
+var llm_js_1 = require("./evaluators/llm.cjs");
+Object.defineProperty(exports, "createLLMAsJudge", { enumerable: true, get: function () { return llm_js_1.createLLMAsJudge; } });
+var strict_js_1 = require("./evaluators/trajectory/strict.cjs");
+Object.defineProperty(exports, "trajectoryStrictMatch", { enumerable: true, get: function () { return strict_js_1.trajectoryStrictMatch; } });
+var subset_js_1 = require("./evaluators/trajectory/subset.cjs");
+Object.defineProperty(exports, "trajectorySubset", { enumerable: true, get: function () { return subset_js_1.trajectorySubset; } });
+var superset_js_1 = require("./evaluators/trajectory/superset.cjs");
+Object.defineProperty(exports, "trajectorySuperset", { enumerable: true, get: function () { return superset_js_1.trajectorySuperset; } });
+var unordered_js_1 = require("./evaluators/trajectory/unordered.cjs");
+Object.defineProperty(exports, "trajectoryUnorderedMatch", { enumerable: true, get: function () { return unordered_js_1.trajectoryUnorderedMatch; } });
+var llm_js_2 = require("./evaluators/trajectory/llm.cjs");
+Object.defineProperty(exports, "createTrajectoryLLMAsJudge", { enumerable: true, get: function () { return llm_js_2.createTrajectoryLLMAsJudge; } });
+Object.defineProperty(exports, "DEFAULT_PROMPT", { enumerable: true, get: function () { return llm_js_2.DEFAULT_PROMPT; } });
+var hallucination_js_1 = require("./evaluators/prompts/hallucination.cjs");
+Object.defineProperty(exports, "HALLUCINATION_PROMPT", { enumerable: true, get: function () { return hallucination_js_1.HALLUCINATION_PROMPT; } });
+var correctness_js_1 = require("./evaluators/prompts/correctness.cjs");
+Object.defineProperty(exports, "CORRECTNESS_PROMPT", { enumerable: true, get: function () { return correctness_js_1.CORRECTNESS_PROMPT; } });
+var conciseness_js_1 = require("./evaluators/prompts/conciseness.cjs");
+Object.defineProperty(exports, "CONCISENESS_PROMPT", { enumerable: true, get: function () { return conciseness_js_1.CONCISENESS_PROMPT; } });
+__exportStar(require("./evaluators/types.cjs"), exports);

package/dist/index.d.ts

@@ -0,0 +1,13 @@
+export { exactMatch } from "./evaluators/exact.js";
+export { createEmbeddingSimilarityEvaluator } from "./evaluators/string/embedding_similarity.js";
+export { levenshteinDistance } from "./evaluators/string/levenshtein.js";
+export { createLLMAsJudge } from "./evaluators/llm.js";
+export { trajectoryStrictMatch } from "./evaluators/trajectory/strict.js";
+export { trajectorySubset } from "./evaluators/trajectory/subset.js";
+export { trajectorySuperset } from "./evaluators/trajectory/superset.js";
+export { trajectoryUnorderedMatch } from "./evaluators/trajectory/unordered.js";
+export { createTrajectoryLLMAsJudge, DEFAULT_PROMPT, } from "./evaluators/trajectory/llm.js";
+export { HALLUCINATION_PROMPT } from "./evaluators/prompts/hallucination.js";
+export { CORRECTNESS_PROMPT } from "./evaluators/prompts/correctness.js";
+export { CONCISENESS_PROMPT } from "./evaluators/prompts/conciseness.js";
+export * from "./evaluators/types.js";

package/dist/index.js

@@ -0,0 +1,13 @@
+export { exactMatch } from "./evaluators/exact.js";
+export { createEmbeddingSimilarityEvaluator } from "./evaluators/string/embedding_similarity.js";
+export { levenshteinDistance } from "./evaluators/string/levenshtein.js";
+export { createLLMAsJudge } from "./evaluators/llm.js";
+export { trajectoryStrictMatch } from "./evaluators/trajectory/strict.js";
+export { trajectorySubset } from "./evaluators/trajectory/subset.js";
+export { trajectorySuperset } from "./evaluators/trajectory/superset.js";
+export { trajectoryUnorderedMatch } from "./evaluators/trajectory/unordered.js";
+export { createTrajectoryLLMAsJudge, DEFAULT_PROMPT, } from "./evaluators/trajectory/llm.js";
+export { HALLUCINATION_PROMPT } from "./evaluators/prompts/hallucination.js";
+export { CORRECTNESS_PROMPT } from "./evaluators/prompts/correctness.js";
+export { CONCISENESS_PROMPT } from "./evaluators/prompts/conciseness.js";
+export * from "./evaluators/types.js";

package/index.cjs

@@ -0,0 +1 @@
+module.exports = require('./dist/index.cjs');

package/index.d.cts

@@ -0,0 +1 @@
+export * from './dist/index.js'

package/index.d.ts

@@ -0,0 +1 @@
+export * from './dist/index.js'

package/index.js

@@ -0,0 +1 @@
+export * from './dist/index.js'

package/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "agentevals",
+  "version": "0.0.0",
+  "packageManager": "yarn@4.3.1",
+  "type": "module",
+  "scripts": {
+    "build": "yarn lc_build --create-entrypoints --pre --tree-shaking",
+    "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/",
+    "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts",
+    "lint": "yarn lint:eslint && yarn lint:dpdm",
+    "lint:fix": "yarn lint:eslint --fix && yarn lint:dpdm",
+    "format": "prettier --config .prettierrc --write \"src\"",
+    "format:check": "prettier --config .prettierrc --check \"src\"",
+    "test": "vitest run"
+  },
+  "dependencies": {
+    "@langchain/openai": "^0.4.4",
+    "langchain": "^0.3.18",
+    "langsmith": "^0.3.10-rc.1"
+  },
+  "peerDependencies": {
+    "@langchain/core": "^0.3.40"
+  },
+  "devDependencies": {
+    "@langchain/core": "^0.3.40",
+    "@langchain/scripts": "0.1.3",
+    "@tsconfig/recommended": "^1.0.8",
+    "dotenv": "^16.4.7",
+    "eslint": "^8.33.0",
+    "eslint-config-airbnb-base": "^15.0.0",
+    "eslint-config-prettier": "^8.6.0",
+    "eslint-plugin-import": "^2.27.5",
+    "eslint-plugin-jest": "^27.6.0",
+    "eslint-plugin-no-instanceof": "^1.0.1",
+    "eslint-plugin-prettier": "^4.2.1",
+    "openai": "^4.85.1",
+    "prettier": "^3.5.1",
+    "typescript": "~5.1.6",
+    "vitest": "^3.0.5"
+  },
+  "files": [
+    "dist/",
+    "index.cjs",
+    "index.js",
+    "index.d.ts",
+    "index.d.cts"
+  ],
+  "exports": {
+    ".": {
+      "types": {
+        "import": "./index.d.ts",
+        "require": "./index.d.cts",
+        "default": "./index.d.ts"
+      },
+      "import": "./index.js",
+      "require": "./index.cjs"
+    },
+    "./package.json": "./package.json"
+  }
+}