langsmith 0.2.15-beta.0 → 0.2.15-rc.0

package/dist/index.cjs

@@ -8,4 +8,4 @@ Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () {
 var fetch_js_1 = require("./singletons/fetch.cjs");
 Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true, get: function () { return fetch_js_1.overrideFetchImplementation; } });
 // Update using yarn bump-version
-exports.__version__ = "0.2.15-beta.0";
+exports.__version__ = "0.2.15-rc.0";

package/dist/index.d.ts

@@ -2,4 +2,4 @@ export { Client, type ClientConfig, type LangSmithTracingClientInterface, } from
 export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
-export declare const __version__ = "0.2.15-beta.0";
+export declare const __version__ = "0.2.15-rc.0";

package/dist/index.js

@@ -2,4 +2,4 @@ export { Client, } from "./client.js";
 export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 // Update using yarn bump-version
-export const __version__ = "0.2.15-beta.0";
+export const __version__ = "0.2.15-rc.0";

package/dist/jest/globals.cjs

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.trackingEnabled = exports.jestAsyncLocalStorageInstance = void 0;
+const node_async_hooks_1 = require("node:async_hooks");
+const env_js_1 = require("../utils/env.cjs");
+exports.jestAsyncLocalStorageInstance = new node_async_hooks_1.AsyncLocalStorage();
+function trackingEnabled() {
+    return (0, env_js_1.getEnvironmentVariable)("LANGSMITH_TEST_TRACKING") === "true";
+}
+exports.trackingEnabled = trackingEnabled;

package/dist/jest/globals.d.ts

@@ -0,0 +1,18 @@
+/// <reference types="node" resolution-mode="require"/>
+import { AsyncLocalStorage } from "node:async_hooks";
+import { Dataset, TracerSession, Example } from "../schemas.js";
+import { Client } from "../client.js";
+export declare const jestAsyncLocalStorageInstance: AsyncLocalStorage<{
+    dataset?: Dataset | undefined;
+    examples?: (Example & {
+        inputHash: string;
+        outputHash: string;
+    })[] | undefined;
+    createdAt: string;
+    project?: TracerSession | undefined;
+    currentExample?: Partial<Example> | undefined;
+    client: Client;
+    suiteUuid: string;
+    suiteName: string;
+}>;
+export declare function trackingEnabled(): boolean;

package/dist/jest/globals.js

@@ -0,0 +1,6 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { getEnvironmentVariable } from "../utils/env.js";
+export const jestAsyncLocalStorageInstance = new AsyncLocalStorage();
+export function trackingEnabled() {
+    return getEnvironmentVariable("LANGSMITH_TEST_TRACKING") === "true";
+}

package/dist/jest/index.cjs

@@ -0,0 +1,267 @@
+"use strict";
+/* eslint-disable import/no-extraneous-dependencies */
+/* eslint-disable @typescript-eslint/no-namespace */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.expect = exports.describe = exports.it = exports.test = void 0;
+const globals_1 = require("@jest/globals");
+const crypto_1 = __importDefault(require("crypto"));
+const uuid_1 = require("uuid");
+const traceable_js_1 = require("../traceable.cjs");
+const run_trees_js_1 = require("../run_trees.cjs");
+const _random_name_js_1 = require("../evaluation/_random_name.cjs");
+const matchers_js_1 = require("./matchers.cjs");
+const globals_js_1 = require("./globals.cjs");
+const chain_js_1 = require("./vendor/chain.cjs");
+globals_1.expect.extend({
+    toBeRelativeCloseTo: matchers_js_1.toBeRelativeCloseTo,
+    toBeAbsoluteCloseTo: matchers_js_1.toBeAbsoluteCloseTo,
+    toBeSemanticCloseTo: matchers_js_1.toBeSemanticCloseTo,
+});
+const objectHash = (obj, depth = 0) => {
+    // Prevent infinite recursion
+    if (depth > 50) {
+        throw new Error("Object is too deep to check equality for serialization. Please use a simpler example.");
+    }
+    if (Array.isArray(obj)) {
+        const arrayHash = obj.map((item) => objectHash(item, depth + 1)).join(",");
+        return crypto_1.default.createHash("sha256").update(arrayHash).digest("hex");
+    }
+    if (obj && typeof obj === "object") {
+        const sortedHash = Object.keys(obj)
+            .sort()
+            .map((key) => `${key}:${objectHash(obj[key], depth + 1)}`)
+            .join(",");
+        return crypto_1.default.createHash("sha256").update(sortedHash).digest("hex");
+    }
+    return crypto_1.default.createHash("sha256").update(JSON.stringify(obj)).digest("hex");
+};
+async function _createProject(client, datasetId) {
+    // Create the project, updating the experimentName until we find a unique one.
+    let project;
+    let experimentName = (0, _random_name_js_1.randomName)();
+    for (let i = 0; i < 10; i++) {
+        try {
+            project = await client.createProject({
+                projectName: experimentName,
+                referenceDatasetId: datasetId,
+                // description: this._description,
+            });
+            return project;
+        }
+        catch (e) {
+            // Naming collision
+            if (e?.name === "LangSmithConflictError") {
+                const ent = (0, uuid_1.v4)().slice(0, 6);
+                experimentName = `${experimentName}-${ent}`;
+            }
+            else {
+                throw e;
+            }
+        }
+    }
+    throw new Error("Could not generate a unique experiment name within 10 attempts." +
+        " Please try again.");
+}
+const setupPromises = new Map();
+async function runDatasetSetup(testClient, datasetName) {
+    let storageValue;
+    if (!(0, globals_js_1.trackingEnabled)()) {
+        storageValue = {
+            createdAt: new Date().toISOString(),
+        };
+    }
+    else {
+        let dataset;
+        try {
+            dataset = await testClient.readDataset({
+                datasetName,
+            });
+        }
+        catch (e) {
+            if (e.message.includes("not found")) {
+                dataset = await testClient.createDataset(datasetName, {
+                    description: `Dataset for unit tests created on ${new Date().toISOString()}`,
+                });
+            }
+            else {
+                throw e;
+            }
+        }
+        const examplesList = testClient.listExamples({
+            datasetName,
+        });
+        const examples = [];
+        for await (const example of examplesList) {
+            const inputHash = objectHash(example.inputs);
+            const outputHash = objectHash(example.outputs ?? {});
+            examples.push({ ...example, inputHash, outputHash });
+        }
+        const project = await _createProject(testClient, dataset.id);
+        storageValue = {
+            dataset,
+            examples,
+            project,
+            client: testClient,
+        };
+    }
+    return storageValue;
+}
+function wrapDescribeMethod(method) {
+    return function (datasetName, fn, config) {
+        return method(datasetName, () => {
+            const suiteUuid = (0, uuid_1.v4)();
+            /**
+             * We cannot rely on setting AsyncLocalStorage in beforeAll or beforeEach,
+             * due to https://github.com/jestjs/jest/issues/13653 and needing to use
+             * the janky .enterWith.
+             *
+             * We also cannot do async setup in describe due to Jest restrictions.
+             * However, .run without asynchronous logic works.
+             */
+            void globals_js_1.jestAsyncLocalStorageInstance.run({
+                suiteUuid,
+                suiteName: datasetName,
+                client: config?.client ?? run_trees_js_1.RunTree.getSharedClient(),
+                createdAt: new Date().toISOString(),
+            }, fn);
+        });
+    };
+}
+const lsDescribe = Object.assign(wrapDescribeMethod(globals_1.describe), {
+    only: wrapDescribeMethod(globals_1.describe.only),
+    skip: wrapDescribeMethod(globals_1.describe.skip),
+});
+exports.describe = lsDescribe;
+function wrapTestMethod(method) {
+    return function (params, config) {
+        // Due to https://github.com/jestjs/jest/issues/13653,
+        // we must access the local store value here before
+        // entering an async context
+        const context = globals_js_1.jestAsyncLocalStorageInstance.getStore();
+        // This typing is wrong, but necessary to avoid lint errors
+        // eslint-disable-next-line @typescript-eslint/no-misused-promises
+        return async function (...args) {
+            let createExamplePromise;
+            const totalRuns = config?.n ?? 1;
+            for (let i = 0; i < totalRuns; i += 1) {
+                // Jest will not group under the same "describe" group if you await the test and
+                // total runs is greater than 1
+                void method(`${args[0]} ${i}`, async () => {
+                    if (context === undefined) {
+                        throw new Error(`Could not retrieve test context.\nPlease make sure you have tracing enabled and you are wrapping all of your test cases in an "ls.describe()" function.`);
+                    }
+                    // Because of https://github.com/jestjs/jest/issues/13653, we have to do asynchronous setup
+                    // within the test itself
+                    if (!setupPromises.get(context.suiteUuid)) {
+                        setupPromises.set(context.suiteUuid, runDatasetSetup(context.client, context.suiteName));
+                    }
+                    const { examples, dataset, createdAt, project, client } = await setupPromises.get(context.suiteUuid);
+                    const testInput = params.inputs;
+                    const testOutput = params.outputs;
+                    const inputHash = objectHash(testInput);
+                    const outputHash = objectHash(testOutput ?? {});
+                    if ((0, globals_js_1.trackingEnabled)()) {
+                        const missingFields = [];
+                        if (examples === undefined) {
+                            missingFields.push("examples");
+                        }
+                        if (dataset === undefined) {
+                            missingFields.push("dataset");
+                        }
+                        if (project === undefined) {
+                            missingFields.push("project");
+                        }
+                        if (client === undefined) {
+                            missingFields.push("client");
+                        }
+                        if (missingFields.length > 0) {
+                            throw new Error(`Failed to initialize test tracking: Could not identify ${missingFields
+                                .map((field) => `"${field}"`)
+                                .join(", ")} while syncing to LangSmith. Please contact us for help.`);
+                        }
+                        const testClient = config?.client ?? client;
+                        let example = (examples ?? []).find((example) => {
+                            return (example.inputHash === inputHash &&
+                                example.outputHash === outputHash);
+                        });
+                        if (example === undefined) {
+                            // Avoid creating multiple of the same example
+                            // when running the same test case multiple times
+                            // Jest runs other tests serially
+                            if (createExamplePromise === undefined) {
+                                createExamplePromise = testClient.createExample(testInput, testOutput, {
+                                    datasetId: dataset?.id,
+                                    createdAt: new Date(createdAt ?? new Date()),
+                                });
+                            }
+                            const newExample = await createExamplePromise;
+                            example = { ...newExample, inputHash, outputHash };
+                        }
+                        // What do I do here?
+                        // examples.push(example);
+                        // .enterWith is OK here
+                        globals_js_1.jestAsyncLocalStorageInstance.enterWith({
+                            ...context,
+                            currentExample: example,
+                            client: testClient,
+                        });
+                        const traceableOptions = {
+                            reference_example_id: example.id,
+                            project_name: project.name,
+                            metadata: {
+                                ...config?.metadata,
+                                example_version: example.modified_at
+                                    ? new Date(example.modified_at).toISOString()
+                                    : new Date(example.created_at).toISOString(),
+                            },
+                            client: testClient,
+                            tracingEnabled: true,
+                            name: "Unit test",
+                        };
+                        // Pass inputs into traceable so tracing works correctly but
+                        // provide both to the user-defined test function
+                        const tracedFunction = (0, traceable_js_1.traceable)(async (_) => {
+                            return args[1]({
+                                inputs: testInput,
+                                outputs: testOutput,
+                            });
+                        }, { ...traceableOptions, ...config });
+                        await tracedFunction(testInput);
+                        await testClient.awaitPendingTraceBatches();
+                    }
+                    else {
+                        // .enterWith is OK here
+                        globals_js_1.jestAsyncLocalStorageInstance.enterWith({
+                            ...context,
+                            currentExample: { inputs: testInput, outputs: testOutput },
+                        });
+                        await args[1]({
+                            inputs: testInput,
+                            outputs: testOutput,
+                        });
+                    }
+                }, ...args.slice(2));
+            }
+        };
+    };
+}
+function eachMethod(table) {
+    return function (name, fn, timeout) {
+        for (let i = 0; i < table.length; i += 1) {
+            const example = table[i];
+            wrapTestMethod(globals_1.test)(example)(`${name} ${i}`, fn, timeout);
+        }
+    };
+}
+const lsTest = Object.assign(wrapTestMethod(globals_1.test), {
+    only: wrapTestMethod(globals_1.test.only),
+    skip: wrapTestMethod(globals_1.test.skip),
+    each: eachMethod,
+});
+exports.test = lsTest;
+exports.it = lsTest;
+const wrappedExpect = (0, chain_js_1.wrapExpect)(globals_1.expect);
+exports.expect = wrappedExpect;

package/dist/jest/index.d.ts

@@ -0,0 +1,61 @@
+import { RunTreeConfig } from "../run_trees.js";
+import { KVMap } from "../schemas.js";
+import type { SimpleEvaluator } from "./vendor/gradedBy.js";
+declare global {
+    namespace jest {
+        interface AsymmetricMatchers {
+            toBeRelativeCloseTo(expected: string, options?: any): void;
+            toBeAbsoluteCloseTo(expected: string, options?: any): void;
+            toBeSemanticCloseTo(expected: string, options?: any): Promise<void>;
+        }
+        interface Matchers<R> {
+            toBeRelativeCloseTo(expected: string, options?: any): R;
+            toBeAbsoluteCloseTo(expected: string, options?: any): R;
+            toBeSemanticCloseTo(expected: string, options?: any): Promise<R>;
+            gradedBy(evaluator: SimpleEvaluator): jest.Matchers<Promise<R>> & {
+                not: jest.Matchers<Promise<R>>;
+                resolves: jest.Matchers<Promise<R>>;
+                rejects: jest.Matchers<Promise<R>>;
+            };
+        }
+    }
+}
+export type LangSmithJestDescribeWrapper = (name: string, fn: () => void | Promise<void>, config?: Partial<RunTreeConfig>) => void;
+declare const lsDescribe: LangSmithJestDescribeWrapper & {
+    only: LangSmithJestDescribeWrapper;
+    skip: LangSmithJestDescribeWrapper;
+};
+export type LangSmithJestTestWrapper<I, O> = (name: string, fn: (params: {
+    inputs: I;
+    outputs: O;
+}) => unknown | Promise<unknown>, timeout?: number) => void;
+declare function eachMethod<I extends KVMap, O extends KVMap>(table: {
+    inputs: I;
+    outputs: O;
+}[]): (name: string, fn: (params: {
+    inputs: I;
+    outputs: O;
+}) => unknown | Promise<unknown>, timeout?: number) => void;
+declare const lsTest: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: {
+    inputs: I;
+    outputs: O;
+}, config?: Partial<RunTreeConfig> & {
+    n?: number;
+}) => LangSmithJestTestWrapper<I, O>) & {
+    only: <I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: {
+        inputs: I;
+        outputs: O;
+    }, config?: Partial<RunTreeConfig> & {
+        n?: number;
+    }) => LangSmithJestTestWrapper<I, O>;
+    skip: <I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: {
+        inputs: I;
+        outputs: O;
+    }, config?: Partial<RunTreeConfig> & {
+        n?: number;
+    }) => LangSmithJestTestWrapper<I, O>;
+    each: typeof eachMethod;
+};
+declare const wrappedExpect: any;
+export { lsTest as test, lsTest as it, lsDescribe as describe, wrappedExpect as expect, };
+export { type SimpleEvaluator };

package/dist/jest/index.js

@@ -0,0 +1,258 @@
+/* eslint-disable import/no-extraneous-dependencies */
+/* eslint-disable @typescript-eslint/no-namespace */
+import { expect, test, describe } from "@jest/globals";
+import crypto from "crypto";
+import { v4 } from "uuid";
+import { traceable } from "../traceable.js";
+import { RunTree } from "../run_trees.js";
+import { randomName } from "../evaluation/_random_name.js";
+import { toBeRelativeCloseTo, toBeAbsoluteCloseTo, toBeSemanticCloseTo, } from "./matchers.js";
+import { jestAsyncLocalStorageInstance, trackingEnabled } from "./globals.js";
+import { wrapExpect } from "./vendor/chain.js";
+expect.extend({
+    toBeRelativeCloseTo,
+    toBeAbsoluteCloseTo,
+    toBeSemanticCloseTo,
+});
+const objectHash = (obj, depth = 0) => {
+    // Prevent infinite recursion
+    if (depth > 50) {
+        throw new Error("Object is too deep to check equality for serialization. Please use a simpler example.");
+    }
+    if (Array.isArray(obj)) {
+        const arrayHash = obj.map((item) => objectHash(item, depth + 1)).join(",");
+        return crypto.createHash("sha256").update(arrayHash).digest("hex");
+    }
+    if (obj && typeof obj === "object") {
+        const sortedHash = Object.keys(obj)
+            .sort()
+            .map((key) => `${key}:${objectHash(obj[key], depth + 1)}`)
+            .join(",");
+        return crypto.createHash("sha256").update(sortedHash).digest("hex");
+    }
+    return crypto.createHash("sha256").update(JSON.stringify(obj)).digest("hex");
+};
+async function _createProject(client, datasetId) {
+    // Create the project, updating the experimentName until we find a unique one.
+    let project;
+    let experimentName = randomName();
+    for (let i = 0; i < 10; i++) {
+        try {
+            project = await client.createProject({
+                projectName: experimentName,
+                referenceDatasetId: datasetId,
+                // description: this._description,
+            });
+            return project;
+        }
+        catch (e) {
+            // Naming collision
+            if (e?.name === "LangSmithConflictError") {
+                const ent = v4().slice(0, 6);
+                experimentName = `${experimentName}-${ent}`;
+            }
+            else {
+                throw e;
+            }
+        }
+    }
+    throw new Error("Could not generate a unique experiment name within 10 attempts." +
+        " Please try again.");
+}
+const setupPromises = new Map();
+async function runDatasetSetup(testClient, datasetName) {
+    let storageValue;
+    if (!trackingEnabled()) {
+        storageValue = {
+            createdAt: new Date().toISOString(),
+        };
+    }
+    else {
+        let dataset;
+        try {
+            dataset = await testClient.readDataset({
+                datasetName,
+            });
+        }
+        catch (e) {
+            if (e.message.includes("not found")) {
+                dataset = await testClient.createDataset(datasetName, {
+                    description: `Dataset for unit tests created on ${new Date().toISOString()}`,
+                });
+            }
+            else {
+                throw e;
+            }
+        }
+        const examplesList = testClient.listExamples({
+            datasetName,
+        });
+        const examples = [];
+        for await (const example of examplesList) {
+            const inputHash = objectHash(example.inputs);
+            const outputHash = objectHash(example.outputs ?? {});
+            examples.push({ ...example, inputHash, outputHash });
+        }
+        const project = await _createProject(testClient, dataset.id);
+        storageValue = {
+            dataset,
+            examples,
+            project,
+            client: testClient,
+        };
+    }
+    return storageValue;
+}
+function wrapDescribeMethod(method) {
+    return function (datasetName, fn, config) {
+        return method(datasetName, () => {
+            const suiteUuid = v4();
+            /**
+             * We cannot rely on setting AsyncLocalStorage in beforeAll or beforeEach,
+             * due to https://github.com/jestjs/jest/issues/13653 and needing to use
+             * the janky .enterWith.
+             *
+             * We also cannot do async setup in describe due to Jest restrictions.
+             * However, .run without asynchronous logic works.
+             */
+            void jestAsyncLocalStorageInstance.run({
+                suiteUuid,
+                suiteName: datasetName,
+                client: config?.client ?? RunTree.getSharedClient(),
+                createdAt: new Date().toISOString(),
+            }, fn);
+        });
+    };
+}
+const lsDescribe = Object.assign(wrapDescribeMethod(describe), {
+    only: wrapDescribeMethod(describe.only),
+    skip: wrapDescribeMethod(describe.skip),
+});
+function wrapTestMethod(method) {
+    return function (params, config) {
+        // Due to https://github.com/jestjs/jest/issues/13653,
+        // we must access the local store value here before
+        // entering an async context
+        const context = jestAsyncLocalStorageInstance.getStore();
+        // This typing is wrong, but necessary to avoid lint errors
+        // eslint-disable-next-line @typescript-eslint/no-misused-promises
+        return async function (...args) {
+            let createExamplePromise;
+            const totalRuns = config?.n ?? 1;
+            for (let i = 0; i < totalRuns; i += 1) {
+                // Jest will not group under the same "describe" group if you await the test and
+                // total runs is greater than 1
+                void method(`${args[0]} ${i}`, async () => {
+                    if (context === undefined) {
+                        throw new Error(`Could not retrieve test context.\nPlease make sure you have tracing enabled and you are wrapping all of your test cases in an "ls.describe()" function.`);
+                    }
+                    // Because of https://github.com/jestjs/jest/issues/13653, we have to do asynchronous setup
+                    // within the test itself
+                    if (!setupPromises.get(context.suiteUuid)) {
+                        setupPromises.set(context.suiteUuid, runDatasetSetup(context.client, context.suiteName));
+                    }
+                    const { examples, dataset, createdAt, project, client } = await setupPromises.get(context.suiteUuid);
+                    const testInput = params.inputs;
+                    const testOutput = params.outputs;
+                    const inputHash = objectHash(testInput);
+                    const outputHash = objectHash(testOutput ?? {});
+                    if (trackingEnabled()) {
+                        const missingFields = [];
+                        if (examples === undefined) {
+                            missingFields.push("examples");
+                        }
+                        if (dataset === undefined) {
+                            missingFields.push("dataset");
+                        }
+                        if (project === undefined) {
+                            missingFields.push("project");
+                        }
+                        if (client === undefined) {
+                            missingFields.push("client");
+                        }
+                        if (missingFields.length > 0) {
+                            throw new Error(`Failed to initialize test tracking: Could not identify ${missingFields
+                                .map((field) => `"${field}"`)
+                                .join(", ")} while syncing to LangSmith. Please contact us for help.`);
+                        }
+                        const testClient = config?.client ?? client;
+                        let example = (examples ?? []).find((example) => {
+                            return (example.inputHash === inputHash &&
+                                example.outputHash === outputHash);
+                        });
+                        if (example === undefined) {
+                            // Avoid creating multiple of the same example
+                            // when running the same test case multiple times
+                            // Jest runs other tests serially
+                            if (createExamplePromise === undefined) {
+                                createExamplePromise = testClient.createExample(testInput, testOutput, {
+                                    datasetId: dataset?.id,
+                                    createdAt: new Date(createdAt ?? new Date()),
+                                });
+                            }
+                            const newExample = await createExamplePromise;
+                            example = { ...newExample, inputHash, outputHash };
+                        }
+                        // What do I do here?
+                        // examples.push(example);
+                        // .enterWith is OK here
+                        jestAsyncLocalStorageInstance.enterWith({
+                            ...context,
+                            currentExample: example,
+                            client: testClient,
+                        });
+                        const traceableOptions = {
+                            reference_example_id: example.id,
+                            project_name: project.name,
+                            metadata: {
+                                ...config?.metadata,
+                                example_version: example.modified_at
+                                    ? new Date(example.modified_at).toISOString()
+                                    : new Date(example.created_at).toISOString(),
+                            },
+                            client: testClient,
+                            tracingEnabled: true,
+                            name: "Unit test",
+                        };
+                        // Pass inputs into traceable so tracing works correctly but
+                        // provide both to the user-defined test function
+                        const tracedFunction = traceable(async (_) => {
+                            return args[1]({
+                                inputs: testInput,
+                                outputs: testOutput,
+                            });
+                        }, { ...traceableOptions, ...config });
+                        await tracedFunction(testInput);
+                        await testClient.awaitPendingTraceBatches();
+                    }
+                    else {
+                        // .enterWith is OK here
+                        jestAsyncLocalStorageInstance.enterWith({
+                            ...context,
+                            currentExample: { inputs: testInput, outputs: testOutput },
+                        });
+                        await args[1]({
+                            inputs: testInput,
+                            outputs: testOutput,
+                        });
+                    }
+                }, ...args.slice(2));
+            }
+        };
+    };
+}
+function eachMethod(table) {
+    return function (name, fn, timeout) {
+        for (let i = 0; i < table.length; i += 1) {
+            const example = table[i];
+            wrapTestMethod(test)(example)(`${name} ${i}`, fn, timeout);
+        }
+    };
+}
+const lsTest = Object.assign(wrapTestMethod(test), {
+    only: wrapTestMethod(test.only),
+    skip: wrapTestMethod(test.skip),
+    each: eachMethod,
+});
+const wrappedExpect = wrapExpect(expect);
+export { lsTest as test, lsTest as it, lsDescribe as describe, wrappedExpect as expect, };

package/dist/jest/matchers.cjs

@@ -0,0 +1,101 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.toBeSemanticCloseTo = exports.toBeAbsoluteCloseTo = exports.toBeRelativeCloseTo = void 0;
+// Levenshtein distance implementation
+function levenshteinDistance(a, b) {
+    if (a.length === 0)
+        return b.length;
+    if (b.length === 0)
+        return a.length;
+    const matrix = Array(b.length + 1)
+        .fill(null)
+        .map(() => Array(a.length + 1).fill(null));
+    for (let i = 0; i <= a.length; i++)
+        matrix[0][i] = i;
+    for (let j = 0; j <= b.length; j++)
+        matrix[j][0] = j;
+    for (let j = 1; j <= b.length; j++) {
+        for (let i = 1; i <= a.length; i++) {
+            const substitutionCost = a[i - 1] === b[j - 1] ? 0 : 1;
+            matrix[j][i] = Math.min(matrix[j][i - 1] + 1, matrix[j - 1][i] + 1, matrix[j - 1][i - 1] + substitutionCost);
+        }
+    }
+    return matrix[b.length][a.length];
+}
+async function toBeRelativeCloseTo(received, expected, options = {}) {
+    const { threshold = 0.1, algorithm = "levenshtein" } = options;
+    let distance;
+    let maxLength;
+    switch (algorithm) {
+        case "levenshtein":
+            distance = levenshteinDistance(received, expected);
+            maxLength = Math.max(received.length, expected.length);
+            break;
+        default:
+            throw new Error(`Unsupported algorithm: ${algorithm}`);
+    }
+    // Calculate relative distance (normalized between 0 and 1)
+    const relativeDistance = maxLength === 0 ? 0 : distance / maxLength;
+    const pass = relativeDistance <= threshold;
+    return {
+        pass,
+        message: () => pass
+            ? `Expected "${received}" not to be relatively close to "${expected}" (threshold: ${threshold}, actual distance: ${relativeDistance})`
+            : `Expected "${received}" to be relatively close to "${expected}" (threshold: ${threshold}, actual distance: ${relativeDistance})`,
+    };
+}
+exports.toBeRelativeCloseTo = toBeRelativeCloseTo;
+async function toBeAbsoluteCloseTo(received, expected, options = {}) {
+    const { threshold = 3, algorithm = "levenshtein" } = options;
+    let distance;
+    switch (algorithm) {
+        case "levenshtein":
+            distance = levenshteinDistance(received, expected);
+            break;
+        default:
+            throw new Error(`Unsupported algorithm: ${algorithm}`);
+    }
+    const pass = distance <= threshold;
+    return {
+        pass,
+        message: () => pass
+            ? `Expected "${received}" not to be absolutely close to "${expected}" (threshold: ${threshold}, actual distance: ${distance})`
+            : `Expected "${received}" to be absolutely close to "${expected}" (threshold: ${threshold}, actual distance: ${distance})`,
+    };
+}
+exports.toBeAbsoluteCloseTo = toBeAbsoluteCloseTo;
+async function toBeSemanticCloseTo(received, expected, options) {
+    const { threshold = 0.2, embedding, algorithm = "cosine" } = options;
+    // Get embeddings for both strings
+    const [receivedEmbedding, expectedEmbedding] = await Promise.all([
+        embedding.embedQuery(received),
+        embedding.embedQuery(expected),
+    ]);
+    // Calculate similarity based on chosen algorithm
+    let similarity;
+    switch (algorithm) {
+        case "cosine": {
+            // Compute cosine similarity
+            const dotProduct = receivedEmbedding.reduce((sum, a, i) => sum + a * expectedEmbedding[i], 0);
+            const receivedMagnitude = Math.sqrt(receivedEmbedding.reduce((sum, a) => sum + a * a, 0));
+            const expectedMagnitude = Math.sqrt(expectedEmbedding.reduce((sum, a) => sum + a * a, 0));
+            similarity = dotProduct / (receivedMagnitude * expectedMagnitude);
+            break;
+        }
+        case "dot-product": {
+            // Compute dot product similarity
+            similarity = receivedEmbedding.reduce((sum, a, i) => sum + a * expectedEmbedding[i], 0);
+            break;
+        }
+        default:
+            throw new Error(`Unsupported algorithm: ${algorithm}`);
+    }
+    const pass = similarity >= 1 - threshold;
+    return {
+        pass,
+        message: () => pass
+            ? `Expected "${received}" not to be semantically close to "${expected}" (threshold: ${threshold}, similarity: ${similarity})`
+            : `Expected "${received}" to be semantically close to "${expected}" (threshold: ${threshold}, similarity: ${similarity})`,
+    };
+}
+exports.toBeSemanticCloseTo = toBeSemanticCloseTo;

package/dist/jest/matchers.d.ts

@@ -0,0 +1,25 @@
+import type { MatcherContext } from "expect";
+export declare function toBeRelativeCloseTo(this: MatcherContext, received: string, expected: string, options?: {
+    threshold?: number;
+    algorithm?: "levenshtein";
+}): Promise<{
+    pass: boolean;
+    message: () => string;
+}>;
+export declare function toBeAbsoluteCloseTo(this: MatcherContext, received: string, expected: string, options?: {
+    threshold?: number;
+    algorithm?: "levenshtein";
+}): Promise<{
+    pass: boolean;
+    message: () => string;
+}>;
+export declare function toBeSemanticCloseTo(this: MatcherContext, received: string, expected: string, options: {
+    threshold?: number;
+    embedding: {
+        embedQuery: (query: string) => number[] | Promise<number[]>;
+    };
+    algorithm?: "cosine" | "dot-product";
+}): Promise<{
+    pass: boolean;
+    message: () => string;
+}>;

package/dist/jest/matchers.js

@@ -0,0 +1,95 @@
+// Levenshtein distance implementation
+function levenshteinDistance(a, b) {
+    if (a.length === 0)
+        return b.length;
+    if (b.length === 0)
+        return a.length;
+    const matrix = Array(b.length + 1)
+        .fill(null)
+        .map(() => Array(a.length + 1).fill(null));
+    for (let i = 0; i <= a.length; i++)
+        matrix[0][i] = i;
+    for (let j = 0; j <= b.length; j++)
+        matrix[j][0] = j;
+    for (let j = 1; j <= b.length; j++) {
+        for (let i = 1; i <= a.length; i++) {
+            const substitutionCost = a[i - 1] === b[j - 1] ? 0 : 1;
+            matrix[j][i] = Math.min(matrix[j][i - 1] + 1, matrix[j - 1][i] + 1, matrix[j - 1][i - 1] + substitutionCost);
+        }
+    }
+    return matrix[b.length][a.length];
+}
+export async function toBeRelativeCloseTo(received, expected, options = {}) {
+    const { threshold = 0.1, algorithm = "levenshtein" } = options;
+    let distance;
+    let maxLength;
+    switch (algorithm) {
+        case "levenshtein":
+            distance = levenshteinDistance(received, expected);
+            maxLength = Math.max(received.length, expected.length);
+            break;
+        default:
+            throw new Error(`Unsupported algorithm: ${algorithm}`);
+    }
+    // Calculate relative distance (normalized between 0 and 1)
+    const relativeDistance = maxLength === 0 ? 0 : distance / maxLength;
+    const pass = relativeDistance <= threshold;
+    return {
+        pass,
+        message: () => pass
+            ? `Expected "${received}" not to be relatively close to "${expected}" (threshold: ${threshold}, actual distance: ${relativeDistance})`
+            : `Expected "${received}" to be relatively close to "${expected}" (threshold: ${threshold}, actual distance: ${relativeDistance})`,
+    };
+}
+export async function toBeAbsoluteCloseTo(received, expected, options = {}) {
+    const { threshold = 3, algorithm = "levenshtein" } = options;
+    let distance;
+    switch (algorithm) {
+        case "levenshtein":
+            distance = levenshteinDistance(received, expected);
+            break;
+        default:
+            throw new Error(`Unsupported algorithm: ${algorithm}`);
+    }
+    const pass = distance <= threshold;
+    return {
+        pass,
+        message: () => pass
+            ? `Expected "${received}" not to be absolutely close to "${expected}" (threshold: ${threshold}, actual distance: ${distance})`
+            : `Expected "${received}" to be absolutely close to "${expected}" (threshold: ${threshold}, actual distance: ${distance})`,
+    };
+}
+export async function toBeSemanticCloseTo(received, expected, options) {
+    const { threshold = 0.2, embedding, algorithm = "cosine" } = options;
+    // Get embeddings for both strings
+    const [receivedEmbedding, expectedEmbedding] = await Promise.all([
+        embedding.embedQuery(received),
+        embedding.embedQuery(expected),
+    ]);
+    // Calculate similarity based on chosen algorithm
+    let similarity;
+    switch (algorithm) {
+        case "cosine": {
+            // Compute cosine similarity
+            const dotProduct = receivedEmbedding.reduce((sum, a, i) => sum + a * expectedEmbedding[i], 0);
+            const receivedMagnitude = Math.sqrt(receivedEmbedding.reduce((sum, a) => sum + a * a, 0));
+            const expectedMagnitude = Math.sqrt(expectedEmbedding.reduce((sum, a) => sum + a * a, 0));
+            similarity = dotProduct / (receivedMagnitude * expectedMagnitude);
+            break;
+        }
+        case "dot-product": {
+            // Compute dot product similarity
+            similarity = receivedEmbedding.reduce((sum, a, i) => sum + a * expectedEmbedding[i], 0);
+            break;
+        }
+        default:
+            throw new Error(`Unsupported algorithm: ${algorithm}`);
+    }
+    const pass = similarity >= 1 - threshold;
+    return {
+        pass,
+        message: () => pass
+            ? `Expected "${received}" not to be semantically close to "${expected}" (threshold: ${threshold}, similarity: ${similarity})`
+            : `Expected "${received}" to be semantically close to "${expected}" (threshold: ${threshold}, similarity: ${similarity})`,
+    };
+}

package/dist/jest/vendor/chain.cjs

@@ -0,0 +1,92 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.wrapExpect = void 0;
+/**
+ * Adapted from https://github.com/mattphillips/jest-chain/blob/main/src/chain.js
+ */
+const gradedBy_js_1 = require("./gradedBy.cjs");
+class JestAssertionError extends Error {
+    constructor(result, callsite) {
+        super(typeof result.message === "function" ? result.message() : result.message);
+        Object.defineProperty(this, "matcherResult", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.matcherResult = result;
+        if (Error.captureStackTrace) {
+            Error.captureStackTrace(this, callsite);
+        }
+    }
+}
+const _wrapMatchers = (matchers, evaluator, originalArgs, originalExpect, staticPath = []) => {
+    return Object.keys(matchers)
+        .filter((name) => typeof matchers[name] === "function")
+        .map((name) => {
+        const newMatcher = async (...args) => {
+            try {
+                const score = await (0, gradedBy_js_1.gradedBy)(originalArgs[0], evaluator);
+                let result = originalExpect(score);
+                for (const pathEntry of staticPath) {
+                    result = result[pathEntry];
+                }
+                result = result[name](...args); // run matcher up to current state
+                if (result && typeof result.then === "function") {
+                    return Object.assign(Promise.resolve(result), matchers);
+                }
+                else {
+                    return matchers;
+                }
+            }
+            catch (error) {
+                if (!error.matcherResult) {
+                    throw error;
+                }
+                else {
+                    throw new JestAssertionError(error.matcherResult, newMatcher);
+                }
+            }
+        };
+        return { [name]: newMatcher };
+    });
+};
+const addGradedBy = (matchers, originalArgs, originalExpect, staticPath = []) => {
+    let spreadMatchers = { ...matchers };
+    // Handle Bun, which uses a class
+    if (Object.keys(matchers).length === 0) {
+        const prototypeProps = Object.getOwnPropertyNames(Object.getPrototypeOf(matchers));
+        spreadMatchers = Object.fromEntries(prototypeProps.map((prop) => {
+            try {
+                return [prop, matchers[prop]];
+            }
+            catch (e) {
+                // Ignore bizarre Bun bug
+                return [];
+            }
+        }));
+    }
+    return Object.assign({}, matchers, {
+        gradedBy: function (evaluator) {
+            const mappedMatchers = _wrapMatchers(spreadMatchers, evaluator, originalArgs, originalExpect, []);
+            // .not etc.
+            const staticMatchers = Object.keys(spreadMatchers)
+                .filter((name) => typeof matchers[name] !== "function")
+                .map((name) => {
+                return {
+                    [name]: Object.assign({}, ..._wrapMatchers(spreadMatchers, evaluator, originalArgs, originalExpect, staticPath.concat(name))),
+                };
+            });
+            return Object.assign({}, ...mappedMatchers, ...staticMatchers);
+        },
+    });
+};
+function wrapExpect(originalExpect) {
+    // proxy the expect function
+    const expectProxy = Object.assign((...args) => addGradedBy(originalExpect(...args), args, originalExpect, []), // partially apply expect to get all matchers and chain them
+    originalExpect // clone additional properties on expect
+    );
+    return expectProxy;
+}
+exports.wrapExpect = wrapExpect;
+globalThis.expect = wrapExpect(globalThis.expect);

package/dist/jest/vendor/chain.d.ts

@@ -0,0 +1 @@
+export declare function wrapExpect(originalExpect: any): any;

package/dist/jest/vendor/chain.js

@@ -0,0 +1,88 @@
+/**
+ * Adapted from https://github.com/mattphillips/jest-chain/blob/main/src/chain.js
+ */
+import { gradedBy } from "./gradedBy.js";
+class JestAssertionError extends Error {
+    constructor(result, callsite) {
+        super(typeof result.message === "function" ? result.message() : result.message);
+        Object.defineProperty(this, "matcherResult", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.matcherResult = result;
+        if (Error.captureStackTrace) {
+            Error.captureStackTrace(this, callsite);
+        }
+    }
+}
+const _wrapMatchers = (matchers, evaluator, originalArgs, originalExpect, staticPath = []) => {
+    return Object.keys(matchers)
+        .filter((name) => typeof matchers[name] === "function")
+        .map((name) => {
+        const newMatcher = async (...args) => {
+            try {
+                const score = await gradedBy(originalArgs[0], evaluator);
+                let result = originalExpect(score);
+                for (const pathEntry of staticPath) {
+                    result = result[pathEntry];
+                }
+                result = result[name](...args); // run matcher up to current state
+                if (result && typeof result.then === "function") {
+                    return Object.assign(Promise.resolve(result), matchers);
+                }
+                else {
+                    return matchers;
+                }
+            }
+            catch (error) {
+                if (!error.matcherResult) {
+                    throw error;
+                }
+                else {
+                    throw new JestAssertionError(error.matcherResult, newMatcher);
+                }
+            }
+        };
+        return { [name]: newMatcher };
+    });
+};
+const addGradedBy = (matchers, originalArgs, originalExpect, staticPath = []) => {
+    let spreadMatchers = { ...matchers };
+    // Handle Bun, which uses a class
+    if (Object.keys(matchers).length === 0) {
+        const prototypeProps = Object.getOwnPropertyNames(Object.getPrototypeOf(matchers));
+        spreadMatchers = Object.fromEntries(prototypeProps.map((prop) => {
+            try {
+                return [prop, matchers[prop]];
+            }
+            catch (e) {
+                // Ignore bizarre Bun bug
+                return [];
+            }
+        }));
+    }
+    return Object.assign({}, matchers, {
+        gradedBy: function (evaluator) {
+            const mappedMatchers = _wrapMatchers(spreadMatchers, evaluator, originalArgs, originalExpect, []);
+            // .not etc.
+            const staticMatchers = Object.keys(spreadMatchers)
+                .filter((name) => typeof matchers[name] !== "function")
+                .map((name) => {
+                return {
+                    [name]: Object.assign({}, ..._wrapMatchers(spreadMatchers, evaluator, originalArgs, originalExpect, staticPath.concat(name))),
+                };
+            });
+            return Object.assign({}, ...mappedMatchers, ...staticMatchers);
+        },
+    });
+};
+export function wrapExpect(originalExpect) {
+    // proxy the expect function
+    const expectProxy = Object.assign((...args) => addGradedBy(originalExpect(...args), args, originalExpect, []), // partially apply expect to get all matchers and chain them
+    originalExpect // clone additional properties on expect
+    );
+    return expectProxy;
+}
+globalThis.expect = wrapExpect(globalThis.expect);

package/dist/jest/vendor/gradedBy.cjs

@@ -0,0 +1,40 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.gradedBy = void 0;
+const traceable_js_1 = require("../../traceable.cjs");
+const globals_js_1 = require("../globals.cjs");
+async function gradedBy(actual, evaluator) {
+    const context = globals_js_1.jestAsyncLocalStorageInstance.getStore();
+    if (context === undefined || context.currentExample === undefined) {
+        throw new Error(`Could not identify current LangSmith context.\nPlease ensure you are calling this matcher within "ls.test()"`);
+    }
+    if ((0, globals_js_1.trackingEnabled)()) {
+        const runTree = (0, traceable_js_1.getCurrentRunTree)();
+        const wrappedEvaluator = (0, traceable_js_1.traceable)(evaluator, {
+            reference_example_id: context.currentExample.id,
+            metadata: {
+                example_version: context.currentExample.modified_at
+                    ? new Date(context.currentExample.modified_at).toISOString()
+                    : new Date(context.currentExample.created_at ?? new Date()).toISOString(),
+            },
+            client: context.client,
+            tracingEnabled: true,
+        });
+        const evalResult = await wrappedEvaluator({
+            input: runTree.inputs,
+            expected: context.currentExample.outputs ?? {},
+            actual,
+        });
+        await context.client?.logEvaluationFeedback(evalResult, runTree);
+        return evalResult.score;
+    }
+    else {
+        const evalResult = await evaluator({
+            input: context.currentExample.inputs ?? {},
+            expected: context.currentExample.outputs ?? {},
+            actual,
+        });
+        return evalResult.score;
+    }
+}
+exports.gradedBy = gradedBy;

package/dist/jest/vendor/gradedBy.d.ts

@@ -0,0 +1,7 @@
+import { EvaluationResult } from "../../evaluation/evaluator.js";
+export type SimpleEvaluator = (params: {
+    input: Record<string, any>;
+    actual: Record<string, any>;
+    expected: Record<string, any>;
+}) => EvaluationResult | Promise<EvaluationResult>;
+export declare function gradedBy(actual: any, evaluator: SimpleEvaluator): Promise<import("../../schemas.js").ScoreType | undefined>;

package/dist/jest/vendor/gradedBy.js

@@ -0,0 +1,36 @@
+import { getCurrentRunTree, traceable } from "../../traceable.js";
+import { jestAsyncLocalStorageInstance, trackingEnabled } from "../globals.js";
+export async function gradedBy(actual, evaluator) {
+    const context = jestAsyncLocalStorageInstance.getStore();
+    if (context === undefined || context.currentExample === undefined) {
+        throw new Error(`Could not identify current LangSmith context.\nPlease ensure you are calling this matcher within "ls.test()"`);
+    }
+    if (trackingEnabled()) {
+        const runTree = getCurrentRunTree();
+        const wrappedEvaluator = traceable(evaluator, {
+            reference_example_id: context.currentExample.id,
+            metadata: {
+                example_version: context.currentExample.modified_at
+                    ? new Date(context.currentExample.modified_at).toISOString()
+                    : new Date(context.currentExample.created_at ?? new Date()).toISOString(),
+            },
+            client: context.client,
+            tracingEnabled: true,
+        });
+        const evalResult = await wrappedEvaluator({
+            input: runTree.inputs,
+            expected: context.currentExample.outputs ?? {},
+            actual,
+        });
+        await context.client?.logEvaluationFeedback(evalResult, runTree);
+        return evalResult.score;
+    }
+    else {
+        const evalResult = await evaluator({
+            input: context.currentExample.inputs ?? {},
+            expected: context.currentExample.outputs ?? {},
+            actual,
+        });
+        return evalResult.score;
+    }
+}

package/dist/run_trees.d.ts

@@ -80,7 +80,7 @@ export declare class RunTree implements BaseRun {
     attachments?: Attachments;
     constructor(originalConfig: RunTreeConfig | RunTree);
     private static getDefaultConfig;
-    private static getSharedClient;
+    static getSharedClient(): Client;
     createChild(config: RunTreeConfig): RunTree;
     end(outputs?: KVMap, error?: string, endTime?: number, metadata?: KVMap): Promise<void>;
     private _convertToCreate;

package/dist/singletons/traceable.cjs

@@ -38,7 +38,7 @@ const getCurrentRunTree = () => {
         throw new Error([
             "Could not get the current run tree.",
             "",
-            "Please make sure you are calling this method within a traceable function or the tracing is enabled.",
+            "Please make sure you are calling this method within a traceable function and that tracing is enabled.",
         ].join("\n"));
     }
     return runTree;

package/dist/singletons/traceable.js

@@ -35,7 +35,7 @@ export const getCurrentRunTree = () => {
         throw new Error([
             "Could not get the current run tree.",
             "",
-            "Please make sure you are calling this method within a traceable function or the tracing is enabled.",
+            "Please make sure you are calling this method within a traceable function and that tracing is enabled.",
         ].join("\n"));
     }
     return runTree;

package/jest.cjs

@@ -0,0 +1 @@
+module.exports = require('./dist/jest/index.cjs');

package/jest.d.cts

@@ -0,0 +1 @@
+export * from './dist/jest/index.js'

package/jest.d.ts

@@ -0,0 +1 @@
+export * from './dist/jest/index.js'

package/jest.js

@@ -0,0 +1 @@
+export * from './dist/jest/index.js'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.2.15-beta.0",
+  "version": "0.2.15-rc.0",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [
@@ -33,6 +33,10 @@
     "langchain.js",
     "langchain.d.ts",
     "langchain.d.cts",
+    "jest.cjs",
+    "jest.js",
+    "jest.d.ts",
+    "jest.d.cts",
     "vercel.cjs",
     "vercel.js",
     "vercel.d.ts",
@@ -228,6 +232,15 @@
       "import": "./langchain.js",
       "require": "./langchain.cjs"
     },
+    "./jest": {
+      "types": {
+        "import": "./jest.d.ts",
+        "require": "./jest.d.cts",
+        "default": "./jest.d.ts"
+      },
+      "import": "./jest.js",
+      "require": "./jest.cjs"
+    },
     "./vercel": {
       "types": {
         "import": "./vercel.d.ts",