langsmith 0.2.13 → 0.2.14-rc.1

package/dist/index.cjs

@@ -8,4 +8,4 @@ Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () {
 var fetch_js_1 = require("./singletons/fetch.cjs");
 Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true, get: function () { return fetch_js_1.overrideFetchImplementation; } });
 // Update using yarn bump-version
-exports.__version__ = "0.2.13";
+exports.__version__ = "0.2.14-rc.1";

package/dist/index.d.ts

@@ -2,4 +2,4 @@ export { Client, type ClientConfig, type LangSmithTracingClientInterface, } from
 export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
-export declare const __version__ = "0.2.13";
+export declare const __version__ = "0.2.14-rc.1";

package/dist/index.js

@@ -2,4 +2,4 @@ export { Client, } from "./client.js";
 export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 // Update using yarn bump-version
-export const __version__ = "0.2.13";
+export const __version__ = "0.2.14-rc.1";

package/dist/jest/globals.cjs

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.trackingEnabled = exports.jestAsyncLocalStorageInstance = void 0;
+const node_async_hooks_1 = require("node:async_hooks");
+const env_js_1 = require("../utils/env.cjs");
+exports.jestAsyncLocalStorageInstance = new node_async_hooks_1.AsyncLocalStorage();
+function trackingEnabled() {
+    return (0, env_js_1.getEnvironmentVariable)("LANGSMITH_TEST_TRACKING") === "true";
+}
+exports.trackingEnabled = trackingEnabled;

package/dist/jest/globals.d.ts

@@ -0,0 +1,15 @@
+/// <reference types="node" resolution-mode="require"/>
+import { AsyncLocalStorage } from "node:async_hooks";
+import { Dataset, TracerSession, Example } from "../schemas.js";
+import { Client } from "../client.js";
+export declare const jestAsyncLocalStorageInstance: AsyncLocalStorage<{
+    dataset?: Dataset | undefined;
+    examples?: (Example & {
+        inputHash: string;
+    })[] | undefined;
+    createdAt: string;
+    project?: TracerSession | undefined;
+    currentExample?: Partial<Example> | undefined;
+    client?: Client | undefined;
+}>;
+export declare function trackingEnabled(): boolean;

package/dist/jest/globals.js

@@ -0,0 +1,6 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { getEnvironmentVariable } from "../utils/env.js";
+export const jestAsyncLocalStorageInstance = new AsyncLocalStorage();
+export function trackingEnabled() {
+    return getEnvironmentVariable("LANGSMITH_TEST_TRACKING") === "true";
+}

package/dist/jest/index.cjs

@@ -0,0 +1,188 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const globals_1 = require("@jest/globals");
+const crypto_1 = __importDefault(require("crypto"));
+const uuid_1 = require("uuid");
+const traceable_js_1 = require("../traceable.cjs");
+const run_trees_js_1 = require("../run_trees.cjs");
+const _random_name_js_1 = require("../evaluation/_random_name.cjs");
+const matchers_js_1 = require("./matchers.cjs");
+const globals_js_1 = require("./globals.cjs");
+const chain_js_1 = __importDefault(require("./vendor/chain.cjs"));
+globals_1.expect.extend({
+    toBeRelativeCloseTo: matchers_js_1.toBeRelativeCloseTo,
+    toBeAbsoluteCloseTo: matchers_js_1.toBeAbsoluteCloseTo,
+    toBeSemanticCloseTo: matchers_js_1.toBeSemanticCloseTo,
+});
+async function _createProject(client, datasetId) {
+    // Create the project, updating the experimentName until we find a unique one.
+    let project;
+    let experimentName = (0, _random_name_js_1.randomName)();
+    for (let i = 0; i < 10; i++) {
+        try {
+            project = await client.createProject({
+                projectName: experimentName,
+                referenceDatasetId: datasetId,
+                // description: this._description,
+            });
+            return project;
+        }
+        catch (e) {
+            // Naming collision
+            if (e?.name === "LangSmithConflictError") {
+                const ent = (0, uuid_1.v4)().slice(0, 6);
+                experimentName = `${experimentName}-${ent}`;
+            }
+            else {
+                throw e;
+            }
+        }
+    }
+    throw new Error("Could not generate a unique experiment name within 10 attempts." +
+        " Please try again.");
+}
+function wrapDescribeMethod(method) {
+    return function (datasetName, fn, config) {
+        const testClient = config?.client ?? run_trees_js_1.RunTree.getSharedClient();
+        return method(datasetName, () => {
+            (0, globals_1.beforeAll)(async () => {
+                if (!(0, globals_js_1.trackingEnabled)()) {
+                    globals_js_1.jestAsyncLocalStorageInstance.enterWith({
+                        createdAt: new Date().toISOString(),
+                    });
+                }
+                else {
+                    let dataset;
+                    try {
+                        dataset = await testClient.readDataset({
+                            datasetName,
+                        });
+                    }
+                    catch (e) {
+                        if (e.message.includes("not found")) {
+                            dataset = await testClient.createDataset(datasetName, {
+                                description: `Dataset for unit tests created on ${new Date().toISOString()}`,
+                            });
+                        }
+                        else {
+                            throw e;
+                        }
+                    }
+                    const examplesList = testClient.listExamples({
+                        datasetName,
+                    });
+                    const examples = [];
+                    for await (const example of examplesList) {
+                        const inputHash = crypto_1.default
+                            .createHash("sha256")
+                            .update(JSON.stringify(example.inputs))
+                            .digest("hex");
+                        examples.push({ ...example, inputHash });
+                    }
+                    const project = await _createProject(testClient, dataset.id);
+                    globals_js_1.jestAsyncLocalStorageInstance.enterWith({
+                        dataset,
+                        examples,
+                        createdAt: new Date().toISOString(),
+                        project,
+                        client: testClient,
+                    });
+                }
+            });
+            fn();
+        });
+    };
+}
+const lsDescribe = Object.assign(wrapDescribeMethod(globals_1.describe), {
+    only: wrapDescribeMethod(globals_1.describe.only),
+    skip: wrapDescribeMethod(globals_1.describe.skip),
+});
+function wrapTestMethod(method) {
+    return function (params, config) {
+        return async function (...args) {
+            return method(args[0], async () => {
+                const testInput = typeof params === "string" ? {} : params.inputs;
+                const testOutput = typeof params === "string" ? {} : params.outputs;
+                const inputHash = crypto_1.default
+                    .createHash("sha256")
+                    .update(JSON.stringify(testInput))
+                    .digest("hex");
+                const context = globals_js_1.jestAsyncLocalStorageInstance.getStore();
+                if (context === undefined) {
+                    throw new Error(`Could not retrieve test context.\nPlease make sure you have tracing enabled and you are wrapping all of your test cases in an "ls.describe()" function.`);
+                }
+                const { examples, dataset, createdAt, project, client } = context;
+                if ((0, globals_js_1.trackingEnabled)()) {
+                    if (examples === undefined ||
+                        dataset === undefined ||
+                        project === undefined ||
+                        client === undefined) {
+                        throw new Error("Failed to initialize test tracking. Please contact us for help.");
+                    }
+                    const testClient = config?.client ?? client;
+                    let example = (examples ?? []).find((example) => {
+                        return example.inputHash === inputHash;
+                    });
+                    if (example === undefined) {
+                        const newExample = await testClient.createExample(testInput, testOutput, {
+                            datasetId: dataset?.id,
+                            createdAt: new Date(createdAt ?? new Date()),
+                        });
+                        example = { ...newExample, inputHash };
+                    }
+                    globals_js_1.jestAsyncLocalStorageInstance.enterWith({
+                        ...context,
+                        currentExample: example,
+                        client: testClient,
+                    });
+                    const traceableOptions = {
+                        reference_example_id: example.id,
+                        project_name: project.name,
+                        metadata: {
+                            ...config?.metadata,
+                            example_version: example.modified_at
+                                ? new Date(example.modified_at).toISOString()
+                                : new Date(example.created_at).toISOString(),
+                        },
+                        client: testClient,
+                        tracingEnabled: true,
+                        name: "Unit test",
+                    };
+                    // Pass inputs into traceable so tracing works correctly but
+                    // provide both to the user-defined test function
+                    const tracedFunction = (0, traceable_js_1.traceable)(async (_) => {
+                        return args[1]({
+                            inputs: testInput,
+                            outputs: testOutput,
+                        });
+                    }, { ...traceableOptions, ...config });
+                    await tracedFunction(testInput);
+                    await testClient.awaitPendingTraceBatches();
+                }
+                else {
+                    globals_js_1.jestAsyncLocalStorageInstance.enterWith({
+                        ...context,
+                        currentExample: { inputs: testInput, outputs: testOutput },
+                    });
+                    await args[1]({
+                        inputs: testInput,
+                        outputs: testOutput,
+                    });
+                }
+            }, ...args.slice(2));
+        };
+    };
+}
+const lsTest = Object.assign(wrapTestMethod(globals_1.test), {
+    only: wrapTestMethod(globals_1.test.only),
+    skip: wrapTestMethod(globals_1.test.skip),
+});
+exports.default = {
+    test: lsTest,
+    it: lsTest,
+    describe: lsDescribe,
+    expect: chain_js_1.default,
+};

package/dist/jest/index.d.ts

@@ -0,0 +1,62 @@
+import { RunTreeConfig } from "../run_trees.js";
+import expectWithGradedBy from "./vendor/chain.js";
+import type { SimpleEvaluator } from "./vendor/gradedBy.js";
+declare global {
+    namespace jest {
+        interface AsymmetricMatchers {
+            toBeRelativeCloseTo(expected: string, options: any): void;
+            toBeAbsoluteCloseTo(expected: string, options: any): void;
+            toBeSemanticCloseTo(expected: string, options: any): Promise<void>;
+        }
+        interface Matchers<R> {
+            toBeRelativeCloseTo(expected: string, options: any): R;
+            toBeAbsoluteCloseTo(expected: string, options: any): R;
+            toBeSemanticCloseTo(expected: string, options: any): Promise<R>;
+            gradedBy(evaluator: SimpleEvaluator): jest.Matchers<Promise<R>> & {
+                not: jest.Matchers<Promise<R>>;
+                resolves: jest.Matchers<Promise<R>>;
+                rejects: jest.Matchers<Promise<R>>;
+            };
+        }
+    }
+}
+export type LangSmithJestDescribeWrapper = (name: string, fn: () => void | Promise<void>, config?: Partial<RunTreeConfig>) => void;
+export type LangSmithJestTestWrapper<I, O> = (name: string, fn: (params: {
+    inputs: I;
+    outputs: O;
+}) => unknown | Promise<unknown>, timeout?: number) => void;
+declare const _default: {
+    test: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: string | {
+        inputs: I;
+        outputs: O;
+    }, config?: Partial<RunTreeConfig> | undefined) => LangSmithJestTestWrapper<I, O>) & {
+        only: <I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: string | {
+            inputs: I;
+            outputs: O;
+        }, config?: Partial<RunTreeConfig> | undefined) => LangSmithJestTestWrapper<I, O>;
+        skip: <I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: string | {
+            inputs: I;
+            outputs: O;
+        }, config?: Partial<RunTreeConfig> | undefined) => LangSmithJestTestWrapper<I, O>;
+    };
+    it: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: string | {
+        inputs: I;
+        outputs: O;
+    }, config?: Partial<RunTreeConfig> | undefined) => LangSmithJestTestWrapper<I, O>) & {
+        only: <I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: string | {
+            inputs: I;
+            outputs: O;
+        }, config?: Partial<RunTreeConfig> | undefined) => LangSmithJestTestWrapper<I, O>;
+        skip: <I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: string | {
+            inputs: I;
+            outputs: O;
+        }, config?: Partial<RunTreeConfig> | undefined) => LangSmithJestTestWrapper<I, O>;
+    };
+    describe: LangSmithJestDescribeWrapper & {
+        only: LangSmithJestDescribeWrapper;
+        skip: LangSmithJestDescribeWrapper;
+    };
+    expect: typeof expectWithGradedBy;
+};
+export default _default;
+export { type SimpleEvaluator };

package/dist/jest/index.js

@@ -0,0 +1,183 @@
+import { expect, test, describe, beforeAll } from "@jest/globals";
+import crypto from "crypto";
+import { v4 } from "uuid";
+import { traceable } from "../traceable.js";
+import { RunTree } from "../run_trees.js";
+import { randomName } from "../evaluation/_random_name.js";
+import { toBeRelativeCloseTo, toBeAbsoluteCloseTo, toBeSemanticCloseTo, } from "./matchers.js";
+import { jestAsyncLocalStorageInstance, trackingEnabled } from "./globals.js";
+import expectWithGradedBy from "./vendor/chain.js";
+expect.extend({
+    toBeRelativeCloseTo,
+    toBeAbsoluteCloseTo,
+    toBeSemanticCloseTo,
+});
+async function _createProject(client, datasetId) {
+    // Create the project, updating the experimentName until we find a unique one.
+    let project;
+    let experimentName = randomName();
+    for (let i = 0; i < 10; i++) {
+        try {
+            project = await client.createProject({
+                projectName: experimentName,
+                referenceDatasetId: datasetId,
+                // description: this._description,
+            });
+            return project;
+        }
+        catch (e) {
+            // Naming collision
+            if (e?.name === "LangSmithConflictError") {
+                const ent = v4().slice(0, 6);
+                experimentName = `${experimentName}-${ent}`;
+            }
+            else {
+                throw e;
+            }
+        }
+    }
+    throw new Error("Could not generate a unique experiment name within 10 attempts." +
+        " Please try again.");
+}
+function wrapDescribeMethod(method) {
+    return function (datasetName, fn, config) {
+        const testClient = config?.client ?? RunTree.getSharedClient();
+        return method(datasetName, () => {
+            beforeAll(async () => {
+                if (!trackingEnabled()) {
+                    jestAsyncLocalStorageInstance.enterWith({
+                        createdAt: new Date().toISOString(),
+                    });
+                }
+                else {
+                    let dataset;
+                    try {
+                        dataset = await testClient.readDataset({
+                            datasetName,
+                        });
+                    }
+                    catch (e) {
+                        if (e.message.includes("not found")) {
+                            dataset = await testClient.createDataset(datasetName, {
+                                description: `Dataset for unit tests created on ${new Date().toISOString()}`,
+                            });
+                        }
+                        else {
+                            throw e;
+                        }
+                    }
+                    const examplesList = testClient.listExamples({
+                        datasetName,
+                    });
+                    const examples = [];
+                    for await (const example of examplesList) {
+                        const inputHash = crypto
+                            .createHash("sha256")
+                            .update(JSON.stringify(example.inputs))
+                            .digest("hex");
+                        examples.push({ ...example, inputHash });
+                    }
+                    const project = await _createProject(testClient, dataset.id);
+                    jestAsyncLocalStorageInstance.enterWith({
+                        dataset,
+                        examples,
+                        createdAt: new Date().toISOString(),
+                        project,
+                        client: testClient,
+                    });
+                }
+            });
+            fn();
+        });
+    };
+}
+const lsDescribe = Object.assign(wrapDescribeMethod(describe), {
+    only: wrapDescribeMethod(describe.only),
+    skip: wrapDescribeMethod(describe.skip),
+});
+function wrapTestMethod(method) {
+    return function (params, config) {
+        return async function (...args) {
+            return method(args[0], async () => {
+                const testInput = typeof params === "string" ? {} : params.inputs;
+                const testOutput = typeof params === "string" ? {} : params.outputs;
+                const inputHash = crypto
+                    .createHash("sha256")
+                    .update(JSON.stringify(testInput))
+                    .digest("hex");
+                const context = jestAsyncLocalStorageInstance.getStore();
+                if (context === undefined) {
+                    throw new Error(`Could not retrieve test context.\nPlease make sure you have tracing enabled and you are wrapping all of your test cases in an "ls.describe()" function.`);
+                }
+                const { examples, dataset, createdAt, project, client } = context;
+                if (trackingEnabled()) {
+                    if (examples === undefined ||
+                        dataset === undefined ||
+                        project === undefined ||
+                        client === undefined) {
+                        throw new Error("Failed to initialize test tracking. Please contact us for help.");
+                    }
+                    const testClient = config?.client ?? client;
+                    let example = (examples ?? []).find((example) => {
+                        return example.inputHash === inputHash;
+                    });
+                    if (example === undefined) {
+                        const newExample = await testClient.createExample(testInput, testOutput, {
+                            datasetId: dataset?.id,
+                            createdAt: new Date(createdAt ?? new Date()),
+                        });
+                        example = { ...newExample, inputHash };
+                    }
+                    jestAsyncLocalStorageInstance.enterWith({
+                        ...context,
+                        currentExample: example,
+                        client: testClient,
+                    });
+                    const traceableOptions = {
+                        reference_example_id: example.id,
+                        project_name: project.name,
+                        metadata: {
+                            ...config?.metadata,
+                            example_version: example.modified_at
+                                ? new Date(example.modified_at).toISOString()
+                                : new Date(example.created_at).toISOString(),
+                        },
+                        client: testClient,
+                        tracingEnabled: true,
+                        name: "Unit test",
+                    };
+                    // Pass inputs into traceable so tracing works correctly but
+                    // provide both to the user-defined test function
+                    const tracedFunction = traceable(async (_) => {
+                        return args[1]({
+                            inputs: testInput,
+                            outputs: testOutput,
+                        });
+                    }, { ...traceableOptions, ...config });
+                    await tracedFunction(testInput);
+                    await testClient.awaitPendingTraceBatches();
+                }
+                else {
+                    jestAsyncLocalStorageInstance.enterWith({
+                        ...context,
+                        currentExample: { inputs: testInput, outputs: testOutput },
+                    });
+                    await args[1]({
+                        inputs: testInput,
+                        outputs: testOutput,
+                    });
+                }
+            }, ...args.slice(2));
+        };
+    };
+}
+const lsTest = Object.assign(wrapTestMethod(test), {
+    only: wrapTestMethod(test.only),
+    skip: wrapTestMethod(test.skip),
+});
+export default {
+    test: lsTest,
+    it: lsTest,
+    describe: lsDescribe,
+    expect: expectWithGradedBy,
+};

package/dist/jest/matchers.cjs

@@ -0,0 +1,149 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.toBeSemanticCloseTo = exports.toBeAbsoluteCloseTo = exports.toBeRelativeCloseTo = void 0;
+// Levenshtein distance implementation
+function levenshteinDistance(a, b) {
+    if (a.length === 0)
+        return b.length;
+    if (b.length === 0)
+        return a.length;
+    const matrix = Array(b.length + 1)
+        .fill(null)
+        .map(() => Array(a.length + 1).fill(null));
+    for (let i = 0; i <= a.length; i++)
+        matrix[0][i] = i;
+    for (let j = 0; j <= b.length; j++)
+        matrix[j][0] = j;
+    for (let j = 1; j <= b.length; j++) {
+        for (let i = 1; i <= a.length; i++) {
+            const substitutionCost = a[i - 1] === b[j - 1] ? 0 : 1;
+            matrix[j][i] = Math.min(matrix[j][i - 1] + 1, matrix[j - 1][i] + 1, matrix[j - 1][i - 1] + substitutionCost);
+        }
+    }
+    return matrix[b.length][a.length];
+}
+async function toBeRelativeCloseTo(received, expected, options = {}) {
+    const { threshold = 0.1, algorithm = "levenshtein" } = options;
+    let distance;
+    let maxLength;
+    switch (algorithm) {
+        case "levenshtein":
+            distance = levenshteinDistance(received, expected);
+            maxLength = Math.max(received.length, expected.length);
+            break;
+        default:
+            throw new Error(`Unsupported algorithm: ${algorithm}`);
+    }
+    // Calculate relative distance (normalized between 0 and 1)
+    const relativeDistance = maxLength === 0 ? 0 : distance / maxLength;
+    const pass = relativeDistance <= threshold;
+    return {
+        pass,
+        message: () => pass
+            ? `Expected "${received}" not to be relatively close to "${expected}" (threshold: ${threshold}, actual distance: ${relativeDistance})`
+            : `Expected "${received}" to be relatively close to "${expected}" (threshold: ${threshold}, actual distance: ${relativeDistance})`,
+    };
+}
+exports.toBeRelativeCloseTo = toBeRelativeCloseTo;
+async function toBeAbsoluteCloseTo(received, expected, options = {}) {
+    const { threshold = 3, algorithm = "levenshtein" } = options;
+    let distance;
+    switch (algorithm) {
+        case "levenshtein":
+            distance = levenshteinDistance(received, expected);
+            break;
+        default:
+            throw new Error(`Unsupported algorithm: ${algorithm}`);
+    }
+    const pass = distance <= threshold;
+    return {
+        pass,
+        message: () => pass
+            ? `Expected "${received}" not to be absolutely close to "${expected}" (threshold: ${threshold}, actual distance: ${distance})`
+            : `Expected "${received}" to be absolutely close to "${expected}" (threshold: ${threshold}, actual distance: ${distance})`,
+    };
+}
+exports.toBeAbsoluteCloseTo = toBeAbsoluteCloseTo;
+async function toBeSemanticCloseTo(received, expected, options) {
+    const { threshold = 0.2, embedding, algorithm = "cosine" } = options;
+    // Get embeddings for both strings
+    const [receivedEmbedding, expectedEmbedding] = await Promise.all([
+        embedding.embedQuery(received),
+        embedding.embedQuery(expected),
+    ]);
+    // Calculate similarity based on chosen algorithm
+    let similarity;
+    switch (algorithm) {
+        case "cosine": {
+            // Compute cosine similarity
+            const dotProduct = receivedEmbedding.reduce((sum, a, i) => sum + a * expectedEmbedding[i], 0);
+            const receivedMagnitude = Math.sqrt(receivedEmbedding.reduce((sum, a) => sum + a * a, 0));
+            const expectedMagnitude = Math.sqrt(expectedEmbedding.reduce((sum, a) => sum + a * a, 0));
+            similarity = dotProduct / (receivedMagnitude * expectedMagnitude);
+            break;
+        }
+        case "dot-product": {
+            // Compute dot product similarity
+            similarity = receivedEmbedding.reduce((sum, a, i) => sum + a * expectedEmbedding[i], 0);
+            break;
+        }
+        default:
+            throw new Error(`Unsupported algorithm: ${algorithm}`);
+    }
+    const pass = similarity >= 1 - threshold;
+    return {
+        pass,
+        message: () => pass
+            ? `Expected "${received}" not to be semantically close to "${expected}" (threshold: ${threshold}, similarity: ${similarity})`
+            : `Expected "${received}" to be semantically close to "${expected}" (threshold: ${threshold}, similarity: ${similarity})`,
+    };
+}
+exports.toBeSemanticCloseTo = toBeSemanticCloseTo;
+// export async function toPassEvaluator(
+//   this: MatcherContext,
+//   actual: KVMap,
+//   evaluator: SimpleEvaluator,
+//   _expected?: KVMap
+// ) {
+//   const runTree = getCurrentRunTree();
+//   const context = localStorage.getStore();
+//   if (context === undefined || context.currentExample === undefined) {
+//     throw new Error(
+//       `Could not identify example context from current context.\nPlease ensure you are calling this matcher within "ls.test()"`
+//     );
+//   }
+//   const wrappedEvaluator = traceable(evaluator, {
+//     reference_example_id: context.currentExample.id,
+//     metadata: {
+//       example_version: context.currentExample.modified_at
+//         ? new Date(context.currentExample.modified_at).toISOString()
+//         : new Date(context.currentExample.created_at).toISOString(),
+//     },
+//     client: context.client,
+//     tracingEnabled: true,
+//   });
+//   const evalResult = await wrappedEvaluator({
+//     input: runTree.inputs,
+//     expected: context.currentExample.outputs ?? {},
+//     actual,
+//   });
+//   await context.client.logEvaluationFeedback(evalResult, runTree);
+//   if (!("results" in evalResult) && !evalResult.score) {
+//     return {
+//       pass: false,
+//       message: () =>
+//         `expected ${this.utils.printReceived(
+//           actual
+//         )} to pass evaluator. Failed with ${JSON.stringify(
+//           evalResult,
+//           null,
+//           2
+//         )}`,
+//     };
+//   }
+//   return {
+//     pass: true,
+//     message: () =>
+//       `evaluator passed with score ${JSON.stringify(evalResult, null, 2)}`,
+//   };
+// }

package/dist/jest/matchers.d.ts

@@ -0,0 +1,25 @@
+import type { MatcherContext } from "expect";
+export declare function toBeRelativeCloseTo(this: MatcherContext, received: string, expected: string, options?: {
+    threshold?: number;
+    algorithm?: "levenshtein";
+}): Promise<{
+    pass: boolean;
+    message: () => string;
+}>;
+export declare function toBeAbsoluteCloseTo(this: MatcherContext, received: string, expected: string, options?: {
+    threshold?: number;
+    algorithm?: "levenshtein";
+}): Promise<{
+    pass: boolean;
+    message: () => string;
+}>;
+export declare function toBeSemanticCloseTo(this: MatcherContext, received: string, expected: string, options: {
+    threshold?: number;
+    embedding: {
+        embedQuery: (query: string) => number[] | Promise<number[]>;
+    };
+    algorithm?: "cosine" | "dot-product";
+}): Promise<{
+    pass: boolean;
+    message: () => string;
+}>;

package/dist/jest/matchers.js

@@ -0,0 +1,143 @@
+// Levenshtein distance implementation
+function levenshteinDistance(a, b) {
+    if (a.length === 0)
+        return b.length;
+    if (b.length === 0)
+        return a.length;
+    const matrix = Array(b.length + 1)
+        .fill(null)
+        .map(() => Array(a.length + 1).fill(null));
+    for (let i = 0; i <= a.length; i++)
+        matrix[0][i] = i;
+    for (let j = 0; j <= b.length; j++)
+        matrix[j][0] = j;
+    for (let j = 1; j <= b.length; j++) {
+        for (let i = 1; i <= a.length; i++) {
+            const substitutionCost = a[i - 1] === b[j - 1] ? 0 : 1;
+            matrix[j][i] = Math.min(matrix[j][i - 1] + 1, matrix[j - 1][i] + 1, matrix[j - 1][i - 1] + substitutionCost);
+        }
+    }
+    return matrix[b.length][a.length];
+}
+export async function toBeRelativeCloseTo(received, expected, options = {}) {
+    const { threshold = 0.1, algorithm = "levenshtein" } = options;
+    let distance;
+    let maxLength;
+    switch (algorithm) {
+        case "levenshtein":
+            distance = levenshteinDistance(received, expected);
+            maxLength = Math.max(received.length, expected.length);
+            break;
+        default:
+            throw new Error(`Unsupported algorithm: ${algorithm}`);
+    }
+    // Calculate relative distance (normalized between 0 and 1)
+    const relativeDistance = maxLength === 0 ? 0 : distance / maxLength;
+    const pass = relativeDistance <= threshold;
+    return {
+        pass,
+        message: () => pass
+            ? `Expected "${received}" not to be relatively close to "${expected}" (threshold: ${threshold}, actual distance: ${relativeDistance})`
+            : `Expected "${received}" to be relatively close to "${expected}" (threshold: ${threshold}, actual distance: ${relativeDistance})`,
+    };
+}
+export async function toBeAbsoluteCloseTo(received, expected, options = {}) {
+    const { threshold = 3, algorithm = "levenshtein" } = options;
+    let distance;
+    switch (algorithm) {
+        case "levenshtein":
+            distance = levenshteinDistance(received, expected);
+            break;
+        default:
+            throw new Error(`Unsupported algorithm: ${algorithm}`);
+    }
+    const pass = distance <= threshold;
+    return {
+        pass,
+        message: () => pass
+            ? `Expected "${received}" not to be absolutely close to "${expected}" (threshold: ${threshold}, actual distance: ${distance})`
+            : `Expected "${received}" to be absolutely close to "${expected}" (threshold: ${threshold}, actual distance: ${distance})`,
+    };
+}
+export async function toBeSemanticCloseTo(received, expected, options) {
+    const { threshold = 0.2, embedding, algorithm = "cosine" } = options;
+    // Get embeddings for both strings
+    const [receivedEmbedding, expectedEmbedding] = await Promise.all([
+        embedding.embedQuery(received),
+        embedding.embedQuery(expected),
+    ]);
+    // Calculate similarity based on chosen algorithm
+    let similarity;
+    switch (algorithm) {
+        case "cosine": {
+            // Compute cosine similarity
+            const dotProduct = receivedEmbedding.reduce((sum, a, i) => sum + a * expectedEmbedding[i], 0);
+            const receivedMagnitude = Math.sqrt(receivedEmbedding.reduce((sum, a) => sum + a * a, 0));
+            const expectedMagnitude = Math.sqrt(expectedEmbedding.reduce((sum, a) => sum + a * a, 0));
+            similarity = dotProduct / (receivedMagnitude * expectedMagnitude);
+            break;
+        }
+        case "dot-product": {
+            // Compute dot product similarity
+            similarity = receivedEmbedding.reduce((sum, a, i) => sum + a * expectedEmbedding[i], 0);
+            break;
+        }
+        default:
+            throw new Error(`Unsupported algorithm: ${algorithm}`);
+    }
+    const pass = similarity >= 1 - threshold;
+    return {
+        pass,
+        message: () => pass
+            ? `Expected "${received}" not to be semantically close to "${expected}" (threshold: ${threshold}, similarity: ${similarity})`
+            : `Expected "${received}" to be semantically close to "${expected}" (threshold: ${threshold}, similarity: ${similarity})`,
+    };
+}
+// export async function toPassEvaluator(
+//   this: MatcherContext,
+//   actual: KVMap,
+//   evaluator: SimpleEvaluator,
+//   _expected?: KVMap
+// ) {
+//   const runTree = getCurrentRunTree();
+//   const context = localStorage.getStore();
+//   if (context === undefined || context.currentExample === undefined) {
+//     throw new Error(
+//       `Could not identify example context from current context.\nPlease ensure you are calling this matcher within "ls.test()"`
+//     );
+//   }
+//   const wrappedEvaluator = traceable(evaluator, {
+//     reference_example_id: context.currentExample.id,
+//     metadata: {
+//       example_version: context.currentExample.modified_at
+//         ? new Date(context.currentExample.modified_at).toISOString()
+//         : new Date(context.currentExample.created_at).toISOString(),
+//     },
+//     client: context.client,
+//     tracingEnabled: true,
+//   });
+//   const evalResult = await wrappedEvaluator({
+//     input: runTree.inputs,
+//     expected: context.currentExample.outputs ?? {},
+//     actual,
+//   });
+//   await context.client.logEvaluationFeedback(evalResult, runTree);
+//   if (!("results" in evalResult) && !evalResult.score) {
+//     return {
+//       pass: false,
+//       message: () =>
+//         `expected ${this.utils.printReceived(
+//           actual
+//         )} to pass evaluator. Failed with ${JSON.stringify(
+//           evalResult,
+//           null,
+//           2
+//         )}`,
+//     };
+//   }
+//   return {
+//     pass: true,
+//     message: () =>
+//       `evaluator passed with score ${JSON.stringify(evalResult, null, 2)}`,
+//   };
+// }

package/dist/jest/vendor/chain.cjs

@@ -0,0 +1,81 @@
+"use strict";
+/**
+ * Adapted from https://github.com/mattphillips/jest-chain/blob/main/src/chain.js
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+const gradedBy_js_1 = require("./gradedBy.cjs");
+class JestAssertionError extends Error {
+    constructor(result, callsite) {
+        super(typeof result.message === "function" ? result.message() : result.message);
+        Object.defineProperty(this, "matcherResult", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.matcherResult = result;
+        if (Error.captureStackTrace) {
+            Error.captureStackTrace(this, callsite);
+        }
+    }
+}
+const _wrapMatchers = (matchers, evaluator, originalArgs, staticPath = []) => {
+    return Object.keys(matchers)
+        .filter((name) => typeof matchers[name] === "function")
+        .map((name) => {
+        const newMatcher = async (...args) => {
+            try {
+                const score = await (0, gradedBy_js_1.gradedBy)(originalArgs[0], evaluator);
+                let result = expect(score);
+                for (const pathEntry of staticPath) {
+                    result = result[pathEntry];
+                }
+                result = result[name](...args); // run matcher up to current state
+                if (result && typeof result.then === "function") {
+                    return Object.assign(Promise.resolve(result), matchers);
+                }
+                else {
+                    return matchers;
+                }
+            }
+            catch (error) {
+                if (!error.matcherResult) {
+                    throw error;
+                }
+                else {
+                    throw new JestAssertionError(error.matcherResult, newMatcher);
+                }
+            }
+        };
+        return { [name]: newMatcher };
+    });
+};
+const addGradedBy = (matchers, originalArgs, staticPath = []) => {
+    return Object.assign({}, matchers, {
+        gradedBy: function (evaluator) {
+            const mappedMatchers = _wrapMatchers(matchers, evaluator, originalArgs);
+            // .not etc.
+            const staticMatchers = Object.keys(matchers)
+                .filter((name) => typeof matchers[name] !== "function")
+                .map((name) => {
+                return {
+                    [name]: Object.assign({}, ..._wrapMatchers(matchers, evaluator, originalArgs, staticPath.concat(name))),
+                };
+            });
+            return Object.assign({}, ...mappedMatchers, ...staticMatchers);
+        },
+    });
+};
+function expectWithGradedBy(expect) {
+    // proxy the expect function
+    const expectProxy = Object.assign((...args) => addGradedBy(expect(...args), args), // partially apply expect to get all matchers and chain them
+    expect // clone additional properties on expect
+    );
+    // expectProxy.extend = (o: any) => {
+    //   expect.extend(o); // add new matchers to expect
+    //   expectProxy = Object.assign(expectProxy, expect); // clone new asymmetric matchers
+    // };
+    return expectProxy;
+}
+exports.default = expectWithGradedBy;
+globalThis.expect = expectWithGradedBy(globalThis.expect);

package/dist/jest/vendor/chain.d.ts

@@ -0,0 +1,4 @@
+/**
+ * Adapted from https://github.com/mattphillips/jest-chain/blob/main/src/chain.js
+ */
+export default function expectWithGradedBy(expect: any): any;

package/dist/jest/vendor/chain.js

@@ -0,0 +1,78 @@
+/**
+ * Adapted from https://github.com/mattphillips/jest-chain/blob/main/src/chain.js
+ */
+import { gradedBy } from "./gradedBy.js";
+class JestAssertionError extends Error {
+    constructor(result, callsite) {
+        super(typeof result.message === "function" ? result.message() : result.message);
+        Object.defineProperty(this, "matcherResult", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.matcherResult = result;
+        if (Error.captureStackTrace) {
+            Error.captureStackTrace(this, callsite);
+        }
+    }
+}
+const _wrapMatchers = (matchers, evaluator, originalArgs, staticPath = []) => {
+    return Object.keys(matchers)
+        .filter((name) => typeof matchers[name] === "function")
+        .map((name) => {
+        const newMatcher = async (...args) => {
+            try {
+                const score = await gradedBy(originalArgs[0], evaluator);
+                let result = expect(score);
+                for (const pathEntry of staticPath) {
+                    result = result[pathEntry];
+                }
+                result = result[name](...args); // run matcher up to current state
+                if (result && typeof result.then === "function") {
+                    return Object.assign(Promise.resolve(result), matchers);
+                }
+                else {
+                    return matchers;
+                }
+            }
+            catch (error) {
+                if (!error.matcherResult) {
+                    throw error;
+                }
+                else {
+                    throw new JestAssertionError(error.matcherResult, newMatcher);
+                }
+            }
+        };
+        return { [name]: newMatcher };
+    });
+};
+const addGradedBy = (matchers, originalArgs, staticPath = []) => {
+    return Object.assign({}, matchers, {
+        gradedBy: function (evaluator) {
+            const mappedMatchers = _wrapMatchers(matchers, evaluator, originalArgs);
+            // .not etc.
+            const staticMatchers = Object.keys(matchers)
+                .filter((name) => typeof matchers[name] !== "function")
+                .map((name) => {
+                return {
+                    [name]: Object.assign({}, ..._wrapMatchers(matchers, evaluator, originalArgs, staticPath.concat(name))),
+                };
+            });
+            return Object.assign({}, ...mappedMatchers, ...staticMatchers);
+        },
+    });
+};
+export default function expectWithGradedBy(expect) {
+    // proxy the expect function
+    const expectProxy = Object.assign((...args) => addGradedBy(expect(...args), args), // partially apply expect to get all matchers and chain them
+    expect // clone additional properties on expect
+    );
+    // expectProxy.extend = (o: any) => {
+    //   expect.extend(o); // add new matchers to expect
+    //   expectProxy = Object.assign(expectProxy, expect); // clone new asymmetric matchers
+    // };
+    return expectProxy;
+}
+globalThis.expect = expectWithGradedBy(globalThis.expect);

package/dist/jest/vendor/gradedBy.cjs

@@ -0,0 +1,40 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.gradedBy = void 0;
+const traceable_js_1 = require("../../traceable.cjs");
+const globals_js_1 = require("../globals.cjs");
+async function gradedBy(actual, evaluator) {
+    const context = globals_js_1.jestAsyncLocalStorageInstance.getStore();
+    if (context === undefined || context.currentExample === undefined) {
+        throw new Error(`Could not identify current LangSmith context.\nPlease ensure you are calling this matcher within "ls.test()"`);
+    }
+    if ((0, globals_js_1.trackingEnabled)()) {
+        const runTree = (0, traceable_js_1.getCurrentRunTree)();
+        const wrappedEvaluator = (0, traceable_js_1.traceable)(evaluator, {
+            reference_example_id: context.currentExample.id,
+            metadata: {
+                example_version: context.currentExample.modified_at
+                    ? new Date(context.currentExample.modified_at).toISOString()
+                    : new Date(context.currentExample.created_at ?? new Date()).toISOString(),
+            },
+            client: context.client,
+            tracingEnabled: true,
+        });
+        const evalResult = await wrappedEvaluator({
+            input: runTree.inputs,
+            expected: context.currentExample.outputs ?? {},
+            actual,
+        });
+        await context.client?.logEvaluationFeedback(evalResult, runTree);
+        return evalResult.score;
+    }
+    else {
+        const evalResult = await evaluator({
+            input: context.currentExample.inputs ?? {},
+            expected: context.currentExample.outputs ?? {},
+            actual,
+        });
+        return evalResult.score;
+    }
+}
+exports.gradedBy = gradedBy;

package/dist/jest/vendor/gradedBy.d.ts

@@ -0,0 +1,7 @@
+import { EvaluationResult } from "../../evaluation/evaluator.js";
+export type SimpleEvaluator = (params: {
+    input: Record<string, any>;
+    actual: Record<string, any>;
+    expected: Record<string, any>;
+}) => EvaluationResult | Promise<EvaluationResult>;
+export declare function gradedBy(actual: any, evaluator: SimpleEvaluator): Promise<import("../../schemas.js").ScoreType | undefined>;

package/dist/jest/vendor/gradedBy.js

@@ -0,0 +1,36 @@
+import { getCurrentRunTree, traceable } from "../../traceable.js";
+import { jestAsyncLocalStorageInstance, trackingEnabled } from "../globals.js";
+export async function gradedBy(actual, evaluator) {
+    const context = jestAsyncLocalStorageInstance.getStore();
+    if (context === undefined || context.currentExample === undefined) {
+        throw new Error(`Could not identify current LangSmith context.\nPlease ensure you are calling this matcher within "ls.test()"`);
+    }
+    if (trackingEnabled()) {
+        const runTree = getCurrentRunTree();
+        const wrappedEvaluator = traceable(evaluator, {
+            reference_example_id: context.currentExample.id,
+            metadata: {
+                example_version: context.currentExample.modified_at
+                    ? new Date(context.currentExample.modified_at).toISOString()
+                    : new Date(context.currentExample.created_at ?? new Date()).toISOString(),
+            },
+            client: context.client,
+            tracingEnabled: true,
+        });
+        const evalResult = await wrappedEvaluator({
+            input: runTree.inputs,
+            expected: context.currentExample.outputs ?? {},
+            actual,
+        });
+        await context.client?.logEvaluationFeedback(evalResult, runTree);
+        return evalResult.score;
+    }
+    else {
+        const evalResult = await evaluator({
+            input: context.currentExample.inputs ?? {},
+            expected: context.currentExample.outputs ?? {},
+            actual,
+        });
+        return evalResult.score;
+    }
+}

package/dist/run_trees.d.ts

@@ -80,7 +80,7 @@ export declare class RunTree implements BaseRun {
     attachments?: Attachments;
     constructor(originalConfig: RunTreeConfig | RunTree);
     private static getDefaultConfig;
-    private static getSharedClient;
+    static getSharedClient(): Client;
     createChild(config: RunTreeConfig): RunTree;
     end(outputs?: KVMap, error?: string, endTime?: number, metadata?: KVMap): Promise<void>;
     private _convertToCreate;

package/dist/singletons/traceable.cjs

@@ -38,7 +38,7 @@ const getCurrentRunTree = () => {
         throw new Error([
             "Could not get the current run tree.",
             "",
-            "Please make sure you are calling this method within a traceable function or the tracing is enabled.",
+            "Please make sure you are calling this method within a traceable function and that tracing is enabled.",
         ].join("\n"));
     }
     return runTree;

package/dist/singletons/traceable.js

@@ -35,7 +35,7 @@ export const getCurrentRunTree = () => {
         throw new Error([
             "Could not get the current run tree.",
             "",
-            "Please make sure you are calling this method within a traceable function or the tracing is enabled.",
+            "Please make sure you are calling this method within a traceable function and that tracing is enabled.",
         ].join("\n"));
     }
     return runTree;

package/jest.cjs

@@ -0,0 +1 @@
+module.exports = require('./dist/jest/index.cjs');

package/jest.d.cts

@@ -0,0 +1 @@
+export * from './dist/jest/index.js'

package/jest.d.ts

@@ -0,0 +1 @@
+export * from './dist/jest/index.js'

package/jest.js

@@ -0,0 +1 @@
+export * from './dist/jest/index.js'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.2.13",
+  "version": "0.2.14-rc.1",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [
@@ -33,6 +33,10 @@
     "langchain.js",
     "langchain.d.ts",
     "langchain.d.cts",
+    "jest.cjs",
+    "jest.js",
+    "jest.d.ts",
+    "jest.d.cts",
     "vercel.cjs",
     "vercel.js",
     "vercel.d.ts",
@@ -228,6 +232,15 @@
       "import": "./langchain.js",
       "require": "./langchain.cjs"
     },
+    "./jest": {
+      "types": {
+        "import": "./jest.d.ts",
+        "require": "./jest.d.cts",
+        "default": "./jest.d.ts"
+      },
+      "import": "./jest.js",
+      "require": "./jest.cjs"
+    },
     "./vercel": {
       "types": {
         "import": "./vercel.d.ts",