langsmith 0.2.14-rc.0 → 0.2.14-rc.2

package/dist/index.cjs

@@ -8,4 +8,4 @@ Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () {
 var fetch_js_1 = require("./singletons/fetch.cjs");
 Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true, get: function () { return fetch_js_1.overrideFetchImplementation; } });
 // Update using yarn bump-version
-exports.__version__ = "0.2.14-rc.0";
+exports.__version__ = "0.2.14-rc.2";

package/dist/index.d.ts

@@ -2,4 +2,4 @@ export { Client, type ClientConfig, type LangSmithTracingClientInterface, } from
 export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
-export declare const __version__ = "0.2.14-rc.0";
+export declare const __version__ = "0.2.14-rc.2";

package/dist/index.js

@@ -2,4 +2,4 @@ export { Client, } from "./client.js";
 export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 // Update using yarn bump-version
-export const __version__ = "0.2.14-rc.0";
+export const __version__ = "0.2.14-rc.2";

package/dist/jest/globals.d.ts

@@ -6,10 +6,13 @@ export declare const jestAsyncLocalStorageInstance: AsyncLocalStorage<{
     dataset?: Dataset | undefined;
     examples?: (Example & {
         inputHash: string;
+        outputHash: string;
     })[] | undefined;
     createdAt: string;
     project?: TracerSession | undefined;
     currentExample?: Partial<Example> | undefined;
-    client?: Client | undefined;
+    client: Client;
+    suiteUuid: string;
+    suiteName: string;
 }>;
 export declare function trackingEnabled(): boolean;

package/dist/jest/index.cjs

@@ -1,4 +1,6 @@
 "use strict";
+/* eslint-disable import/no-extraneous-dependencies */
+/* eslint-disable @typescript-eslint/no-namespace */
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -44,55 +46,75 @@ async function _createProject(client, datasetId) {
     throw new Error("Could not generate a unique experiment name within 10 attempts." +
         " Please try again.");
 }
+const setupPromises = new Map();
+async function runDatasetSetup(testClient, datasetName) {
+    let storageValue;
+    if (!(0, globals_js_1.trackingEnabled)()) {
+        storageValue = {
+            createdAt: new Date().toISOString(),
+        };
+    }
+    else {
+        let dataset;
+        try {
+            dataset = await testClient.readDataset({
+                datasetName,
+            });
+        }
+        catch (e) {
+            if (e.message.includes("not found")) {
+                dataset = await testClient.createDataset(datasetName, {
+                    description: `Dataset for unit tests created on ${new Date().toISOString()}`,
+                });
+            }
+            else {
+                throw e;
+            }
+        }
+        const examplesList = testClient.listExamples({
+            datasetName,
+        });
+        const examples = [];
+        for await (const example of examplesList) {
+            const inputHash = crypto_1.default
+                .createHash("sha256")
+                .update(JSON.stringify(example.inputs))
+                .digest("hex");
+            const outputHash = crypto_1.default
+                .createHash("sha256")
+                .update(JSON.stringify(example.inputs))
+                .digest("hex");
+            examples.push({ ...example, inputHash, outputHash });
+        }
+        const project = await _createProject(testClient, dataset.id);
+        storageValue = {
+            dataset,
+            examples,
+            project,
+            client: testClient,
+        };
+    }
+    return storageValue;
+}
 function wrapDescribeMethod(method) {
     return function (datasetName, fn, config) {
-        const testClient = config?.client ?? run_trees_js_1.RunTree.getSharedClient();
         return method(datasetName, () => {
-            (0, globals_1.beforeAll)(async () => {
-                if ((0, globals_js_1.trackingEnabled)()) {
-                    globals_js_1.jestAsyncLocalStorageInstance.enterWith({
-                        createdAt: new Date().toISOString(),
-                    });
-                }
-                else {
-                    let dataset;
-                    try {
-                        dataset = await testClient.readDataset({
-                            datasetName,
-                        });
-                    }
-                    catch (e) {
-                        if (e.message.includes("not found")) {
-                            dataset = await testClient.createDataset(datasetName, {
-                                description: `Dataset for unit tests created on ${new Date().toISOString()}`,
-                            });
-                        }
-                        else {
-                            throw e;
-                        }
-                    }
-                    const examplesList = testClient.listExamples({
-                        datasetName,
-                    });
-                    const examples = [];
-                    for await (const example of examplesList) {
-                        const inputHash = crypto_1.default
-                            .createHash("sha256")
-                            .update(JSON.stringify(example.inputs))
-                            .digest("hex");
-                        examples.push({ ...example, inputHash });
-                    }
-                    const project = await _createProject(testClient, dataset.id);
-                    globals_js_1.jestAsyncLocalStorageInstance.enterWith({
-                        dataset,
-                        examples,
-                        createdAt: new Date().toISOString(),
-                        project,
-                        client: testClient,
-                    });
-                }
-            });
-            fn();
+            const suiteUuid = (0, uuid_1.v4)();
+            /**
+             * We cannot rely on setting AsyncLocalStorage in beforeAll or beforeEach,
+             * due to https://github.com/jestjs/jest/issues/13653 and needing to use
+             * the janky .enterWith.
+             *
+             * We also cannot do async setup in describe due to Jest restrictions.
+             * However, .run works and since the below function does not contain synchronously,
+             * it works.
+             */
+            void globals_js_1.jestAsyncLocalStorageInstance.run({
+                suiteUuid,
+                suiteName: datasetName,
+                client: config?.client ?? run_trees_js_1.RunTree.getSharedClient(),
+                createdAt: new Date().toISOString(),
+            }, fn);
         });
     };
 }
@@ -102,37 +124,62 @@ const lsDescribe = Object.assign(wrapDescribeMethod(globals_1.describe), {
 });
 function wrapTestMethod(method) {
     return function (params, config) {
+        const context = globals_js_1.jestAsyncLocalStorageInstance.getStore();
+        // This typing is wrong, but necessary to avoid lint errors
+        // eslint-disable-next-line @typescript-eslint/no-misused-promises
         return async function (...args) {
             return method(args[0], async () => {
+                if (context === undefined) {
+                    throw new Error(`Could not retrieve test context.\nPlease make sure you have tracing enabled and you are wrapping all of your test cases in an "ls.describe()" function.`);
+                }
+                // Because of https://github.com/jestjs/jest/issues/13653, we have to do asynchronous setup
+                // within the test itself
+                if (!setupPromises.get(context.suiteUuid)) {
+                    setupPromises.set(context.suiteUuid, runDatasetSetup(context.client, context.suiteName));
+                }
+                const { examples, dataset, createdAt, project, client } = await setupPromises.get(context.suiteUuid);
                 const testInput = typeof params === "string" ? {} : params.inputs;
                 const testOutput = typeof params === "string" ? {} : params.outputs;
                 const inputHash = crypto_1.default
                     .createHash("sha256")
                     .update(JSON.stringify(testInput))
                     .digest("hex");
-                const context = globals_js_1.jestAsyncLocalStorageInstance.getStore();
-                if (context === undefined) {
-                    throw new Error(`Could not retrieve test context.\nPlease make sure you have tracing enabled and you are wrapping all of your test cases in an "ls.describe()" function.`);
-                }
-                const { examples, dataset, createdAt, project, client } = context;
+                const outputHash = crypto_1.default
+                    .createHash("sha256")
+                    .update(JSON.stringify(testOutput))
+                    .digest("hex");
                 if ((0, globals_js_1.trackingEnabled)()) {
-                    if (examples === undefined ||
-                        dataset === undefined ||
-                        project === undefined ||
-                        client === undefined) {
-                        throw new Error("Failed to initialize test tracking. Please contact us for help.");
+                    const missingFields = [];
+                    if (examples === undefined) {
+                        missingFields.push("examples");
+                    }
+                    if (dataset === undefined) {
+                        missingFields.push("dataset");
+                    }
+                    if (project === undefined) {
+                        missingFields.push("project");
+                    }
+                    if (client === undefined) {
+                        missingFields.push("client");
+                    }
+                    if (missingFields.length > 0) {
+                        throw new Error(`Failed to initialize test tracking: Could not identify ${missingFields
+                            .map((field) => `"${field}"`)
+                            .join(", ")} while syncing to LangSmith. Please contact us for help.`);
                     }
                     const testClient = config?.client ?? client;
                     let example = (examples ?? []).find((example) => {
-                        return example.inputHash === inputHash;
+                        return (example.inputHash === inputHash &&
+                            example.outputHash === outputHash);
                     });
                     if (example === undefined) {
                         const newExample = await testClient.createExample(testInput, testOutput, {
                             datasetId: dataset?.id,
                             createdAt: new Date(createdAt ?? new Date()),
                         });
-                        example = { ...newExample, inputHash };
+                        example = { ...newExample, inputHash, outputHash };
                     }
+                    // .enterWith is OK here
                     globals_js_1.jestAsyncLocalStorageInstance.enterWith({
                         ...context,
                         currentExample: example,
@@ -163,6 +210,7 @@ function wrapTestMethod(method) {
                     await testClient.awaitPendingTraceBatches();
                 }
                 else {
+                    // .enterWith is OK here
                     globals_js_1.jestAsyncLocalStorageInstance.enterWith({
                         ...context,
                         currentExample: { inputs: testInput, outputs: testOutput },

package/dist/jest/index.d.ts

@@ -4,14 +4,14 @@ import type { SimpleEvaluator } from "./vendor/gradedBy.js";
 declare global {
     namespace jest {
         interface AsymmetricMatchers {
-            toBeRelativeCloseTo(expected: string, options: any): void;
-            toBeAbsoluteCloseTo(expected: string, options: any): void;
-            toBeSemanticCloseTo(expected: string, options: any): Promise<void>;
+            toBeRelativeCloseTo(expected: string, options?: any): void;
+            toBeAbsoluteCloseTo(expected: string, options?: any): void;
+            toBeSemanticCloseTo(expected: string, options?: any): Promise<void>;
         }
         interface Matchers<R> {
-            toBeRelativeCloseTo(expected: string, options: any): R;
-            toBeAbsoluteCloseTo(expected: string, options: any): R;
-            toBeSemanticCloseTo(expected: string, options: any): Promise<R>;
+            toBeRelativeCloseTo(expected: string, options?: any): R;
+            toBeAbsoluteCloseTo(expected: string, options?: any): R;
+            toBeSemanticCloseTo(expected: string, options?: any): Promise<R>;
             gradedBy(evaluator: SimpleEvaluator): jest.Matchers<Promise<R>> & {
                 not: jest.Matchers<Promise<R>>;
                 resolves: jest.Matchers<Promise<R>>;

package/dist/jest/index.js

@@ -1,4 +1,6 @@
-import { expect, test, describe, beforeAll } from "@jest/globals";
+/* eslint-disable import/no-extraneous-dependencies */
+/* eslint-disable @typescript-eslint/no-namespace */
+import { expect, test, describe } from "@jest/globals";
 import crypto from "crypto";
 import { v4 } from "uuid";
 import { traceable } from "../traceable.js";
@@ -39,55 +41,75 @@ async function _createProject(client, datasetId) {
     throw new Error("Could not generate a unique experiment name within 10 attempts." +
         " Please try again.");
 }
+const setupPromises = new Map();
+async function runDatasetSetup(testClient, datasetName) {
+    let storageValue;
+    if (!trackingEnabled()) {
+        storageValue = {
+            createdAt: new Date().toISOString(),
+        };
+    }
+    else {
+        let dataset;
+        try {
+            dataset = await testClient.readDataset({
+                datasetName,
+            });
+        }
+        catch (e) {
+            if (e.message.includes("not found")) {
+                dataset = await testClient.createDataset(datasetName, {
+                    description: `Dataset for unit tests created on ${new Date().toISOString()}`,
+                });
+            }
+            else {
+                throw e;
+            }
+        }
+        const examplesList = testClient.listExamples({
+            datasetName,
+        });
+        const examples = [];
+        for await (const example of examplesList) {
+            const inputHash = crypto
+                .createHash("sha256")
+                .update(JSON.stringify(example.inputs))
+                .digest("hex");
+            const outputHash = crypto
+                .createHash("sha256")
+                .update(JSON.stringify(example.inputs))
+                .digest("hex");
+            examples.push({ ...example, inputHash, outputHash });
+        }
+        const project = await _createProject(testClient, dataset.id);
+        storageValue = {
+            dataset,
+            examples,
+            project,
+            client: testClient,
+        };
+    }
+    return storageValue;
+}
 function wrapDescribeMethod(method) {
     return function (datasetName, fn, config) {
-        const testClient = config?.client ?? RunTree.getSharedClient();
         return method(datasetName, () => {
-            beforeAll(async () => {
-                if (trackingEnabled()) {
-                    jestAsyncLocalStorageInstance.enterWith({
-                        createdAt: new Date().toISOString(),
-                    });
-                }
-                else {
-                    let dataset;
-                    try {
-                        dataset = await testClient.readDataset({
-                            datasetName,
-                        });
-                    }
-                    catch (e) {
-                        if (e.message.includes("not found")) {
-                            dataset = await testClient.createDataset(datasetName, {
-                                description: `Dataset for unit tests created on ${new Date().toISOString()}`,
-                            });
-                        }
-                        else {
-                            throw e;
-                        }
-                    }
-                    const examplesList = testClient.listExamples({
-                        datasetName,
-                    });
-                    const examples = [];
-                    for await (const example of examplesList) {
-                        const inputHash = crypto
-                            .createHash("sha256")
-                            .update(JSON.stringify(example.inputs))
-                            .digest("hex");
-                        examples.push({ ...example, inputHash });
-                    }
-                    const project = await _createProject(testClient, dataset.id);
-                    jestAsyncLocalStorageInstance.enterWith({
-                        dataset,
-                        examples,
-                        createdAt: new Date().toISOString(),
-                        project,
-                        client: testClient,
-                    });
-                }
-            });
-            fn();
+            const suiteUuid = v4();
+            /**
+             * We cannot rely on setting AsyncLocalStorage in beforeAll or beforeEach,
+             * due to https://github.com/jestjs/jest/issues/13653 and needing to use
+             * the janky .enterWith.
+             *
+             * We also cannot do async setup in describe due to Jest restrictions.
+             * However, .run works and since the below function does not contain synchronously,
+             * it works.
+             */
+            void jestAsyncLocalStorageInstance.run({
+                suiteUuid,
+                suiteName: datasetName,
+                client: config?.client ?? RunTree.getSharedClient(),
+                createdAt: new Date().toISOString(),
+            }, fn);
         });
     };
 }
@@ -97,37 +119,62 @@ const lsDescribe = Object.assign(wrapDescribeMethod(describe), {
 });
 function wrapTestMethod(method) {
     return function (params, config) {
+        const context = jestAsyncLocalStorageInstance.getStore();
+        // This typing is wrong, but necessary to avoid lint errors
+        // eslint-disable-next-line @typescript-eslint/no-misused-promises
         return async function (...args) {
             return method(args[0], async () => {
+                if (context === undefined) {
+                    throw new Error(`Could not retrieve test context.\nPlease make sure you have tracing enabled and you are wrapping all of your test cases in an "ls.describe()" function.`);
+                }
+                // Because of https://github.com/jestjs/jest/issues/13653, we have to do asynchronous setup
+                // within the test itself
+                if (!setupPromises.get(context.suiteUuid)) {
+                    setupPromises.set(context.suiteUuid, runDatasetSetup(context.client, context.suiteName));
+                }
+                const { examples, dataset, createdAt, project, client } = await setupPromises.get(context.suiteUuid);
                 const testInput = typeof params === "string" ? {} : params.inputs;
                 const testOutput = typeof params === "string" ? {} : params.outputs;
                 const inputHash = crypto
                     .createHash("sha256")
                     .update(JSON.stringify(testInput))
                     .digest("hex");
-                const context = jestAsyncLocalStorageInstance.getStore();
-                if (context === undefined) {
-                    throw new Error(`Could not retrieve test context.\nPlease make sure you have tracing enabled and you are wrapping all of your test cases in an "ls.describe()" function.`);
-                }
-                const { examples, dataset, createdAt, project, client } = context;
+                const outputHash = crypto
+                    .createHash("sha256")
+                    .update(JSON.stringify(testOutput))
+                    .digest("hex");
                 if (trackingEnabled()) {
-                    if (examples === undefined ||
-                        dataset === undefined ||
-                        project === undefined ||
-                        client === undefined) {
-                        throw new Error("Failed to initialize test tracking. Please contact us for help.");
+                    const missingFields = [];
+                    if (examples === undefined) {
+                        missingFields.push("examples");
+                    }
+                    if (dataset === undefined) {
+                        missingFields.push("dataset");
+                    }
+                    if (project === undefined) {
+                        missingFields.push("project");
+                    }
+                    if (client === undefined) {
+                        missingFields.push("client");
+                    }
+                    if (missingFields.length > 0) {
+                        throw new Error(`Failed to initialize test tracking: Could not identify ${missingFields
+                            .map((field) => `"${field}"`)
+                            .join(", ")} while syncing to LangSmith. Please contact us for help.`);
                     }
                     const testClient = config?.client ?? client;
                     let example = (examples ?? []).find((example) => {
-                        return example.inputHash === inputHash;
+                        return (example.inputHash === inputHash &&
+                            example.outputHash === outputHash);
                     });
                     if (example === undefined) {
                         const newExample = await testClient.createExample(testInput, testOutput, {
                             datasetId: dataset?.id,
                             createdAt: new Date(createdAt ?? new Date()),
                         });
-                        example = { ...newExample, inputHash };
+                        example = { ...newExample, inputHash, outputHash };
                     }
+                    // .enterWith is OK here
                     jestAsyncLocalStorageInstance.enterWith({
                         ...context,
                         currentExample: example,
@@ -158,6 +205,7 @@ function wrapTestMethod(method) {
                     await testClient.awaitPendingTraceBatches();
                 }
                 else {
+                    // .enterWith is OK here
                     jestAsyncLocalStorageInstance.enterWith({
                         ...context,
                         currentExample: { inputs: testInput, outputs: testOutput },

package/dist/jest/vendor/chain.cjs

@@ -71,10 +71,6 @@ function expectWithGradedBy(expect) {
     const expectProxy = Object.assign((...args) => addGradedBy(expect(...args), args), // partially apply expect to get all matchers and chain them
     expect // clone additional properties on expect
     );
-    // expectProxy.extend = (o: any) => {
-    //   expect.extend(o); // add new matchers to expect
-    //   expectProxy = Object.assign(expectProxy, expect); // clone new asymmetric matchers
-    // };
     return expectProxy;
 }
 exports.default = expectWithGradedBy;

package/dist/jest/vendor/chain.js

@@ -69,10 +69,6 @@ export default function expectWithGradedBy(expect) {
     const expectProxy = Object.assign((...args) => addGradedBy(expect(...args), args), // partially apply expect to get all matchers and chain them
     expect // clone additional properties on expect
     );
-    // expectProxy.extend = (o: any) => {
-    //   expect.extend(o); // add new matchers to expect
-    //   expectProxy = Object.assign(expectProxy, expect); // clone new asymmetric matchers
-    // };
     return expectProxy;
 }
 globalThis.expect = expectWithGradedBy(globalThis.expect);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.2.14-rc.0",
+  "version": "0.2.14-rc.2",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [