langsmith 0.3.19 → 0.3.21

package/dist/client.cjs

@@ -435,7 +435,7 @@ class Client {
         }
         return headers;
     }
-    processInputs(inputs) {
+    async processInputs(inputs) {
         if (this.hideInputs === false) {
             return inputs;
         }
@@ -447,7 +447,7 @@ class Client {
         }
         return inputs;
     }
-    processOutputs(outputs) {
+    async processOutputs(outputs) {
         if (this.hideOutputs === false) {
             return outputs;
         }
@@ -459,13 +459,13 @@ class Client {
         }
         return outputs;
     }
-    prepareRunCreateOrUpdateInputs(run) {
+    async prepareRunCreateOrUpdateInputs(run) {
         const runParams = { ...run };
         if (runParams.inputs !== undefined) {
-            runParams.inputs = this.processInputs(runParams.inputs);
+            runParams.inputs = await this.processInputs(runParams.inputs);
         }
         if (runParams.outputs !== undefined) {
-            runParams.outputs = this.processOutputs(runParams.outputs);
+            runParams.outputs = await this.processOutputs(runParams.outputs);
         }
         return runParams;
     }
@@ -718,7 +718,7 @@ class Client {
         const headers = { ...this.headers, "Content-Type": "application/json" };
         const session_name = run.project_name;
         delete run.project_name;
-        const runCreate = this.prepareRunCreateOrUpdateInputs({
+        const runCreate = await this.prepareRunCreateOrUpdateInputs({
             session_name,
             ...run,
             start_time: run.start_time ?? Date.now(),
@@ -750,8 +750,8 @@ class Client {
         if (runCreates === undefined && runUpdates === undefined) {
             return;
         }
-        let preparedCreateParams = runCreates?.map((create) => this.prepareRunCreateOrUpdateInputs(create)) ?? [];
-        let preparedUpdateParams = runUpdates?.map((update) => this.prepareRunCreateOrUpdateInputs(update)) ?? [];
+        let preparedCreateParams = await Promise.all(runCreates?.map((create) => this.prepareRunCreateOrUpdateInputs(create)) ?? []);
+        let preparedUpdateParams = await Promise.all(runUpdates?.map((update) => this.prepareRunCreateOrUpdateInputs(update)) ?? []);
         if (preparedCreateParams.length > 0 && preparedUpdateParams.length > 0) {
             const createById = preparedCreateParams.reduce((params, run) => {
                 if (!run.id) {
@@ -827,7 +827,7 @@ class Client {
         const allAttachments = {};
         let preparedCreateParams = [];
         for (const create of runCreates ?? []) {
-            const preparedCreate = this.prepareRunCreateOrUpdateInputs(create);
+            const preparedCreate = await this.prepareRunCreateOrUpdateInputs(create);
             if (preparedCreate.id !== undefined &&
                 preparedCreate.attachments !== undefined) {
                 allAttachments[preparedCreate.id] = preparedCreate.attachments;
@@ -837,7 +837,7 @@ class Client {
         }
         let preparedUpdateParams = [];
         for (const update of runUpdates ?? []) {
-            preparedUpdateParams.push(this.prepareRunCreateOrUpdateInputs(update));
+            preparedUpdateParams.push(await this.prepareRunCreateOrUpdateInputs(update));
         }
         // require trace_id and dotted_order
         const invalidRunCreate = preparedCreateParams.find((runCreate) => {
@@ -989,10 +989,10 @@ class Client {
     async updateRun(runId, run) {
         (0, _uuid_js_1.assertUuid)(runId);
         if (run.inputs) {
-            run.inputs = this.processInputs(run.inputs);
+            run.inputs = await this.processInputs(run.inputs);
         }
         if (run.outputs) {
-            run.outputs = this.processOutputs(run.outputs);
+            run.outputs = await this.processOutputs(run.outputs);
         }
         // TODO: Untangle types
         const data = { ...run, id: runId };

package/dist/client.d.ts

@@ -7,9 +7,9 @@ export interface ClientConfig {
     callerOptions?: AsyncCallerParams;
     timeout_ms?: number;
     webUrl?: string;
-    anonymizer?: (values: KVMap) => KVMap;
-    hideInputs?: boolean | ((inputs: KVMap) => KVMap);
-    hideOutputs?: boolean | ((outputs: KVMap) => KVMap);
+    anonymizer?: (values: KVMap) => KVMap | Promise<KVMap>;
+    hideInputs?: boolean | ((inputs: KVMap) => KVMap | Promise<KVMap>);
+    hideOutputs?: boolean | ((outputs: KVMap) => KVMap | Promise<KVMap>);
     autoBatchTracing?: boolean;
     batchSizeBytesLimit?: number;
     blockOnRootRunFinalization?: boolean;

package/dist/client.js

@@ -397,7 +397,7 @@ export class Client {
         }
         return headers;
     }
-    processInputs(inputs) {
+    async processInputs(inputs) {
         if (this.hideInputs === false) {
             return inputs;
         }
@@ -409,7 +409,7 @@ export class Client {
         }
         return inputs;
     }
-    processOutputs(outputs) {
+    async processOutputs(outputs) {
         if (this.hideOutputs === false) {
             return outputs;
         }
@@ -421,13 +421,13 @@ export class Client {
         }
         return outputs;
     }
-    prepareRunCreateOrUpdateInputs(run) {
+    async prepareRunCreateOrUpdateInputs(run) {
         const runParams = { ...run };
         if (runParams.inputs !== undefined) {
-            runParams.inputs = this.processInputs(runParams.inputs);
+            runParams.inputs = await this.processInputs(runParams.inputs);
         }
         if (runParams.outputs !== undefined) {
-            runParams.outputs = this.processOutputs(runParams.outputs);
+            runParams.outputs = await this.processOutputs(runParams.outputs);
         }
         return runParams;
     }
@@ -680,7 +680,7 @@ export class Client {
         const headers = { ...this.headers, "Content-Type": "application/json" };
         const session_name = run.project_name;
         delete run.project_name;
-        const runCreate = this.prepareRunCreateOrUpdateInputs({
+        const runCreate = await this.prepareRunCreateOrUpdateInputs({
             session_name,
             ...run,
             start_time: run.start_time ?? Date.now(),
@@ -712,8 +712,8 @@ export class Client {
         if (runCreates === undefined && runUpdates === undefined) {
             return;
         }
-        let preparedCreateParams = runCreates?.map((create) => this.prepareRunCreateOrUpdateInputs(create)) ?? [];
-        let preparedUpdateParams = runUpdates?.map((update) => this.prepareRunCreateOrUpdateInputs(update)) ?? [];
+        let preparedCreateParams = await Promise.all(runCreates?.map((create) => this.prepareRunCreateOrUpdateInputs(create)) ?? []);
+        let preparedUpdateParams = await Promise.all(runUpdates?.map((update) => this.prepareRunCreateOrUpdateInputs(update)) ?? []);
         if (preparedCreateParams.length > 0 && preparedUpdateParams.length > 0) {
             const createById = preparedCreateParams.reduce((params, run) => {
                 if (!run.id) {
@@ -789,7 +789,7 @@ export class Client {
         const allAttachments = {};
         let preparedCreateParams = [];
         for (const create of runCreates ?? []) {
-            const preparedCreate = this.prepareRunCreateOrUpdateInputs(create);
+            const preparedCreate = await this.prepareRunCreateOrUpdateInputs(create);
             if (preparedCreate.id !== undefined &&
                 preparedCreate.attachments !== undefined) {
                 allAttachments[preparedCreate.id] = preparedCreate.attachments;
@@ -799,7 +799,7 @@ export class Client {
         }
         let preparedUpdateParams = [];
         for (const update of runUpdates ?? []) {
-            preparedUpdateParams.push(this.prepareRunCreateOrUpdateInputs(update));
+            preparedUpdateParams.push(await this.prepareRunCreateOrUpdateInputs(update));
         }
         // require trace_id and dotted_order
         const invalidRunCreate = preparedCreateParams.find((runCreate) => {
@@ -951,10 +951,10 @@ export class Client {
     async updateRun(runId, run) {
         assertUuid(runId);
         if (run.inputs) {
-            run.inputs = this.processInputs(run.inputs);
+            run.inputs = await this.processInputs(run.inputs);
         }
         if (run.outputs) {
-            run.outputs = this.processOutputs(run.outputs);
+            run.outputs = await this.processOutputs(run.outputs);
         }
         // TODO: Untangle types
         const data = { ...run, id: runId };

package/dist/index.cjs

@@ -8,4 +8,4 @@ Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () {
 var fetch_js_1 = require("./singletons/fetch.cjs");
 Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true, get: function () { return fetch_js_1.overrideFetchImplementation; } });
 // Update using yarn bump-version
-exports.__version__ = "0.3.19";
+exports.__version__ = "0.3.21";

package/dist/index.d.ts

@@ -2,4 +2,4 @@ export { Client, type ClientConfig, type LangSmithTracingClientInterface, } from
 export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
-export declare const __version__ = "0.3.19";
+export declare const __version__ = "0.3.21";

package/dist/index.js

@@ -2,4 +2,4 @@ export { Client, } from "./client.js";
 export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 // Update using yarn bump-version
-export const __version__ = "0.3.19";
+export const __version__ = "0.3.21";

package/dist/jest/index.d.ts

@@ -85,6 +85,42 @@ declare const test: (<I extends Record<string, any> = Record<string, any>, O ext
             referenceOutputs?: O;
         } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
     };
+    concurrent: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+        inputs: I;
+        referenceOutputs?: O;
+    } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+        each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        };
+        skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        };
+    };
     each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
         inputs: I;
         referenceOutputs?: O;
@@ -120,6 +156,42 @@ declare const test: (<I extends Record<string, any> = Record<string, any>, O ext
             referenceOutputs?: O;
         } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
     };
+    concurrent: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+        inputs: I;
+        referenceOutputs?: O;
+    } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+        each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        };
+        skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        };
+    };
     each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
         inputs: I;
         referenceOutputs?: O;
@@ -130,6 +202,7 @@ declare const test: (<I extends Record<string, any> = Record<string, any>, O ext
 }, describe: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper & {
     only: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper;
     skip: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper;
+    concurrent: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper;
 }, expect: jest.Expect;
 export { 
 /**

package/dist/utils/jestlike/globals.d.ts

@@ -19,6 +19,7 @@ export type TestWrapperAsyncLocalStorageData = {
     suiteUuid: string;
     suiteName: string;
     testRootRunTree?: RunTree;
+    setupPromise?: Promise<void>;
 };
 export declare const testWrapperAsyncLocalStorageInstance: AsyncLocalStorage<TestWrapperAsyncLocalStorageData>;
 export declare function trackingEnabled(context: TestWrapperAsyncLocalStorageData): boolean;

package/dist/utils/jestlike/index.cjs

@@ -229,13 +229,27 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         }
         return storageValue;
     }
-    function wrapDescribeMethod(method) {
+    function wrapDescribeMethod(method, methodName) {
         if ((0, env_js_1.isJsDom)()) {
             console.error(`[LANGSMITH]: You seem to be using a jsdom environment. This is not supported and you may experience unexpected behavior. Please set the "environment" or "testEnvironment" field in your test config file to "node".`);
         }
         return function (testSuiteName, fn, experimentConfig) {
+            if (typeof method !== "function") {
+                throw new Error(`"${methodName}" is not supported by your test runner.`);
+            }
+            if (globals_js_1.testWrapperAsyncLocalStorageInstance.getStore() !== undefined) {
+                throw new Error([
+                    `You seem to be nesting an ls.describe block named "${testSuiteName}" inside another ls.describe block.`,
+                    "This is not supported because each ls.describe block corresponds to a LangSmith dataset.",
+                    "To logically group tests, nest the native Jest or Vitest describe methods instead.",
+                ].join("\n"));
+            }
             const client = experimentConfig?.client ?? globals_js_1.DEFAULT_TEST_CLIENT;
             const suiteName = experimentConfig?.testSuiteName ?? testSuiteName;
+            let setupPromiseResolver;
+            const setupPromise = new Promise((resolve) => {
+                setupPromiseResolver = resolve;
+            });
             return method(suiteName, () => {
                 const startTime = new Date();
                 const suiteUuid = (0, uuid_1.v4)();
@@ -269,10 +283,12 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                         metadata: suiteMetadata,
                     },
                     enableTestTracking: experimentConfig?.enableTestTracking,
+                    setupPromise,
                 };
                 beforeAll(async () => {
                     const storageValue = await runDatasetSetup(context);
                     datasetSetupInfo.set(suiteUuid, storageValue);
+                    setupPromiseResolver();
                 });
                 afterAll(async () => {
                     await Promise.all([
@@ -353,9 +369,10 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
             });
         };
     }
-    const lsDescribe = Object.assign(wrapDescribeMethod(describe), {
-        only: wrapDescribeMethod(describe.only),
-        skip: wrapDescribeMethod(describe.skip),
+    const lsDescribe = Object.assign(wrapDescribeMethod(describe, "describe"), {
+        only: wrapDescribeMethod(describe.only, "describe.only"),
+        skip: wrapDescribeMethod(describe.skip, "describe.skip"),
+        concurrent: wrapDescribeMethod(describe.concurrent, "describe.concurrent"),
     });
     function wrapTestMethod(method) {
         return function (name, lsParams, testFn, timeout) {
@@ -374,7 +391,12 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                 // Jest will not group tests under the same "describe" group if you await the test and
                 // total runs is greater than 1.
                 const resultsPath = path.join(os.tmpdir(), "langsmith_test_results", `${testUuid}.json`);
-                void method(`${name}${totalRuns > 1 ? `, run ${i}` : ""}${constants_js_1.TEST_ID_DELIMITER}${testUuid}`, async () => {
+                void method(`${name}${totalRuns > 1 ? `, run ${i}` : ""}${constants_js_1.TEST_ID_DELIMITER}${testUuid}`, async (...args) => {
+                    // Jest will magically introspect args and pass a "done" callback if
+                    // we use a non-spread parameter. To obtain and pass Vitest test context
+                    // through into the test function, we must therefore refer to Vitest
+                    // args using this signature
+                    const jestlikeArgs = args[0];
                     if (context === undefined) {
                         throw new Error([
                             `Could not retrieve test context.`,
@@ -382,6 +404,11 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                             `See this page for more information: https://docs.smith.langchain.com/evaluation/how_to_guides/vitest_jest`,
                         ].join("\n"));
                     }
+                    // Jest .concurrent is super buggy and doesn't wait for beforeAll to complete
+                    // before running test functions, so we need to wait for the setup promise
+                    // to resolve before we can continue.
+                    // Seee https://github.com/jestjs/jest/issues/4281
+                    await context.setupPromise;
                     if (!datasetSetupInfo.get(context.suiteUuid)) {
                         throw new Error("Dataset failed to initialize. Please check your LangSmith environment variables.");
                     }
@@ -414,11 +441,13 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                                 throw new Error("Could not identify test context after setting test root run tree. Please contact us for help.");
                             }
                             try {
-                                const res = await testFn({
+                                const res = await testFn(Object.assign(typeof jestlikeArgs === "object" && jestlikeArgs != null
+                                    ? jestlikeArgs
+                                    : {}, {
                                     ...rest,
                                     inputs: testInput,
                                     referenceOutputs: testOutput,
-                                });
+                                }));
                                 (0, globals_js_1._logTestFeedback)({
                                     exampleId,
                                     feedback: { key: "pass", score: true },
@@ -582,6 +611,16 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         }
         return eachMethod;
     }
+    // Roughly mirrors: https://jestjs.io/docs/api#methods
+    const concurrentMethod = Object.assign(wrapTestMethod(test.concurrent), {
+        each: createEachMethod(test.concurrent),
+        only: Object.assign(wrapTestMethod(test.concurrent.only), {
+            each: createEachMethod(test.concurrent.only),
+        }),
+        skip: Object.assign(wrapTestMethod(test.concurrent.skip), {
+            each: createEachMethod(test.concurrent.skip),
+        }),
+    });
     const lsTest = Object.assign(wrapTestMethod(test), {
         only: Object.assign(wrapTestMethod(test.only), {
             each: createEachMethod(test.only),
@@ -589,6 +628,7 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         skip: Object.assign(wrapTestMethod(test.skip), {
             each: createEachMethod(test.skip),
         }),
+        concurrent: concurrentMethod,
         each: createEachMethod(test),
     });
     const wrappedExpect = (0, chain_js_1.wrapExpect)(expect);

package/dist/utils/jestlike/index.d.ts

@@ -36,6 +36,42 @@ export declare function generateWrapperFromJestlikeMethods(methods: Record<strin
                 referenceOutputs?: O;
             } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
         };
+        concurrent: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends KVMap, O extends KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+            only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+                each: <I extends KVMap, O extends KVMap>(table: ({
+                    inputs: I;
+                    referenceOutputs?: O;
+                } & Record<string, any>)[], config?: LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                    inputs: I;
+                    referenceOutputs?: O;
+                } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+            };
+            skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+                each: <I extends KVMap, O extends KVMap>(table: ({
+                    inputs: I;
+                    referenceOutputs?: O;
+                } & Record<string, any>)[], config?: LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                    inputs: I;
+                    referenceOutputs?: O;
+                } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+            };
+        };
         each: <I extends KVMap, O extends KVMap>(table: ({
             inputs: I;
             referenceOutputs?: O;
@@ -72,6 +108,42 @@ export declare function generateWrapperFromJestlikeMethods(methods: Record<strin
                 referenceOutputs?: O;
             } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
         };
+        concurrent: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends KVMap, O extends KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+            only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+                each: <I extends KVMap, O extends KVMap>(table: ({
+                    inputs: I;
+                    referenceOutputs?: O;
+                } & Record<string, any>)[], config?: LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                    inputs: I;
+                    referenceOutputs?: O;
+                } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+            };
+            skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+                each: <I extends KVMap, O extends KVMap>(table: ({
+                    inputs: I;
+                    referenceOutputs?: O;
+                } & Record<string, any>)[], config?: LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                    inputs: I;
+                    referenceOutputs?: O;
+                } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+            };
+        };
         each: <I extends KVMap, O extends KVMap>(table: ({
             inputs: I;
             referenceOutputs?: O;
@@ -83,6 +155,7 @@ export declare function generateWrapperFromJestlikeMethods(methods: Record<strin
     describe: LangSmithJestlikeDescribeWrapper & {
         only: LangSmithJestlikeDescribeWrapper;
         skip: LangSmithJestlikeDescribeWrapper;
+        concurrent: LangSmithJestlikeDescribeWrapper;
     };
     expect: jest.Expect;
     toBeRelativeCloseTo: typeof toBeRelativeCloseTo;

package/dist/utils/jestlike/index.js

@@ -182,13 +182,27 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         }
         return storageValue;
     }
-    function wrapDescribeMethod(method) {
+    function wrapDescribeMethod(method, methodName) {
         if (isJsDom()) {
             console.error(`[LANGSMITH]: You seem to be using a jsdom environment. This is not supported and you may experience unexpected behavior. Please set the "environment" or "testEnvironment" field in your test config file to "node".`);
         }
         return function (testSuiteName, fn, experimentConfig) {
+            if (typeof method !== "function") {
+                throw new Error(`"${methodName}" is not supported by your test runner.`);
+            }
+            if (testWrapperAsyncLocalStorageInstance.getStore() !== undefined) {
+                throw new Error([
+                    `You seem to be nesting an ls.describe block named "${testSuiteName}" inside another ls.describe block.`,
+                    "This is not supported because each ls.describe block corresponds to a LangSmith dataset.",
+                    "To logically group tests, nest the native Jest or Vitest describe methods instead.",
+                ].join("\n"));
+            }
             const client = experimentConfig?.client ?? DEFAULT_TEST_CLIENT;
             const suiteName = experimentConfig?.testSuiteName ?? testSuiteName;
+            let setupPromiseResolver;
+            const setupPromise = new Promise((resolve) => {
+                setupPromiseResolver = resolve;
+            });
             return method(suiteName, () => {
                 const startTime = new Date();
                 const suiteUuid = v4();
@@ -222,10 +236,12 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                         metadata: suiteMetadata,
                     },
                     enableTestTracking: experimentConfig?.enableTestTracking,
+                    setupPromise,
                 };
                 beforeAll(async () => {
                     const storageValue = await runDatasetSetup(context);
                     datasetSetupInfo.set(suiteUuid, storageValue);
+                    setupPromiseResolver();
                 });
                 afterAll(async () => {
                     await Promise.all([
@@ -306,9 +322,10 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
             });
         };
     }
-    const lsDescribe = Object.assign(wrapDescribeMethod(describe), {
-        only: wrapDescribeMethod(describe.only),
-        skip: wrapDescribeMethod(describe.skip),
+    const lsDescribe = Object.assign(wrapDescribeMethod(describe, "describe"), {
+        only: wrapDescribeMethod(describe.only, "describe.only"),
+        skip: wrapDescribeMethod(describe.skip, "describe.skip"),
+        concurrent: wrapDescribeMethod(describe.concurrent, "describe.concurrent"),
     });
     function wrapTestMethod(method) {
         return function (name, lsParams, testFn, timeout) {
@@ -327,7 +344,12 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                 // Jest will not group tests under the same "describe" group if you await the test and
                 // total runs is greater than 1.
                 const resultsPath = path.join(os.tmpdir(), "langsmith_test_results", `${testUuid}.json`);
-                void method(`${name}${totalRuns > 1 ? `, run ${i}` : ""}${TEST_ID_DELIMITER}${testUuid}`, async () => {
+                void method(`${name}${totalRuns > 1 ? `, run ${i}` : ""}${TEST_ID_DELIMITER}${testUuid}`, async (...args) => {
+                    // Jest will magically introspect args and pass a "done" callback if
+                    // we use a non-spread parameter. To obtain and pass Vitest test context
+                    // through into the test function, we must therefore refer to Vitest
+                    // args using this signature
+                    const jestlikeArgs = args[0];
                     if (context === undefined) {
                         throw new Error([
                             `Could not retrieve test context.`,
@@ -335,6 +357,11 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                             `See this page for more information: https://docs.smith.langchain.com/evaluation/how_to_guides/vitest_jest`,
                         ].join("\n"));
                     }
+                    // Jest .concurrent is super buggy and doesn't wait for beforeAll to complete
+                    // before running test functions, so we need to wait for the setup promise
+                    // to resolve before we can continue.
+                    // Seee https://github.com/jestjs/jest/issues/4281
+                    await context.setupPromise;
                     if (!datasetSetupInfo.get(context.suiteUuid)) {
                         throw new Error("Dataset failed to initialize. Please check your LangSmith environment variables.");
                     }
@@ -367,11 +394,13 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                                 throw new Error("Could not identify test context after setting test root run tree. Please contact us for help.");
                             }
                             try {
-                                const res = await testFn({
+                                const res = await testFn(Object.assign(typeof jestlikeArgs === "object" && jestlikeArgs != null
+                                    ? jestlikeArgs
+                                    : {}, {
                                     ...rest,
                                     inputs: testInput,
                                     referenceOutputs: testOutput,
-                                });
+                                }));
                                 _logTestFeedback({
                                     exampleId,
                                     feedback: { key: "pass", score: true },
@@ -535,6 +564,16 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         }
         return eachMethod;
     }
+    // Roughly mirrors: https://jestjs.io/docs/api#methods
+    const concurrentMethod = Object.assign(wrapTestMethod(test.concurrent), {
+        each: createEachMethod(test.concurrent),
+        only: Object.assign(wrapTestMethod(test.concurrent.only), {
+            each: createEachMethod(test.concurrent.only),
+        }),
+        skip: Object.assign(wrapTestMethod(test.concurrent.skip), {
+            each: createEachMethod(test.concurrent.skip),
+        }),
+    });
     const lsTest = Object.assign(wrapTestMethod(test), {
         only: Object.assign(wrapTestMethod(test.only), {
             each: createEachMethod(test.only),
@@ -542,6 +581,7 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         skip: Object.assign(wrapTestMethod(test.skip), {
             each: createEachMethod(test.skip),
         }),
+        concurrent: concurrentMethod,
         each: createEachMethod(test),
     });
     const wrappedExpect = wrapExpect(expect);

package/dist/vitest/index.d.ts

@@ -83,6 +83,42 @@ declare const test: (<I extends Record<string, any> = Record<string, any>, O ext
             referenceOutputs?: O;
         } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
     };
+    concurrent: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+        inputs: I;
+        referenceOutputs?: O;
+    } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+        each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        };
+        skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        };
+    };
     each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
         inputs: I;
         referenceOutputs?: O;
@@ -118,6 +154,42 @@ declare const test: (<I extends Record<string, any> = Record<string, any>, O ext
             referenceOutputs?: O;
         } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
     };
+    concurrent: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+        inputs: I;
+        referenceOutputs?: O;
+    } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+        each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        };
+        skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        };
+    };
     each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
         inputs: I;
         referenceOutputs?: O;
@@ -128,6 +200,7 @@ declare const test: (<I extends Record<string, any> = Record<string, any>, O ext
 }, describe: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper & {
     only: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper;
     skip: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper;
+    concurrent: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper;
 }, expect: jest.Expect;
 export { 
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.3.19",
+  "version": "0.3.21",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [