langsmith 0.3.20 → 0.3.21

package/dist/index.cjs

@@ -8,4 +8,4 @@ Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () {
 var fetch_js_1 = require("./singletons/fetch.cjs");
 Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true, get: function () { return fetch_js_1.overrideFetchImplementation; } });
 // Update using yarn bump-version
-exports.__version__ = "0.3.20";
+exports.__version__ = "0.3.21";

package/dist/index.d.ts

@@ -2,4 +2,4 @@ export { Client, type ClientConfig, type LangSmithTracingClientInterface, } from
 export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
-export declare const __version__ = "0.3.20";
+export declare const __version__ = "0.3.21";

package/dist/index.js

@@ -2,4 +2,4 @@ export { Client, } from "./client.js";
 export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 // Update using yarn bump-version
-export const __version__ = "0.3.20";
+export const __version__ = "0.3.21";

package/dist/jest/index.d.ts

@@ -85,6 +85,42 @@ declare const test: (<I extends Record<string, any> = Record<string, any>, O ext
             referenceOutputs?: O;
         } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
     };
+    concurrent: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+        inputs: I;
+        referenceOutputs?: O;
+    } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+        each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        };
+        skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        };
+    };
     each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
         inputs: I;
         referenceOutputs?: O;
@@ -120,6 +156,42 @@ declare const test: (<I extends Record<string, any> = Record<string, any>, O ext
             referenceOutputs?: O;
         } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
     };
+    concurrent: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+        inputs: I;
+        referenceOutputs?: O;
+    } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+        each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        };
+        skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        };
+    };
     each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
         inputs: I;
         referenceOutputs?: O;
@@ -130,6 +202,7 @@ declare const test: (<I extends Record<string, any> = Record<string, any>, O ext
 }, describe: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper & {
     only: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper;
     skip: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper;
+    concurrent: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper;
 }, expect: jest.Expect;
 export { 
 /**

package/dist/utils/jestlike/globals.d.ts

@@ -19,6 +19,7 @@ export type TestWrapperAsyncLocalStorageData = {
     suiteUuid: string;
     suiteName: string;
     testRootRunTree?: RunTree;
+    setupPromise?: Promise<void>;
 };
 export declare const testWrapperAsyncLocalStorageInstance: AsyncLocalStorage<TestWrapperAsyncLocalStorageData>;
 export declare function trackingEnabled(context: TestWrapperAsyncLocalStorageData): boolean;

package/dist/utils/jestlike/index.cjs

@@ -229,13 +229,27 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         }
         return storageValue;
     }
-    function wrapDescribeMethod(method) {
+    function wrapDescribeMethod(method, methodName) {
         if ((0, env_js_1.isJsDom)()) {
             console.error(`[LANGSMITH]: You seem to be using a jsdom environment. This is not supported and you may experience unexpected behavior. Please set the "environment" or "testEnvironment" field in your test config file to "node".`);
         }
         return function (testSuiteName, fn, experimentConfig) {
+            if (typeof method !== "function") {
+                throw new Error(`"${methodName}" is not supported by your test runner.`);
+            }
+            if (globals_js_1.testWrapperAsyncLocalStorageInstance.getStore() !== undefined) {
+                throw new Error([
+                    `You seem to be nesting an ls.describe block named "${testSuiteName}" inside another ls.describe block.`,
+                    "This is not supported because each ls.describe block corresponds to a LangSmith dataset.",
+                    "To logically group tests, nest the native Jest or Vitest describe methods instead.",
+                ].join("\n"));
+            }
             const client = experimentConfig?.client ?? globals_js_1.DEFAULT_TEST_CLIENT;
             const suiteName = experimentConfig?.testSuiteName ?? testSuiteName;
+            let setupPromiseResolver;
+            const setupPromise = new Promise((resolve) => {
+                setupPromiseResolver = resolve;
+            });
             return method(suiteName, () => {
                 const startTime = new Date();
                 const suiteUuid = (0, uuid_1.v4)();
@@ -269,10 +283,12 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                         metadata: suiteMetadata,
                     },
                     enableTestTracking: experimentConfig?.enableTestTracking,
+                    setupPromise,
                 };
                 beforeAll(async () => {
                     const storageValue = await runDatasetSetup(context);
                     datasetSetupInfo.set(suiteUuid, storageValue);
+                    setupPromiseResolver();
                 });
                 afterAll(async () => {
                     await Promise.all([
@@ -353,9 +369,10 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
             });
         };
     }
-    const lsDescribe = Object.assign(wrapDescribeMethod(describe), {
-        only: wrapDescribeMethod(describe.only),
-        skip: wrapDescribeMethod(describe.skip),
+    const lsDescribe = Object.assign(wrapDescribeMethod(describe, "describe"), {
+        only: wrapDescribeMethod(describe.only, "describe.only"),
+        skip: wrapDescribeMethod(describe.skip, "describe.skip"),
+        concurrent: wrapDescribeMethod(describe.concurrent, "describe.concurrent"),
     });
     function wrapTestMethod(method) {
         return function (name, lsParams, testFn, timeout) {
@@ -374,7 +391,12 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                 // Jest will not group tests under the same "describe" group if you await the test and
                 // total runs is greater than 1.
                 const resultsPath = path.join(os.tmpdir(), "langsmith_test_results", `${testUuid}.json`);
-                void method(`${name}${totalRuns > 1 ? `, run ${i}` : ""}${constants_js_1.TEST_ID_DELIMITER}${testUuid}`, async () => {
+                void method(`${name}${totalRuns > 1 ? `, run ${i}` : ""}${constants_js_1.TEST_ID_DELIMITER}${testUuid}`, async (...args) => {
+                    // Jest will magically introspect args and pass a "done" callback if
+                    // we use a non-spread parameter. To obtain and pass Vitest test context
+                    // through into the test function, we must therefore refer to Vitest
+                    // args using this signature
+                    const jestlikeArgs = args[0];
                     if (context === undefined) {
                         throw new Error([
                             `Could not retrieve test context.`,
@@ -382,6 +404,11 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                             `See this page for more information: https://docs.smith.langchain.com/evaluation/how_to_guides/vitest_jest`,
                         ].join("\n"));
                     }
+                    // Jest .concurrent is super buggy and doesn't wait for beforeAll to complete
+                    // before running test functions, so we need to wait for the setup promise
+                    // to resolve before we can continue.
+                    // Seee https://github.com/jestjs/jest/issues/4281
+                    await context.setupPromise;
                     if (!datasetSetupInfo.get(context.suiteUuid)) {
                         throw new Error("Dataset failed to initialize. Please check your LangSmith environment variables.");
                     }
@@ -414,11 +441,13 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                                 throw new Error("Could not identify test context after setting test root run tree. Please contact us for help.");
                             }
                             try {
-                                const res = await testFn({
+                                const res = await testFn(Object.assign(typeof jestlikeArgs === "object" && jestlikeArgs != null
+                                    ? jestlikeArgs
+                                    : {}, {
                                     ...rest,
                                     inputs: testInput,
                                     referenceOutputs: testOutput,
-                                });
+                                }));
                                 (0, globals_js_1._logTestFeedback)({
                                     exampleId,
                                     feedback: { key: "pass", score: true },
@@ -582,6 +611,16 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         }
         return eachMethod;
     }
+    // Roughly mirrors: https://jestjs.io/docs/api#methods
+    const concurrentMethod = Object.assign(wrapTestMethod(test.concurrent), {
+        each: createEachMethod(test.concurrent),
+        only: Object.assign(wrapTestMethod(test.concurrent.only), {
+            each: createEachMethod(test.concurrent.only),
+        }),
+        skip: Object.assign(wrapTestMethod(test.concurrent.skip), {
+            each: createEachMethod(test.concurrent.skip),
+        }),
+    });
     const lsTest = Object.assign(wrapTestMethod(test), {
         only: Object.assign(wrapTestMethod(test.only), {
             each: createEachMethod(test.only),
@@ -589,6 +628,7 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         skip: Object.assign(wrapTestMethod(test.skip), {
             each: createEachMethod(test.skip),
         }),
+        concurrent: concurrentMethod,
         each: createEachMethod(test),
     });
     const wrappedExpect = (0, chain_js_1.wrapExpect)(expect);

package/dist/utils/jestlike/index.d.ts

@@ -36,6 +36,42 @@ export declare function generateWrapperFromJestlikeMethods(methods: Record<strin
                 referenceOutputs?: O;
             } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
         };
+        concurrent: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends KVMap, O extends KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+            only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+                each: <I extends KVMap, O extends KVMap>(table: ({
+                    inputs: I;
+                    referenceOutputs?: O;
+                } & Record<string, any>)[], config?: LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                    inputs: I;
+                    referenceOutputs?: O;
+                } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+            };
+            skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+                each: <I extends KVMap, O extends KVMap>(table: ({
+                    inputs: I;
+                    referenceOutputs?: O;
+                } & Record<string, any>)[], config?: LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                    inputs: I;
+                    referenceOutputs?: O;
+                } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+            };
+        };
         each: <I extends KVMap, O extends KVMap>(table: ({
             inputs: I;
             referenceOutputs?: O;
@@ -72,6 +108,42 @@ export declare function generateWrapperFromJestlikeMethods(methods: Record<strin
                 referenceOutputs?: O;
             } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
         };
+        concurrent: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends KVMap, O extends KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+            only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+                each: <I extends KVMap, O extends KVMap>(table: ({
+                    inputs: I;
+                    referenceOutputs?: O;
+                } & Record<string, any>)[], config?: LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                    inputs: I;
+                    referenceOutputs?: O;
+                } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+            };
+            skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+                each: <I extends KVMap, O extends KVMap>(table: ({
+                    inputs: I;
+                    referenceOutputs?: O;
+                } & Record<string, any>)[], config?: LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                    inputs: I;
+                    referenceOutputs?: O;
+                } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+            };
+        };
         each: <I extends KVMap, O extends KVMap>(table: ({
             inputs: I;
             referenceOutputs?: O;
@@ -83,6 +155,7 @@ export declare function generateWrapperFromJestlikeMethods(methods: Record<strin
     describe: LangSmithJestlikeDescribeWrapper & {
         only: LangSmithJestlikeDescribeWrapper;
         skip: LangSmithJestlikeDescribeWrapper;
+        concurrent: LangSmithJestlikeDescribeWrapper;
     };
     expect: jest.Expect;
     toBeRelativeCloseTo: typeof toBeRelativeCloseTo;

package/dist/utils/jestlike/index.js

@@ -182,13 +182,27 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         }
         return storageValue;
     }
-    function wrapDescribeMethod(method) {
+    function wrapDescribeMethod(method, methodName) {
         if (isJsDom()) {
             console.error(`[LANGSMITH]: You seem to be using a jsdom environment. This is not supported and you may experience unexpected behavior. Please set the "environment" or "testEnvironment" field in your test config file to "node".`);
         }
         return function (testSuiteName, fn, experimentConfig) {
+            if (typeof method !== "function") {
+                throw new Error(`"${methodName}" is not supported by your test runner.`);
+            }
+            if (testWrapperAsyncLocalStorageInstance.getStore() !== undefined) {
+                throw new Error([
+                    `You seem to be nesting an ls.describe block named "${testSuiteName}" inside another ls.describe block.`,
+                    "This is not supported because each ls.describe block corresponds to a LangSmith dataset.",
+                    "To logically group tests, nest the native Jest or Vitest describe methods instead.",
+                ].join("\n"));
+            }
             const client = experimentConfig?.client ?? DEFAULT_TEST_CLIENT;
             const suiteName = experimentConfig?.testSuiteName ?? testSuiteName;
+            let setupPromiseResolver;
+            const setupPromise = new Promise((resolve) => {
+                setupPromiseResolver = resolve;
+            });
             return method(suiteName, () => {
                 const startTime = new Date();
                 const suiteUuid = v4();
@@ -222,10 +236,12 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                         metadata: suiteMetadata,
                     },
                     enableTestTracking: experimentConfig?.enableTestTracking,
+                    setupPromise,
                 };
                 beforeAll(async () => {
                     const storageValue = await runDatasetSetup(context);
                     datasetSetupInfo.set(suiteUuid, storageValue);
+                    setupPromiseResolver();
                 });
                 afterAll(async () => {
                     await Promise.all([
@@ -306,9 +322,10 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
             });
         };
     }
-    const lsDescribe = Object.assign(wrapDescribeMethod(describe), {
-        only: wrapDescribeMethod(describe.only),
-        skip: wrapDescribeMethod(describe.skip),
+    const lsDescribe = Object.assign(wrapDescribeMethod(describe, "describe"), {
+        only: wrapDescribeMethod(describe.only, "describe.only"),
+        skip: wrapDescribeMethod(describe.skip, "describe.skip"),
+        concurrent: wrapDescribeMethod(describe.concurrent, "describe.concurrent"),
     });
     function wrapTestMethod(method) {
         return function (name, lsParams, testFn, timeout) {
@@ -327,7 +344,12 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                 // Jest will not group tests under the same "describe" group if you await the test and
                 // total runs is greater than 1.
                 const resultsPath = path.join(os.tmpdir(), "langsmith_test_results", `${testUuid}.json`);
-                void method(`${name}${totalRuns > 1 ? `, run ${i}` : ""}${TEST_ID_DELIMITER}${testUuid}`, async () => {
+                void method(`${name}${totalRuns > 1 ? `, run ${i}` : ""}${TEST_ID_DELIMITER}${testUuid}`, async (...args) => {
+                    // Jest will magically introspect args and pass a "done" callback if
+                    // we use a non-spread parameter. To obtain and pass Vitest test context
+                    // through into the test function, we must therefore refer to Vitest
+                    // args using this signature
+                    const jestlikeArgs = args[0];
                     if (context === undefined) {
                         throw new Error([
                             `Could not retrieve test context.`,
@@ -335,6 +357,11 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                             `See this page for more information: https://docs.smith.langchain.com/evaluation/how_to_guides/vitest_jest`,
                         ].join("\n"));
                     }
+                    // Jest .concurrent is super buggy and doesn't wait for beforeAll to complete
+                    // before running test functions, so we need to wait for the setup promise
+                    // to resolve before we can continue.
+                    // Seee https://github.com/jestjs/jest/issues/4281
+                    await context.setupPromise;
                     if (!datasetSetupInfo.get(context.suiteUuid)) {
                         throw new Error("Dataset failed to initialize. Please check your LangSmith environment variables.");
                     }
@@ -367,11 +394,13 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                                 throw new Error("Could not identify test context after setting test root run tree. Please contact us for help.");
                             }
                             try {
-                                const res = await testFn({
+                                const res = await testFn(Object.assign(typeof jestlikeArgs === "object" && jestlikeArgs != null
+                                    ? jestlikeArgs
+                                    : {}, {
                                     ...rest,
                                     inputs: testInput,
                                     referenceOutputs: testOutput,
-                                });
+                                }));
                                 _logTestFeedback({
                                     exampleId,
                                     feedback: { key: "pass", score: true },
@@ -535,6 +564,16 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         }
         return eachMethod;
     }
+    // Roughly mirrors: https://jestjs.io/docs/api#methods
+    const concurrentMethod = Object.assign(wrapTestMethod(test.concurrent), {
+        each: createEachMethod(test.concurrent),
+        only: Object.assign(wrapTestMethod(test.concurrent.only), {
+            each: createEachMethod(test.concurrent.only),
+        }),
+        skip: Object.assign(wrapTestMethod(test.concurrent.skip), {
+            each: createEachMethod(test.concurrent.skip),
+        }),
+    });
     const lsTest = Object.assign(wrapTestMethod(test), {
         only: Object.assign(wrapTestMethod(test.only), {
             each: createEachMethod(test.only),
@@ -542,6 +581,7 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         skip: Object.assign(wrapTestMethod(test.skip), {
             each: createEachMethod(test.skip),
         }),
+        concurrent: concurrentMethod,
         each: createEachMethod(test),
     });
     const wrappedExpect = wrapExpect(expect);

package/dist/vitest/index.d.ts

@@ -83,6 +83,42 @@ declare const test: (<I extends Record<string, any> = Record<string, any>, O ext
             referenceOutputs?: O;
         } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
     };
+    concurrent: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+        inputs: I;
+        referenceOutputs?: O;
+    } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+        each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        };
+        skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        };
+    };
     each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
         inputs: I;
         referenceOutputs?: O;
@@ -118,6 +154,42 @@ declare const test: (<I extends Record<string, any> = Record<string, any>, O ext
             referenceOutputs?: O;
         } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
     };
+    concurrent: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+        inputs: I;
+        referenceOutputs?: O;
+    } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+        each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        };
+        skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
+            inputs: I;
+            referenceOutputs?: O;
+        } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
+            each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
+                inputs: I;
+                referenceOutputs?: O;
+            } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
+        };
+    };
     each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
         inputs: I;
         referenceOutputs?: O;
@@ -128,6 +200,7 @@ declare const test: (<I extends Record<string, any> = Record<string, any>, O ext
 }, describe: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper & {
     only: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper;
     skip: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper;
+    concurrent: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper;
 }, expect: jest.Expect;
 export { 
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.3.20",
+  "version": "0.3.21",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [