langsmith 0.2.16-rc.13 → 0.2.16-rc.15

package/dist/client.cjs

@@ -1784,6 +1784,40 @@ class Client {
         await (0, error_js_1.raiseForStatus)(response, "update dataset");
         return (await response.json());
     }
+    /**
+     * Updates a tag on a dataset.
+     *
+     * If the tag is already assigned to a different version of this dataset,
+     * the tag will be moved to the new version. The as_of parameter is used to
+     * determine which version of the dataset to apply the new tags to.
+     *
+     * It must be an exact version of the dataset to succeed. You can
+     * use the "readDatasetVersion" method to find the exact version
+     * to apply the tags to.
+     * @param params.datasetId The ID of the dataset to update. Must be provided if "datasetName" is not provided.
+     * @param params.datasetName The name of the dataset to update. Must be provided if "datasetId" is not provided.
+     * @param params.asOf The timestamp of the dataset to apply the new tags to.
+     * @param params.tag The new tag to apply to the dataset.
+     */
+    async updateDatasetTag(props) {
+        const { datasetId, datasetName, asOf, tag } = props;
+        if (!datasetId && !datasetName) {
+            throw new Error("Must provide either datasetName or datasetId");
+        }
+        const _datasetId = datasetId ?? (await this.readDataset({ datasetName })).id;
+        (0, _uuid_js_1.assertUuid)(_datasetId);
+        const response = await this.caller.call((0, fetch_js_1._getFetchImplementation)(), `${this.apiUrl}/datasets/${_datasetId}/tags`, {
+            method: "PUT",
+            headers: { ...this.headers, "Content-Type": "application/json" },
+            body: JSON.stringify({
+                as_of: typeof asOf === "string" ? asOf : asOf.toISOString(),
+                tag,
+            }),
+            signal: AbortSignal.timeout(this.timeout_ms),
+            ...this.fetchOptions,
+        });
+        await (0, error_js_1.raiseForStatus)(response, "update dataset tags");
+    }
     async deleteDataset({ datasetId, datasetName, }) {
         let path = "/datasets";
         let datasetId_ = datasetId;
@@ -2099,6 +2133,47 @@ class Client {
         const result = await response.json();
         return result;
     }
+    /**
+     * Get dataset version by closest date or exact tag.
+     *
+     * Use this to resolve the nearest version to a given timestamp or for a given tag.
+     *
+     * @param options The options for getting the dataset version
+     * @param options.datasetId The ID of the dataset
+     * @param options.datasetName The name of the dataset
+     * @param options.asOf The timestamp of the dataset to retrieve
+     * @param options.tag The tag of the dataset to retrieve
+     * @returns The dataset version
+     */
+    async readDatasetVersion({ datasetId, datasetName, asOf, tag, }) {
+        let resolvedDatasetId;
+        if (!datasetId) {
+            const dataset = await this.readDataset({ datasetName });
+            resolvedDatasetId = dataset.id;
+        }
+        else {
+            resolvedDatasetId = datasetId;
+        }
+        (0, _uuid_js_1.assertUuid)(resolvedDatasetId);
+        if ((asOf && tag) || (!asOf && !tag)) {
+            throw new Error("Exactly one of asOf and tag must be specified.");
+        }
+        const params = new URLSearchParams();
+        if (asOf !== undefined) {
+            params.append("as_of", typeof asOf === "string" ? asOf : asOf.toISOString());
+        }
+        if (tag !== undefined) {
+            params.append("tag", tag);
+        }
+        const response = await this.caller.call((0, fetch_js_1._getFetchImplementation)(), `${this.apiUrl}/datasets/${resolvedDatasetId}/version?${params.toString()}`, {
+            method: "GET",
+            headers: { ...this.headers },
+            signal: AbortSignal.timeout(this.timeout_ms),
+            ...this.fetchOptions,
+        });
+        await (0, error_js_1.raiseForStatus)(response, "read dataset version");
+        return await response.json();
+    }
     async listDatasetSplits({ datasetId, datasetName, asOf, }) {
         let datasetId_;
         if (datasetId === undefined && datasetName === undefined) {

package/dist/client.d.ts

@@ -1,5 +1,5 @@
 import { AsyncCallerParams } from "./utils/async_caller.js";
-import { ComparativeExperiment, DataType, Dataset, DatasetDiffInfo, DatasetShareSchema, Example, ExampleUpdate, ExampleUpdateWithId, Feedback, FeedbackConfig, FeedbackIngestToken, KVMap, LangChainBaseMessage, LangSmithSettings, LikePromptResponse, Prompt, PromptCommit, PromptSortField, Run, RunCreate, RunUpdate, ScoreType, ExampleSearch, TimeDelta, TracerSession, TracerSessionResult, ValueType, AnnotationQueue, RunWithAnnotationQueueInfo, Attachments, ExampleUploadWithAttachments, UploadExamplesResponse, ExampleUpdateWithAttachments, UpdateExamplesResponse } from "./schemas.js";
+import { ComparativeExperiment, DataType, Dataset, DatasetDiffInfo, DatasetShareSchema, Example, ExampleUpdate, ExampleUpdateWithId, Feedback, FeedbackConfig, FeedbackIngestToken, KVMap, LangChainBaseMessage, LangSmithSettings, LikePromptResponse, Prompt, PromptCommit, PromptSortField, Run, RunCreate, RunUpdate, ScoreType, ExampleSearch, TimeDelta, TracerSession, TracerSessionResult, ValueType, AnnotationQueue, RunWithAnnotationQueueInfo, Attachments, ExampleUploadWithAttachments, UploadExamplesResponse, ExampleUpdateWithAttachments, UpdateExamplesResponse, DatasetVersion } from "./schemas.js";
 import { EvaluationResult, EvaluationResults, RunEvaluator } from "./evaluation/evaluator.js";
 export interface ClientConfig {
     apiUrl?: string;
@@ -487,6 +487,27 @@ export declare class Client implements LangSmithTracingClientInterface {
         name?: string;
         description?: string;
     }): Promise<Dataset>;
+    /**
+     * Updates a tag on a dataset.
+     *
+     * If the tag is already assigned to a different version of this dataset,
+     * the tag will be moved to the new version. The as_of parameter is used to
+     * determine which version of the dataset to apply the new tags to.
+     *
+     * It must be an exact version of the dataset to succeed. You can
+     * use the "readDatasetVersion" method to find the exact version
+     * to apply the tags to.
+     * @param params.datasetId The ID of the dataset to update. Must be provided if "datasetName" is not provided.
+     * @param params.datasetName The name of the dataset to update. Must be provided if "datasetId" is not provided.
+     * @param params.asOf The timestamp of the dataset to apply the new tags to.
+     * @param params.tag The new tag to apply to the dataset.
+     */
+    updateDatasetTag(props: {
+        datasetId?: string;
+        datasetName?: string;
+        asOf: string | Date;
+        tag: string;
+    }): Promise<void>;
     deleteDataset({ datasetId, datasetName, }: {
         datasetId?: string;
         datasetName?: string;
@@ -559,6 +580,24 @@ export declare class Client implements LangSmithTracingClientInterface {
     deleteExample(exampleId: string): Promise<void>;
     updateExample(exampleId: string, update: ExampleUpdate): Promise<object>;
     updateExamples(update: ExampleUpdateWithId[]): Promise<object>;
+    /**
+     * Get dataset version by closest date or exact tag.
+     *
+     * Use this to resolve the nearest version to a given timestamp or for a given tag.
+     *
+     * @param options The options for getting the dataset version
+     * @param options.datasetId The ID of the dataset
+     * @param options.datasetName The name of the dataset
+     * @param options.asOf The timestamp of the dataset to retrieve
+     * @param options.tag The tag of the dataset to retrieve
+     * @returns The dataset version
+     */
+    readDatasetVersion({ datasetId, datasetName, asOf, tag, }: {
+        datasetId?: string;
+        datasetName?: string;
+        asOf?: string | Date;
+        tag?: string;
+    }): Promise<DatasetVersion>;
     listDatasetSplits({ datasetId, datasetName, asOf, }: {
         datasetId?: string;
         datasetName?: string;

package/dist/client.js

@@ -1756,6 +1756,40 @@ export class Client {
         await raiseForStatus(response, "update dataset");
         return (await response.json());
     }
+    /**
+     * Updates a tag on a dataset.
+     *
+     * If the tag is already assigned to a different version of this dataset,
+     * the tag will be moved to the new version. The as_of parameter is used to
+     * determine which version of the dataset to apply the new tags to.
+     *
+     * It must be an exact version of the dataset to succeed. You can
+     * use the "readDatasetVersion" method to find the exact version
+     * to apply the tags to.
+     * @param params.datasetId The ID of the dataset to update. Must be provided if "datasetName" is not provided.
+     * @param params.datasetName The name of the dataset to update. Must be provided if "datasetId" is not provided.
+     * @param params.asOf The timestamp of the dataset to apply the new tags to.
+     * @param params.tag The new tag to apply to the dataset.
+     */
+    async updateDatasetTag(props) {
+        const { datasetId, datasetName, asOf, tag } = props;
+        if (!datasetId && !datasetName) {
+            throw new Error("Must provide either datasetName or datasetId");
+        }
+        const _datasetId = datasetId ?? (await this.readDataset({ datasetName })).id;
+        assertUuid(_datasetId);
+        const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/datasets/${_datasetId}/tags`, {
+            method: "PUT",
+            headers: { ...this.headers, "Content-Type": "application/json" },
+            body: JSON.stringify({
+                as_of: typeof asOf === "string" ? asOf : asOf.toISOString(),
+                tag,
+            }),
+            signal: AbortSignal.timeout(this.timeout_ms),
+            ...this.fetchOptions,
+        });
+        await raiseForStatus(response, "update dataset tags");
+    }
     async deleteDataset({ datasetId, datasetName, }) {
         let path = "/datasets";
         let datasetId_ = datasetId;
@@ -2071,6 +2105,47 @@ export class Client {
         const result = await response.json();
         return result;
     }
+    /**
+     * Get dataset version by closest date or exact tag.
+     *
+     * Use this to resolve the nearest version to a given timestamp or for a given tag.
+     *
+     * @param options The options for getting the dataset version
+     * @param options.datasetId The ID of the dataset
+     * @param options.datasetName The name of the dataset
+     * @param options.asOf The timestamp of the dataset to retrieve
+     * @param options.tag The tag of the dataset to retrieve
+     * @returns The dataset version
+     */
+    async readDatasetVersion({ datasetId, datasetName, asOf, tag, }) {
+        let resolvedDatasetId;
+        if (!datasetId) {
+            const dataset = await this.readDataset({ datasetName });
+            resolvedDatasetId = dataset.id;
+        }
+        else {
+            resolvedDatasetId = datasetId;
+        }
+        assertUuid(resolvedDatasetId);
+        if ((asOf && tag) || (!asOf && !tag)) {
+            throw new Error("Exactly one of asOf and tag must be specified.");
+        }
+        const params = new URLSearchParams();
+        if (asOf !== undefined) {
+            params.append("as_of", typeof asOf === "string" ? asOf : asOf.toISOString());
+        }
+        if (tag !== undefined) {
+            params.append("tag", tag);
+        }
+        const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/datasets/${resolvedDatasetId}/version?${params.toString()}`, {
+            method: "GET",
+            headers: { ...this.headers },
+            signal: AbortSignal.timeout(this.timeout_ms),
+            ...this.fetchOptions,
+        });
+        await raiseForStatus(response, "read dataset version");
+        return await response.json();
+    }
     async listDatasetSplits({ datasetId, datasetName, asOf, }) {
         let datasetId_;
         if (datasetId === undefined && datasetName === undefined) {

package/dist/index.cjs

@@ -8,4 +8,4 @@ Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () {
 var fetch_js_1 = require("./singletons/fetch.cjs");
 Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true, get: function () { return fetch_js_1.overrideFetchImplementation; } });
 // Update using yarn bump-version
-exports.__version__ = "0.2.16-rc.13";
+exports.__version__ = "0.2.16-rc.15";

package/dist/index.d.ts

@@ -2,4 +2,4 @@ export { Client, type ClientConfig, type LangSmithTracingClientInterface, } from
 export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
-export declare const __version__ = "0.2.16-rc.13";
+export declare const __version__ = "0.2.16-rc.15";

package/dist/index.js

@@ -2,4 +2,4 @@ export { Client, } from "./client.js";
 export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 // Update using yarn bump-version
-export const __version__ = "0.2.16-rc.13";
+export const __version__ = "0.2.16-rc.15";

package/dist/jest/index.d.ts

@@ -60,30 +60,30 @@ declare global {
 }
 declare const test: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
     inputs: I;
-    referenceOutputs: O;
+    referenceOutputs?: O | undefined;
 } & Record<string, any>) => unknown, timeout?: number | undefined) => void) & {
     only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
         inputs: I;
-        referenceOutputs: O;
+        referenceOutputs?: O | undefined;
     } & Record<string, any>) => unknown, timeout?: number | undefined) => void) & {
         each: <I_1 extends import("../schemas.js").KVMap, O_1 extends import("../schemas.js").KVMap>(table: ({
             inputs: I_1;
             referenceOutputs: O_1;
         } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig | undefined) => (name: string, fn: (params: {
             inputs: I_1;
-            referenceOutputs: O_1;
+            referenceOutputs?: O_1 | undefined;
         } & Record<string, any>) => unknown, timeout?: number | undefined) => void;
     };
     skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
         inputs: I;
-        referenceOutputs: O;
+        referenceOutputs?: O | undefined;
     } & Record<string, any>) => unknown, timeout?: number | undefined) => void) & {
         each: <I_1 extends import("../schemas.js").KVMap, O_1 extends import("../schemas.js").KVMap>(table: ({
             inputs: I_1;
             referenceOutputs: O_1;
         } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig | undefined) => (name: string, fn: (params: {
             inputs: I_1;
-            referenceOutputs: O_1;
+            referenceOutputs?: O_1 | undefined;
         } & Record<string, any>) => unknown, timeout?: number | undefined) => void;
     };
     each: <I_1 extends import("../schemas.js").KVMap, O_1 extends import("../schemas.js").KVMap>(table: ({
@@ -91,34 +91,34 @@ declare const test: (<I extends Record<string, any> = Record<string, any>, O ext
         referenceOutputs: O_1;
     } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig | undefined) => (name: string, fn: (params: {
         inputs: I_1;
-        referenceOutputs: O_1;
+        referenceOutputs?: O_1 | undefined;
     } & Record<string, any>) => unknown, timeout?: number | undefined) => void;
 }, it: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
     inputs: I;
-    referenceOutputs: O;
+    referenceOutputs?: O | undefined;
 } & Record<string, any>) => unknown, timeout?: number | undefined) => void) & {
     only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
         inputs: I;
-        referenceOutputs: O;
+        referenceOutputs?: O | undefined;
     } & Record<string, any>) => unknown, timeout?: number | undefined) => void) & {
         each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
             inputs: I;
             referenceOutputs: O;
         } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig | undefined) => (name: string, fn: (params: {
             inputs: I;
-            referenceOutputs: O;
+            referenceOutputs?: O | undefined;
         } & Record<string, any>) => unknown, timeout?: number | undefined) => void;
     };
     skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
         inputs: I;
-        referenceOutputs: O;
+        referenceOutputs?: O | undefined;
     } & Record<string, any>) => unknown, timeout?: number | undefined) => void) & {
         each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
             inputs: I;
             referenceOutputs: O;
         } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig | undefined) => (name: string, fn: (params: {
             inputs: I;
-            referenceOutputs: O;
+            referenceOutputs?: O | undefined;
         } & Record<string, any>) => unknown, timeout?: number | undefined) => void;
     };
     each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
@@ -126,7 +126,7 @@ declare const test: (<I extends Record<string, any> = Record<string, any>, O ext
         referenceOutputs: O;
     } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig | undefined) => (name: string, fn: (params: {
         inputs: I;
-        referenceOutputs: O;
+        referenceOutputs?: O | undefined;
     } & Record<string, any>) => unknown, timeout?: number | undefined) => void;
 }, describe: import("../utils/jestlike/types.js").LangSmithJestDescribeWrapper & {
     only: import("../utils/jestlike/types.js").LangSmithJestDescribeWrapper;

package/dist/schemas.d.ts

@@ -268,6 +268,10 @@ export interface DatasetShareSchema {
     share_token: string;
     url: string;
 }
+export interface DatasetVersion {
+    tags?: string[];
+    as_of: string;
+}
 export interface FeedbackSourceBase {
     type: string;
     metadata?: KVMap;

package/dist/utils/jestlike/index.cjs

@@ -34,11 +34,13 @@ const uuid_1 = require("uuid");
 const os = __importStar(require("node:os"));
 const path = __importStar(require("node:path"));
 const fs = __importStar(require("node:fs/promises"));
+const child_process_1 = require("child_process");
 const traceable_js_1 = require("../../traceable.cjs");
 const _random_name_js_1 = require("../../evaluation/_random_name.cjs");
 const matchers_js_1 = require("./matchers.cjs");
 const globals_js_1 = require("./globals.cjs");
 const chain_js_1 = require("./vendor/chain.cjs");
+const DEFAULT_TEST_TIMEOUT = 30_000;
 const UUID5_NAMESPACE = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
 // From https://stackoverflow.com/a/29497680
 exports.STRIP_ANSI_REGEX = 
@@ -221,6 +223,7 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         return function (datasetName, fn, experimentConfig) {
             const client = experimentConfig?.client ?? globals_js_1.DEFAULT_TEST_CLIENT;
             return method(datasetName, () => {
+                const startTime = new Date();
                 const suiteUuid = (0, uuid_1.v4)();
                 const context = {
                     suiteUuid,
@@ -246,6 +249,62 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                         ...globals_js_1.syncExamplePromises.values(),
                         ...globals_js_1.evaluatorLogFeedbackPromises.values(),
                     ]);
+                    if (!(0, globals_js_1.trackingEnabled)(context)) {
+                        return;
+                    }
+                    const examples = [...globals_js_1.syncExamplePromises.values()];
+                    if (examples.length === 0) {
+                        return;
+                    }
+                    const endTime = new Date();
+                    let branch;
+                    let commit;
+                    try {
+                        branch = (0, child_process_1.execSync)("git rev-parse --abbrev-ref HEAD")
+                            .toString()
+                            .trim();
+                        commit = (0, child_process_1.execSync)("git rev-parse HEAD").toString().trim();
+                    }
+                    catch {
+                        return;
+                    }
+                    if (branch === undefined || commit === undefined) {
+                        return;
+                    }
+                    try {
+                        let finalModifiedAt = examples.reduce((latestModifiedAt, example) => {
+                            if (new Date(latestModifiedAt).getTime() >
+                                new Date(example.modified_at).getTime()) {
+                                return latestModifiedAt;
+                            }
+                            else {
+                                return example.modified_at;
+                            }
+                        }, examples[0].modified_at);
+                        if (new Date(finalModifiedAt).getTime() < startTime.getTime()) {
+                            finalModifiedAt = endTime.toISOString();
+                        }
+                        const datasetInfo = datasetSetupInfo.get(suiteUuid);
+                        const { as_of } = await client.readDatasetVersion({
+                            datasetId: datasetInfo.dataset.id,
+                            asOf: finalModifiedAt,
+                        });
+                        await Promise.all([
+                            client.updateDatasetTag({
+                                datasetId: datasetInfo.dataset.id,
+                                asOf: as_of,
+                                tag: `git:branch:${branch}`,
+                            }),
+                            client.updateDatasetTag({
+                                datasetId: datasetInfo.dataset.id,
+                                asOf: as_of,
+                                tag: `git:commit:${commit}`,
+                            }),
+                        ]);
+                    }
+                    catch (e) {
+                        console.warn(`[LANGSMITH]: Failed to tag dataset: ${e.message}`);
+                    }
                 });
                 /**
                  * We cannot rely on setting AsyncLocalStorage in beforeAll or beforeEach,
@@ -458,7 +517,7 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                             experimentUrl,
                         }));
                     }
-                }, timeout);
+                }, timeout ?? DEFAULT_TEST_TIMEOUT);
             }
         };
     }

package/dist/utils/jestlike/index.d.ts

@@ -12,30 +12,30 @@ export declare function logOutputs(output: Record<string, unknown>): void;
 export declare function generateWrapperFromJestlikeMethods(methods: Record<string, any>, testRunnerName: string): {
     test: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
         inputs: I;
-        referenceOutputs: O;
+        referenceOutputs?: O;
     } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
         only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
             inputs: I;
-            referenceOutputs: O;
+            referenceOutputs?: O;
         } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
             each: <I_1 extends KVMap, O_1 extends KVMap>(table: ({
                 inputs: I_1;
                 referenceOutputs: O_1;
             } & Record<string, any>)[], config?: LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
                 inputs: I_1;
-                referenceOutputs: O_1;
+                referenceOutputs?: O_1 | undefined;
             } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
         };
         skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
             inputs: I;
-            referenceOutputs: O;
+            referenceOutputs?: O;
         } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
             each: <I_1 extends KVMap, O_1 extends KVMap>(table: ({
                 inputs: I_1;
                 referenceOutputs: O_1;
             } & Record<string, any>)[], config?: LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
                 inputs: I_1;
-                referenceOutputs: O_1;
+                referenceOutputs?: O_1 | undefined;
             } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
         };
         each: <I_1 extends KVMap, O_1 extends KVMap>(table: ({
@@ -43,35 +43,35 @@ export declare function generateWrapperFromJestlikeMethods(methods: Record<strin
             referenceOutputs: O_1;
         } & Record<string, any>)[], config?: LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
             inputs: I_1;
-            referenceOutputs: O_1;
+            referenceOutputs?: O_1 | undefined;
         } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
     };
     it: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
         inputs: I;
-        referenceOutputs: O;
+        referenceOutputs?: O;
     } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
         only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
             inputs: I;
-            referenceOutputs: O;
+            referenceOutputs?: O;
         } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
             each: <I_1 extends KVMap, O_1 extends KVMap>(table: ({
                 inputs: I_1;
                 referenceOutputs: O_1;
             } & Record<string, any>)[], config?: LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
                 inputs: I_1;
-                referenceOutputs: O_1;
+                referenceOutputs?: O_1 | undefined;
             } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
         };
         skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
             inputs: I;
-            referenceOutputs: O;
+            referenceOutputs?: O;
         } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void) & {
             each: <I_1 extends KVMap, O_1 extends KVMap>(table: ({
                 inputs: I_1;
                 referenceOutputs: O_1;
             } & Record<string, any>)[], config?: LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
                 inputs: I_1;
-                referenceOutputs: O_1;
+                referenceOutputs?: O_1 | undefined;
             } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
         };
         each: <I_1 extends KVMap, O_1 extends KVMap>(table: ({
@@ -79,7 +79,7 @@ export declare function generateWrapperFromJestlikeMethods(methods: Record<strin
             referenceOutputs: O_1;
         } & Record<string, any>)[], config?: LangSmithJestlikeWrapperConfig) => (name: string, fn: (params: {
             inputs: I_1;
-            referenceOutputs: O_1;
+            referenceOutputs?: O_1 | undefined;
         } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
     };
     describe: LangSmithJestDescribeWrapper & {

package/dist/utils/jestlike/index.js

@@ -5,11 +5,13 @@ import { v4, v5 } from "uuid";
 import * as os from "node:os";
 import * as path from "node:path";
 import * as fs from "node:fs/promises";
+import { execSync } from "child_process";
 import { getCurrentRunTree, traceable } from "../../traceable.js";
 import { randomName } from "../../evaluation/_random_name.js";
 import { toBeRelativeCloseTo, toBeAbsoluteCloseTo, toBeSemanticCloseTo, } from "./matchers.js";
 import { evaluatorLogFeedbackPromises, testWrapperAsyncLocalStorageInstance, _logTestFeedback, syncExamplePromises, trackingEnabled, DEFAULT_TEST_CLIENT, } from "./globals.js";
 import { wrapExpect } from "./vendor/chain.js";
+const DEFAULT_TEST_TIMEOUT = 30_000;
 const UUID5_NAMESPACE = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
 // From https://stackoverflow.com/a/29497680
 export const STRIP_ANSI_REGEX = 
@@ -190,6 +192,7 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         return function (datasetName, fn, experimentConfig) {
             const client = experimentConfig?.client ?? DEFAULT_TEST_CLIENT;
             return method(datasetName, () => {
+                const startTime = new Date();
                 const suiteUuid = v4();
                 const context = {
                     suiteUuid,
@@ -215,6 +218,62 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                         ...syncExamplePromises.values(),
                         ...evaluatorLogFeedbackPromises.values(),
                     ]);
+                    if (!trackingEnabled(context)) {
+                        return;
+                    }
+                    const examples = [...syncExamplePromises.values()];
+                    if (examples.length === 0) {
+                        return;
+                    }
+                    const endTime = new Date();
+                    let branch;
+                    let commit;
+                    try {
+                        branch = execSync("git rev-parse --abbrev-ref HEAD")
+                            .toString()
+                            .trim();
+                        commit = execSync("git rev-parse HEAD").toString().trim();
+                    }
+                    catch {
+                        return;
+                    }
+                    if (branch === undefined || commit === undefined) {
+                        return;
+                    }
+                    try {
+                        let finalModifiedAt = examples.reduce((latestModifiedAt, example) => {
+                            if (new Date(latestModifiedAt).getTime() >
+                                new Date(example.modified_at).getTime()) {
+                                return latestModifiedAt;
+                            }
+                            else {
+                                return example.modified_at;
+                            }
+                        }, examples[0].modified_at);
+                        if (new Date(finalModifiedAt).getTime() < startTime.getTime()) {
+                            finalModifiedAt = endTime.toISOString();
+                        }
+                        const datasetInfo = datasetSetupInfo.get(suiteUuid);
+                        const { as_of } = await client.readDatasetVersion({
+                            datasetId: datasetInfo.dataset.id,
+                            asOf: finalModifiedAt,
+                        });
+                        await Promise.all([
+                            client.updateDatasetTag({
+                                datasetId: datasetInfo.dataset.id,
+                                asOf: as_of,
+                                tag: `git:branch:${branch}`,
+                            }),
+                            client.updateDatasetTag({
+                                datasetId: datasetInfo.dataset.id,
+                                asOf: as_of,
+                                tag: `git:commit:${commit}`,
+                            }),
+                        ]);
+                    }
+                    catch (e) {
+                        console.warn(`[LANGSMITH]: Failed to tag dataset: ${e.message}`);
+                    }
                 });
                 /**
                  * We cannot rely on setting AsyncLocalStorage in beforeAll or beforeEach,
@@ -427,7 +486,7 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                             experimentUrl,
                         }));
                     }
-                }, timeout);
+                }, timeout ?? DEFAULT_TEST_TIMEOUT);
             }
         };
     }

package/dist/utils/jestlike/vendor/evaluatedBy.cjs

@@ -66,11 +66,10 @@ async function evaluatedBy(outputs, evaluator) {
             `See this page for more information: https://docs.smith.langchain.com/evaluation/how_to_guides/vitest_jest`,
         ].join("\n"));
     }
-    const runTree = (0, traceable_js_1.getCurrentRunTree)();
     const wrappedEvaluator = wrapEvaluator(evaluator);
     const evalRunId = (0, uuid_1.v4)();
     const evalResult = await wrappedEvaluator({
-        inputs: runTree.inputs,
+        inputs: context.currentExample?.inputs ?? {},
         referenceOutputs: context?.currentExample?.outputs ?? {},
         outputs,
     }, { runId: evalRunId });

package/dist/utils/jestlike/vendor/evaluatedBy.js

@@ -62,11 +62,10 @@ export async function evaluatedBy(outputs, evaluator) {
             `See this page for more information: https://docs.smith.langchain.com/evaluation/how_to_guides/vitest_jest`,
         ].join("\n"));
     }
-    const runTree = getCurrentRunTree();
     const wrappedEvaluator = wrapEvaluator(evaluator);
     const evalRunId = v4();
     const evalResult = await wrappedEvaluator({
-        inputs: runTree.inputs,
+        inputs: context.currentExample?.inputs ?? {},
         referenceOutputs: context?.currentExample?.outputs ?? {},
         outputs,
     }, { runId: evalRunId });

package/dist/vitest/index.d.ts

@@ -58,30 +58,30 @@ declare module "vitest" {
 }
 declare const test: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
     inputs: I;
-    referenceOutputs: O;
+    referenceOutputs?: O | undefined;
 } & Record<string, any>) => unknown, timeout?: number | undefined) => void) & {
     only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
         inputs: I;
-        referenceOutputs: O;
+        referenceOutputs?: O | undefined;
     } & Record<string, any>) => unknown, timeout?: number | undefined) => void) & {
         each: <I_1 extends import("../schemas.js").KVMap, O_1 extends import("../schemas.js").KVMap>(table: ({
             inputs: I_1;
             referenceOutputs: O_1;
         } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig | undefined) => (name: string, fn: (params: {
             inputs: I_1;
-            referenceOutputs: O_1;
+            referenceOutputs?: O_1 | undefined;
         } & Record<string, any>) => unknown, timeout?: number | undefined) => void;
     };
     skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
         inputs: I;
-        referenceOutputs: O;
+        referenceOutputs?: O | undefined;
     } & Record<string, any>) => unknown, timeout?: number | undefined) => void) & {
         each: <I_1 extends import("../schemas.js").KVMap, O_1 extends import("../schemas.js").KVMap>(table: ({
             inputs: I_1;
             referenceOutputs: O_1;
         } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig | undefined) => (name: string, fn: (params: {
             inputs: I_1;
-            referenceOutputs: O_1;
+            referenceOutputs?: O_1 | undefined;
         } & Record<string, any>) => unknown, timeout?: number | undefined) => void;
     };
     each: <I_1 extends import("../schemas.js").KVMap, O_1 extends import("../schemas.js").KVMap>(table: ({
@@ -89,34 +89,34 @@ declare const test: (<I extends Record<string, any> = Record<string, any>, O ext
         referenceOutputs: O_1;
     } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig | undefined) => (name: string, fn: (params: {
         inputs: I_1;
-        referenceOutputs: O_1;
+        referenceOutputs?: O_1 | undefined;
     } & Record<string, any>) => unknown, timeout?: number | undefined) => void;
 }, it: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
     inputs: I;
-    referenceOutputs: O;
+    referenceOutputs?: O | undefined;
 } & Record<string, any>) => unknown, timeout?: number | undefined) => void) & {
     only: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
         inputs: I;
-        referenceOutputs: O;
+        referenceOutputs?: O | undefined;
     } & Record<string, any>) => unknown, timeout?: number | undefined) => void) & {
         each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
             inputs: I;
             referenceOutputs: O;
         } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig | undefined) => (name: string, fn: (params: {
             inputs: I;
-            referenceOutputs: O;
+            referenceOutputs?: O | undefined;
         } & Record<string, any>) => unknown, timeout?: number | undefined) => void;
     };
     skip: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
         inputs: I;
-        referenceOutputs: O;
+        referenceOutputs?: O | undefined;
     } & Record<string, any>) => unknown, timeout?: number | undefined) => void) & {
         each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
             inputs: I;
             referenceOutputs: O;
         } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig | undefined) => (name: string, fn: (params: {
             inputs: I;
-            referenceOutputs: O;
+            referenceOutputs?: O | undefined;
         } & Record<string, any>) => unknown, timeout?: number | undefined) => void;
     };
     each: <I extends import("../schemas.js").KVMap, O extends import("../schemas.js").KVMap>(table: ({
@@ -124,7 +124,7 @@ declare const test: (<I extends Record<string, any> = Record<string, any>, O ext
         referenceOutputs: O;
     } & Record<string, any>)[], config?: import("../utils/jestlike/types.js").LangSmithJestlikeWrapperConfig | undefined) => (name: string, fn: (params: {
         inputs: I;
-        referenceOutputs: O;
+        referenceOutputs?: O | undefined;
     } & Record<string, any>) => unknown, timeout?: number | undefined) => void;
 }, describe: import("../utils/jestlike/types.js").LangSmithJestDescribeWrapper & {
     only: import("../utils/jestlike/types.js").LangSmithJestDescribeWrapper;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.2.16-rc.13",
+  "version": "0.2.16-rc.15",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [
@@ -94,7 +94,6 @@
     "test:integration": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000",
     "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000",
     "watch:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --watch --config jest.config.cjs --testTimeout 100000",
-    "eval:vitest": "vitest run --config ls.vitest.config.ts",
     "lint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/",
     "lint:fix": "yarn lint --fix",
     "format": "prettier --write 'src/**/*.{ts,tsx}'",