langsmith 0.3.5 → 0.3.7

package/dist/index.cjs

@@ -8,4 +8,4 @@ Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () {
 var fetch_js_1 = require("./singletons/fetch.cjs");
 Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true, get: function () { return fetch_js_1.overrideFetchImplementation; } });
 // Update using yarn bump-version
-exports.__version__ = "0.3.5";
+exports.__version__ = "0.3.7";

package/dist/index.d.ts

@@ -2,4 +2,4 @@ export { Client, type ClientConfig, type LangSmithTracingClientInterface, } from
 export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
-export declare const __version__ = "0.3.5";
+export declare const __version__ = "0.3.7";

package/dist/index.js

@@ -2,4 +2,4 @@ export { Client, } from "./client.js";
 export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 // Update using yarn bump-version
-export const __version__ = "0.3.5";
+export const __version__ = "0.3.7";

package/dist/jest/index.cjs

@@ -16,12 +16,9 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.wrapEvaluator = exports.logOutputs = exports.logFeedback = exports.toBeSemanticCloseTo = exports.toBeAbsoluteCloseTo = exports.toBeRelativeCloseTo = exports.expect = exports.describe = exports.it = exports.test = void 0;
+exports.wrapEvaluator = exports.logOutputs = exports.logFeedback = exports.expect = exports.describe = exports.it = exports.test = void 0;
 const globals_1 = require("@jest/globals");
 const matchers_js_1 = require("../utils/jestlike/matchers.cjs");
-Object.defineProperty(exports, "toBeRelativeCloseTo", { enumerable: true, get: function () { return matchers_js_1.toBeRelativeCloseTo; } });
-Object.defineProperty(exports, "toBeAbsoluteCloseTo", { enumerable: true, get: function () { return matchers_js_1.toBeAbsoluteCloseTo; } });
-Object.defineProperty(exports, "toBeSemanticCloseTo", { enumerable: true, get: function () { return matchers_js_1.toBeSemanticCloseTo; } });
 const evaluatedBy_js_1 = require("../utils/jestlike/vendor/evaluatedBy.cjs");
 Object.defineProperty(exports, "wrapEvaluator", { enumerable: true, get: function () { return evaluatedBy_js_1.wrapEvaluator; } });
 const index_js_1 = require("../utils/jestlike/index.cjs");

package/dist/jest/index.d.ts

@@ -1,5 +1,5 @@
 /// <reference types="jest" />
-import { toBeRelativeCloseTo, toBeAbsoluteCloseTo, toBeSemanticCloseTo, type AbsoluteCloseToMatcherOptions, type SemanticCloseToMatcherOptions, type RelativeCloseToMatcherOptions } from "../utils/jestlike/matchers.js";
+import { type AbsoluteCloseToMatcherOptions, type SemanticCloseToMatcherOptions, type RelativeCloseToMatcherOptions } from "../utils/jestlike/matchers.js";
 import type { SimpleEvaluator } from "../utils/jestlike/vendor/evaluatedBy.js";
 import { wrapEvaluator } from "../utils/jestlike/vendor/evaluatedBy.js";
 import { logFeedback, logOutputs } from "../utils/jestlike/index.js";
@@ -128,9 +128,9 @@ declare const test: (<I extends Record<string, any> = Record<string, any>, O ext
         inputs: I;
         referenceOutputs?: O | undefined;
     } & Record<string, any>) => unknown, timeout?: number | undefined) => void;
-}, describe: import("../utils/jestlike/types.js").LangSmithJestDescribeWrapper & {
-    only: import("../utils/jestlike/types.js").LangSmithJestDescribeWrapper;
-    skip: import("../utils/jestlike/types.js").LangSmithJestDescribeWrapper;
+}, describe: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper & {
+    only: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper;
+    skip: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper;
 }, expect: jest.Expect;
 export { 
 /**
@@ -324,12 +324,6 @@ describe,
  * ```
  */
 expect, 
-/** Whether the actual string value is close to the expected value in relative terms. */
-toBeRelativeCloseTo, 
-/** Whether the actual string value is close to the expected value in absolute terms. */
-toBeAbsoluteCloseTo, 
-/** Whether the actual string value is close to the expected value as scored by an embeddings model. */
-toBeSemanticCloseTo, 
 /**
  * Log feedback associated with the current test, usually generated by some kind of
  * evaluator.

package/dist/jest/index.js

@@ -209,12 +209,6 @@ describe,
  * ```
  */
 expect, 
-/** Whether the actual string value is close to the expected value in relative terms. */
-toBeRelativeCloseTo, 
-/** Whether the actual string value is close to the expected value in absolute terms. */
-toBeAbsoluteCloseTo, 
-/** Whether the actual string value is close to the expected value as scored by an embeddings model. */
-toBeSemanticCloseTo, 
 /**
  * Log feedback associated with the current test, usually generated by some kind of
  * evaluator.

package/dist/utils/jestlike/index.cjs

@@ -40,6 +40,7 @@ const _random_name_js_1 = require("../../evaluation/_random_name.cjs");
 const matchers_js_1 = require("./matchers.cjs");
 const globals_js_1 = require("./globals.cjs");
 const chain_js_1 = require("./vendor/chain.cjs");
+const env_js_1 = require("../env.cjs");
 const DEFAULT_TEST_TIMEOUT = 30_000;
 const UUID5_NAMESPACE = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
 // From https://stackoverflow.com/a/29497680
@@ -209,7 +210,7 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                 datasetId: dataset.id,
             });
             const experimentUrl = `${datasetUrl}/compare?selectedSessions=${project.id}`;
-            console.log(`[LANGSMITH]: Experiment starting! View results at ${experimentUrl}`);
+            console.log(`[LANGSMITH]: Experiment starting for dataset "${datasetName}"!\n[LANGSMITH]: View results at ${experimentUrl}`);
             storageValue = {
                 dataset,
                 project,
@@ -220,22 +221,40 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         return storageValue;
     }
     function wrapDescribeMethod(method) {
-        return function (datasetName, fn, experimentConfig) {
+        return function (testSuiteName, fn, experimentConfig) {
             const client = experimentConfig?.client ?? globals_js_1.DEFAULT_TEST_CLIENT;
-            return method(datasetName, () => {
+            const suiteName = experimentConfig?.testSuiteName ?? testSuiteName;
+            return method(suiteName, () => {
                 const startTime = new Date();
                 const suiteUuid = (0, uuid_1.v4)();
+                const environment = experimentConfig?.metadata?.ENVIRONMENT ??
+                    (0, env_js_1.getEnvironmentVariable)("ENVIRONMENT");
+                const nodeEnv = experimentConfig?.metadata?.NODE_ENV ??
+                    (0, env_js_1.getEnvironmentVariable)("NODE_ENV");
+                const langsmithEnvironment = experimentConfig?.metadata?.LANGSMITH_ENVIRONMENT ??
+                    (0, env_js_1.getEnvironmentVariable)("LANGSMITH_ENVIRONMENT");
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                const suiteMetadata = {
+                    ...experimentConfig?.metadata,
+                    __ls_runner: testRunnerName,
+                };
+                if (environment !== undefined) {
+                    suiteMetadata.ENVIRONMENT = environment;
+                }
+                if (nodeEnv !== undefined) {
+                    suiteMetadata.NODE_ENV = nodeEnv;
+                }
+                if (langsmithEnvironment !== undefined) {
+                    suiteMetadata.LANGSMITH_ENVIRONMENT = langsmithEnvironment;
+                }
                 const context = {
                     suiteUuid,
-                    suiteName: datasetName,
+                    suiteName,
                     client,
                     createdAt: new Date().toISOString(),
                     projectConfig: {
                         ...experimentConfig,
-                        metadata: {
-                            ...experimentConfig?.metadata,
-                            __ls_runner: testRunnerName,
-                        },
+                        metadata: suiteMetadata,
                     },
                     enableTestTracking: experimentConfig?.enableTestTracking,
                 };
@@ -259,11 +278,13 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                     const endTime = new Date();
                     let branch;
                     let commit;
+                    let dirty;
                     try {
                         branch = (0, child_process_1.execSync)("git rev-parse --abbrev-ref HEAD")
                             .toString()
                             .trim();
                         commit = (0, child_process_1.execSync)("git rev-parse HEAD").toString().trim();
+                        dirty = (0, child_process_1.execSync)("git status --porcelain").toString().trim() !== "";
                     }
                     catch {
                         return;
@@ -285,24 +306,22 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                             finalModifiedAt = endTime.toISOString();
                         }
                         const datasetInfo = datasetSetupInfo.get(suiteUuid);
-                        const { as_of } = await client.readDatasetVersion({
+                        await client.updateProject(datasetInfo.project.id, {
+                            metadata: {
+                                ...suiteMetadata,
+                                commit,
+                                branch,
+                                dirty,
+                            },
+                        });
+                        await client.updateDatasetTag({
                             datasetId: datasetInfo.dataset.id,
                             asOf: finalModifiedAt,
+                            tag: `git:commit:${commit}`,
                         });
-                        await Promise.all([
-                            client.updateDatasetTag({
-                                datasetId: datasetInfo.dataset.id,
-                                asOf: as_of,
-                                tag: `git:branch:${branch}`,
-                            }),
-                            client.updateDatasetTag({
-                                datasetId: datasetInfo.dataset.id,
-                                asOf: as_of,
-                                tag: `git:commit:${commit}`,
-                            }),
-                        ]);
                     }
-                    catch {
+                    catch (e) {
+                        console.error(e);
                         return;
                     }
                 });

package/dist/utils/jestlike/index.d.ts

@@ -2,7 +2,7 @@
 import { KVMap } from "../../schemas.js";
 import { toBeRelativeCloseTo, toBeAbsoluteCloseTo, toBeSemanticCloseTo } from "./matchers.js";
 import { SimpleEvaluationResult } from "./types.js";
-import type { LangSmithJestlikeWrapperConfig, LangSmithJestlikeWrapperParams, LangSmithJestDescribeWrapper } from "./types.js";
+import type { LangSmithJestlikeWrapperConfig, LangSmithJestlikeWrapperParams, LangSmithJestlikeDescribeWrapper } from "./types.js";
 export declare const STRIP_ANSI_REGEX: RegExp;
 export declare const TEST_ID_DELIMITER = ", test_id=";
 export declare function logFeedback(feedback: SimpleEvaluationResult, config?: {
@@ -83,9 +83,9 @@ export declare function generateWrapperFromJestlikeMethods(methods: Record<strin
             referenceOutputs?: O_1 | undefined;
         } & Record<string, any>) => unknown | Promise<unknown>, timeout?: number) => void;
     };
-    describe: LangSmithJestDescribeWrapper & {
-        only: LangSmithJestDescribeWrapper;
-        skip: LangSmithJestDescribeWrapper;
+    describe: LangSmithJestlikeDescribeWrapper & {
+        only: LangSmithJestlikeDescribeWrapper;
+        skip: LangSmithJestlikeDescribeWrapper;
     };
     expect: jest.Expect;
     toBeRelativeCloseTo: typeof toBeRelativeCloseTo;

package/dist/utils/jestlike/index.js

@@ -11,6 +11,7 @@ import { randomName } from "../../evaluation/_random_name.js";
 import { toBeRelativeCloseTo, toBeAbsoluteCloseTo, toBeSemanticCloseTo, } from "./matchers.js";
 import { evaluatorLogFeedbackPromises, testWrapperAsyncLocalStorageInstance, _logTestFeedback, syncExamplePromises, trackingEnabled, DEFAULT_TEST_CLIENT, } from "./globals.js";
 import { wrapExpect } from "./vendor/chain.js";
+import { getEnvironmentVariable } from "../env.js";
 const DEFAULT_TEST_TIMEOUT = 30_000;
 const UUID5_NAMESPACE = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
 // From https://stackoverflow.com/a/29497680
@@ -177,7 +178,7 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                 datasetId: dataset.id,
             });
             const experimentUrl = `${datasetUrl}/compare?selectedSessions=${project.id}`;
-            console.log(`[LANGSMITH]: Experiment starting! View results at ${experimentUrl}`);
+            console.log(`[LANGSMITH]: Experiment starting for dataset "${datasetName}"!\n[LANGSMITH]: View results at ${experimentUrl}`);
             storageValue = {
                 dataset,
                 project,
@@ -188,22 +189,40 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         return storageValue;
     }
     function wrapDescribeMethod(method) {
-        return function (datasetName, fn, experimentConfig) {
+        return function (testSuiteName, fn, experimentConfig) {
             const client = experimentConfig?.client ?? DEFAULT_TEST_CLIENT;
-            return method(datasetName, () => {
+            const suiteName = experimentConfig?.testSuiteName ?? testSuiteName;
+            return method(suiteName, () => {
                 const startTime = new Date();
                 const suiteUuid = v4();
+                const environment = experimentConfig?.metadata?.ENVIRONMENT ??
+                    getEnvironmentVariable("ENVIRONMENT");
+                const nodeEnv = experimentConfig?.metadata?.NODE_ENV ??
+                    getEnvironmentVariable("NODE_ENV");
+                const langsmithEnvironment = experimentConfig?.metadata?.LANGSMITH_ENVIRONMENT ??
+                    getEnvironmentVariable("LANGSMITH_ENVIRONMENT");
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                const suiteMetadata = {
+                    ...experimentConfig?.metadata,
+                    __ls_runner: testRunnerName,
+                };
+                if (environment !== undefined) {
+                    suiteMetadata.ENVIRONMENT = environment;
+                }
+                if (nodeEnv !== undefined) {
+                    suiteMetadata.NODE_ENV = nodeEnv;
+                }
+                if (langsmithEnvironment !== undefined) {
+                    suiteMetadata.LANGSMITH_ENVIRONMENT = langsmithEnvironment;
+                }
                 const context = {
                     suiteUuid,
-                    suiteName: datasetName,
+                    suiteName,
                     client,
                     createdAt: new Date().toISOString(),
                     projectConfig: {
                         ...experimentConfig,
-                        metadata: {
-                            ...experimentConfig?.metadata,
-                            __ls_runner: testRunnerName,
-                        },
+                        metadata: suiteMetadata,
                     },
                     enableTestTracking: experimentConfig?.enableTestTracking,
                 };
@@ -227,11 +246,13 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                     const endTime = new Date();
                     let branch;
                     let commit;
+                    let dirty;
                     try {
                         branch = execSync("git rev-parse --abbrev-ref HEAD")
                             .toString()
                             .trim();
                         commit = execSync("git rev-parse HEAD").toString().trim();
+                        dirty = execSync("git status --porcelain").toString().trim() !== "";
                     }
                     catch {
                         return;
@@ -253,24 +274,22 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                             finalModifiedAt = endTime.toISOString();
                         }
                         const datasetInfo = datasetSetupInfo.get(suiteUuid);
-                        const { as_of } = await client.readDatasetVersion({
+                        await client.updateProject(datasetInfo.project.id, {
+                            metadata: {
+                                ...suiteMetadata,
+                                commit,
+                                branch,
+                                dirty,
+                            },
+                        });
+                        await client.updateDatasetTag({
                             datasetId: datasetInfo.dataset.id,
                             asOf: finalModifiedAt,
+                            tag: `git:commit:${commit}`,
                         });
-                        await Promise.all([
-                            client.updateDatasetTag({
-                                datasetId: datasetInfo.dataset.id,
-                                asOf: as_of,
-                                tag: `git:branch:${branch}`,
-                            }),
-                            client.updateDatasetTag({
-                                datasetId: datasetInfo.dataset.id,
-                                asOf: as_of,
-                                tag: `git:commit:${commit}`,
-                            }),
-                        ]);
                     }
-                    catch {
+                    catch (e) {
+                        console.error(e);
                         return;
                     }
                 });

package/dist/utils/jestlike/types.d.ts

@@ -1,4 +1,6 @@
+import { CreateProjectParams } from "../../client.js";
 import { EvaluationResult } from "../../evaluation/evaluator.js";
+import { Client } from "../../index.js";
 import type { RunTreeConfig } from "../../run_trees.js";
 import type { SimpleEvaluator } from "./vendor/evaluatedBy.js";
 export { type SimpleEvaluator };
@@ -11,7 +13,14 @@ export type LangSmithJestlikeWrapperParams<I, O> = {
     referenceOutputs: O;
     config?: LangSmithJestlikeWrapperConfig;
 };
-export type LangSmithJestDescribeWrapper = (name: string, fn: () => void | Promise<void>, config?: Partial<RunTreeConfig>) => void;
+export type LangSmithJestlikeDescribeWrapperConfig = {
+    client?: Client;
+    enableTestTracking?: boolean;
+    testSuiteName?: string;
+} & Partial<Omit<CreateProjectParams, "referenceDatasetId">>;
+export type LangSmithJestlikeDescribeWrapper = (name: string, fn: () => void | Promise<void>, config?: LangSmithJestlikeDescribeWrapperConfig) => void;
+/** @deprecated Import as `LangSmithJestlikeDescribeWrapper` instead. */
+export type LangSmithJestDescribeWrapper = LangSmithJestlikeDescribeWrapper;
 export type SimpleEvaluationResult = {
     key: EvaluationResult["key"];
     score: NonNullable<EvaluationResult["score"]>;

package/dist/vitest/index.cjs

@@ -16,12 +16,9 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.wrapEvaluator = exports.logOutputs = exports.logFeedback = exports.toBeSemanticCloseTo = exports.toBeAbsoluteCloseTo = exports.toBeRelativeCloseTo = exports.expect = exports.describe = exports.it = exports.test = void 0;
+exports.wrapEvaluator = exports.logOutputs = exports.logFeedback = exports.expect = exports.describe = exports.it = exports.test = void 0;
 const vitest_1 = require("vitest");
 const matchers_js_1 = require("../utils/jestlike/matchers.cjs");
-Object.defineProperty(exports, "toBeRelativeCloseTo", { enumerable: true, get: function () { return matchers_js_1.toBeRelativeCloseTo; } });
-Object.defineProperty(exports, "toBeAbsoluteCloseTo", { enumerable: true, get: function () { return matchers_js_1.toBeAbsoluteCloseTo; } });
-Object.defineProperty(exports, "toBeSemanticCloseTo", { enumerable: true, get: function () { return matchers_js_1.toBeSemanticCloseTo; } });
 const evaluatedBy_js_1 = require("../utils/jestlike/vendor/evaluatedBy.cjs");
 Object.defineProperty(exports, "wrapEvaluator", { enumerable: true, get: function () { return evaluatedBy_js_1.wrapEvaluator; } });
 const index_js_1 = require("../utils/jestlike/index.cjs");

package/dist/vitest/index.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="jest" />
 import { Assertion } from "vitest";
-import { toBeRelativeCloseTo, toBeAbsoluteCloseTo, toBeSemanticCloseTo, type AbsoluteCloseToMatcherOptions, type SemanticCloseToMatcherOptions, type RelativeCloseToMatcherOptions } from "../utils/jestlike/matchers.js";
+import { type AbsoluteCloseToMatcherOptions, type SemanticCloseToMatcherOptions, type RelativeCloseToMatcherOptions } from "../utils/jestlike/matchers.js";
 import type { SimpleEvaluator } from "../utils/jestlike/vendor/evaluatedBy.js";
 import { wrapEvaluator } from "../utils/jestlike/vendor/evaluatedBy.js";
 import { logFeedback, logOutputs } from "../utils/jestlike/index.js";
@@ -126,9 +126,9 @@ declare const test: (<I extends Record<string, any> = Record<string, any>, O ext
         inputs: I;
         referenceOutputs?: O | undefined;
     } & Record<string, any>) => unknown, timeout?: number | undefined) => void;
-}, describe: import("../utils/jestlike/types.js").LangSmithJestDescribeWrapper & {
-    only: import("../utils/jestlike/types.js").LangSmithJestDescribeWrapper;
-    skip: import("../utils/jestlike/types.js").LangSmithJestDescribeWrapper;
+}, describe: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper & {
+    only: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper;
+    skip: import("../utils/jestlike/types.js").LangSmithJestlikeDescribeWrapper;
 }, expect: jest.Expect;
 export { 
 /**
@@ -322,12 +322,6 @@ describe,
  * ```
  */
 expect, 
-/** Whether the actual string value is close to the expected value in relative terms. */
-toBeRelativeCloseTo, 
-/** Whether the actual string value is close to the expected value in absolute terms. */
-toBeAbsoluteCloseTo, 
-/** Whether the actual string value is close to the expected value as scored by an embeddings model. */
-toBeSemanticCloseTo, 
 /**
  * Log feedback associated with the current test, usually generated by some kind of
  * evaluator.

package/dist/vitest/index.js

@@ -209,12 +209,6 @@ describe,
  * ```
  */
 expect, 
-/** Whether the actual string value is close to the expected value in relative terms. */
-toBeRelativeCloseTo, 
-/** Whether the actual string value is close to the expected value in absolute terms. */
-toBeAbsoluteCloseTo, 
-/** Whether the actual string value is close to the expected value as scored by an embeddings model. */
-toBeSemanticCloseTo, 
 /**
  * Log feedback associated with the current test, usually generated by some kind of
  * evaluator.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.3.5",
+  "version": "0.3.7",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [