langsmith 0.3.7 → 0.3.9-rc.0

package/dist/index.cjs

@@ -8,4 +8,4 @@ Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () {
 var fetch_js_1 = require("./singletons/fetch.cjs");
 Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true, get: function () { return fetch_js_1.overrideFetchImplementation; } });
 // Update using yarn bump-version
-exports.__version__ = "0.3.7";
+exports.__version__ = "0.3.8";

package/dist/index.d.ts

@@ -2,4 +2,4 @@ export { Client, type ClientConfig, type LangSmithTracingClientInterface, } from
 export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
-export declare const __version__ = "0.3.7";
+export declare const __version__ = "0.3.8";

package/dist/index.js

@@ -2,4 +2,4 @@ export { Client, } from "./client.js";
 export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 // Update using yarn bump-version
-export const __version__ = "0.3.7";
+export const __version__ = "0.3.8";

package/dist/traceable.cjs

@@ -363,6 +363,18 @@ function traceable(wrappedFunc, config) {
                                 controller.close();
                                 break;
                             }
+                            if (currentRunTree?.run_type === "llm") {
+                                if (currentRunTree.events === undefined) {
+                                    currentRunTree.events = [];
+                                }
+                                currentRunTree.events.push({
+                                    name: "new_token",
+                                    time: new Date().toISOString(),
+                                    kwargs: {
+                                        token: result.value,
+                                    },
+                                });
+                            }
                             chunks.push(result.value);
                             controller.enqueue(result.value);
                         }
@@ -389,6 +401,18 @@ function traceable(wrappedFunc, config) {
                             finished = true;
                             break;
                         }
+                        if (currentRunTree?.run_type === "llm") {
+                            if (currentRunTree.events === undefined) {
+                                currentRunTree.events = [];
+                            }
+                            currentRunTree.events.push({
+                                name: "new_token",
+                                time: new Date().toISOString(),
+                                kwargs: {
+                                    token: value,
+                                },
+                            });
+                        }
                         chunks.push(value);
                         yield value;
                     }

package/dist/traceable.js

@@ -360,6 +360,18 @@ export function traceable(wrappedFunc, config) {
                                 controller.close();
                                 break;
                             }
+                            if (currentRunTree?.run_type === "llm") {
+                                if (currentRunTree.events === undefined) {
+                                    currentRunTree.events = [];
+                                }
+                                currentRunTree.events.push({
+                                    name: "new_token",
+                                    time: new Date().toISOString(),
+                                    kwargs: {
+                                        token: result.value,
+                                    },
+                                });
+                            }
                             chunks.push(result.value);
                             controller.enqueue(result.value);
                         }
@@ -386,6 +398,18 @@ export function traceable(wrappedFunc, config) {
                             finished = true;
                             break;
                         }
+                        if (currentRunTree?.run_type === "llm") {
+                            if (currentRunTree.events === undefined) {
+                                currentRunTree.events = [];
+                            }
+                            currentRunTree.events.push({
+                                name: "new_token",
+                                time: new Date().toISOString(),
+                                kwargs: {
+                                    token: value,
+                                },
+                            });
+                        }
                         chunks.push(value);
                         yield value;
                     }

package/dist/utils/jestlike/constants.cjs

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TEST_ID_DELIMITER = exports.STRIP_ANSI_REGEX = exports.UUID5_NAMESPACE = exports.DEFAULT_TEST_TIMEOUT = void 0;
+exports.DEFAULT_TEST_TIMEOUT = 30_000;
+exports.UUID5_NAMESPACE = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
+// From https://stackoverflow.com/a/29497680
+exports.STRIP_ANSI_REGEX = 
+// eslint-disable-next-line no-control-regex
+/[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g;
+exports.TEST_ID_DELIMITER = ", test_id=";

package/dist/utils/jestlike/constants.d.ts

@@ -0,0 +1,4 @@
+export declare const DEFAULT_TEST_TIMEOUT = 30000;
+export declare const UUID5_NAMESPACE = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
+export declare const STRIP_ANSI_REGEX: RegExp;
+export declare const TEST_ID_DELIMITER = ", test_id=";

package/dist/utils/jestlike/constants.js

@@ -0,0 +1,7 @@
+export const DEFAULT_TEST_TIMEOUT = 30_000;
+export const UUID5_NAMESPACE = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
+// From https://stackoverflow.com/a/29497680
+export const STRIP_ANSI_REGEX = 
+// eslint-disable-next-line no-control-regex
+/[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g;
+export const TEST_ID_DELIMITER = ", test_id=";

package/dist/utils/jestlike/index.cjs

@@ -24,11 +24,14 @@ var __importStar = (this && this.__importStar) || function (mod) {
     __setModuleDefault(result, mod);
     return result;
 };
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateWrapperFromJestlikeMethods = exports.objectHash = exports.logOutputs = exports.logFeedback = exports.TEST_ID_DELIMITER = exports.STRIP_ANSI_REGEX = void 0;
+exports.wrapEvaluator = exports.generateWrapperFromJestlikeMethods = exports._objectHash = exports.logOutputs = exports.logFeedback = void 0;
 const crypto_1 = __importDefault(require("crypto"));
 const uuid_1 = require("uuid");
 const os = __importStar(require("node:os"));
@@ -41,13 +44,7 @@ const matchers_js_1 = require("./matchers.cjs");
 const globals_js_1 = require("./globals.cjs");
 const chain_js_1 = require("./vendor/chain.cjs");
 const env_js_1 = require("../env.cjs");
-const DEFAULT_TEST_TIMEOUT = 30_000;
-const UUID5_NAMESPACE = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
-// From https://stackoverflow.com/a/29497680
-exports.STRIP_ANSI_REGEX = 
-// eslint-disable-next-line no-control-regex
-/[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g;
-exports.TEST_ID_DELIMITER = ", test_id=";
+const constants_js_1 = require("./constants.cjs");
 function logFeedback(feedback, config) {
     const context = globals_js_1.testWrapperAsyncLocalStorageInstance.getStore();
     if (context === undefined) {
@@ -87,19 +84,19 @@ function logOutputs(output) {
     context.setLoggedOutput(output);
 }
 exports.logOutputs = logOutputs;
-function objectHash(obj, depth = 0) {
+function _objectHash(obj, depth = 0) {
     // Prevent infinite recursion
     if (depth > 50) {
         throw new Error("Object is too deep to check equality for serialization. Please use a simpler example.");
     }
     if (Array.isArray(obj)) {
-        const arrayHash = obj.map((item) => objectHash(item, depth + 1)).join(",");
+        const arrayHash = obj.map((item) => _objectHash(item, depth + 1)).join(",");
         return crypto_1.default.createHash("sha256").update(arrayHash).digest("hex");
     }
     if (obj && typeof obj === "object") {
         const sortedHash = Object.keys(obj)
             .sort()
-            .map((key) => `${key}:${objectHash(obj[key], depth + 1)}`)
+            .map((key) => `${key}:${_objectHash(obj[key], depth + 1)}`)
             .join(",");
         return crypto_1.default.createHash("sha256").update(sortedHash).digest("hex");
     }
@@ -109,7 +106,7 @@ function objectHash(obj, depth = 0) {
         .update(JSON.stringify(obj ?? null))
         .digest("hex"));
 }
-exports.objectHash = objectHash;
+exports._objectHash = _objectHash;
 function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
     const { expect, test, describe, beforeAll, afterAll } = methods;
     async function _createProject(client, datasetId, projectConfig) {
@@ -143,18 +140,18 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
     function getExampleId(datasetId, inputs, outputs) {
         const identifier = JSON.stringify({
             datasetId,
-            inputsHash: objectHash(inputs),
-            outputsHash: objectHash(outputs ?? {}),
+            inputsHash: _objectHash(inputs),
+            outputsHash: _objectHash(outputs ?? {}),
         });
-        return (0, uuid_1.v5)(identifier, UUID5_NAMESPACE);
+        return (0, uuid_1.v5)(identifier, constants_js_1.UUID5_NAMESPACE);
     }
     async function syncExample(params) {
         const { client, exampleId, inputs, outputs, metadata, createdAt, datasetId, } = params;
         let example;
         try {
             example = await client.readExample(exampleId);
-            if (objectHash(example.inputs) !== objectHash(inputs) ||
-                objectHash(example.outputs ?? {}) !== objectHash(outputs ?? {}) ||
+            if (_objectHash(example.inputs) !== _objectHash(inputs) ||
+                _objectHash(example.outputs ?? {}) !== _objectHash(outputs ?? {}) ||
                 example.dataset_id !== datasetId) {
                 await client.updateExample(exampleId, {
                     inputs,
@@ -362,7 +359,7 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                 // Jest will not group tests under the same "describe" group if you await the test and
                 // total runs is greater than 1.
                 const resultsPath = path.join(os.tmpdir(), "langsmith_test_results", `${testUuid}.json`);
-                void method(`${name}${totalRuns > 1 ? `, run ${i}` : ""}${exports.TEST_ID_DELIMITER}${testUuid}`, async () => {
+                void method(`${name}${totalRuns > 1 ? `, run ${i}` : ""}${constants_js_1.TEST_ID_DELIMITER}${testUuid}`, async () => {
                     if (context === undefined) {
                         throw new Error([
                             `Could not retrieve test context.`,
@@ -428,7 +425,7 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                                 client: testContext.client,
                             });
                             const rawError = e;
-                            const strippedErrorMessage = e.message.replace(exports.STRIP_ANSI_REGEX, "");
+                            const strippedErrorMessage = e.message.replace(constants_js_1.STRIP_ANSI_REGEX, "");
                             const langsmithFriendlyError = new Error(strippedErrorMessage);
                             langsmithFriendlyError.rawJestError = rawError;
                             throw langsmithFriendlyError;
@@ -536,7 +533,7 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                             experimentUrl,
                         }));
                     }
-                }, timeout ?? DEFAULT_TEST_TIMEOUT);
+                }, timeout ?? constants_js_1.DEFAULT_TEST_TIMEOUT);
             }
         };
     }
@@ -584,3 +581,6 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
     };
 }
 exports.generateWrapperFromJestlikeMethods = generateWrapperFromJestlikeMethods;
+var evaluatedBy_js_1 = require("./vendor/evaluatedBy.cjs");
+Object.defineProperty(exports, "wrapEvaluator", { enumerable: true, get: function () { return evaluatedBy_js_1.wrapEvaluator; } });
+__exportStar(require("./types.cjs"), exports);

package/dist/utils/jestlike/index.d.ts

@@ -3,13 +3,11 @@ import { KVMap } from "../../schemas.js";
 import { toBeRelativeCloseTo, toBeAbsoluteCloseTo, toBeSemanticCloseTo } from "./matchers.js";
 import { SimpleEvaluationResult } from "./types.js";
 import type { LangSmithJestlikeWrapperConfig, LangSmithJestlikeWrapperParams, LangSmithJestlikeDescribeWrapper } from "./types.js";
-export declare const STRIP_ANSI_REGEX: RegExp;
-export declare const TEST_ID_DELIMITER = ", test_id=";
 export declare function logFeedback(feedback: SimpleEvaluationResult, config?: {
     sourceRunId?: string;
 }): void;
 export declare function logOutputs(output: Record<string, unknown>): void;
-export declare function objectHash(obj: KVMap, depth?: number): string;
+export declare function _objectHash(obj: KVMap, depth?: number): string;
 export declare function generateWrapperFromJestlikeMethods(methods: Record<string, any>, testRunnerName: string): {
     test: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(name: string, lsParams: LangSmithJestlikeWrapperParams<I, O>, testFn: (data: {
         inputs: I;
@@ -92,3 +90,5 @@ export declare function generateWrapperFromJestlikeMethods(methods: Record<strin
     toBeAbsoluteCloseTo: typeof toBeAbsoluteCloseTo;
     toBeSemanticCloseTo: typeof toBeSemanticCloseTo;
 };
+export { wrapEvaluator } from "./vendor/evaluatedBy.js";
+export * from "./types.js";

package/dist/utils/jestlike/index.js

@@ -12,13 +12,7 @@ import { toBeRelativeCloseTo, toBeAbsoluteCloseTo, toBeSemanticCloseTo, } from "
 import { evaluatorLogFeedbackPromises, testWrapperAsyncLocalStorageInstance, _logTestFeedback, syncExamplePromises, trackingEnabled, DEFAULT_TEST_CLIENT, } from "./globals.js";
 import { wrapExpect } from "./vendor/chain.js";
 import { getEnvironmentVariable } from "../env.js";
-const DEFAULT_TEST_TIMEOUT = 30_000;
-const UUID5_NAMESPACE = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
-// From https://stackoverflow.com/a/29497680
-export const STRIP_ANSI_REGEX = 
-// eslint-disable-next-line no-control-regex
-/[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g;
-export const TEST_ID_DELIMITER = ", test_id=";
+import { STRIP_ANSI_REGEX, TEST_ID_DELIMITER, DEFAULT_TEST_TIMEOUT, UUID5_NAMESPACE, } from "./constants.js";
 export function logFeedback(feedback, config) {
     const context = testWrapperAsyncLocalStorageInstance.getStore();
     if (context === undefined) {
@@ -56,19 +50,19 @@ export function logOutputs(output) {
     }
     context.setLoggedOutput(output);
 }
-export function objectHash(obj, depth = 0) {
+export function _objectHash(obj, depth = 0) {
     // Prevent infinite recursion
     if (depth > 50) {
         throw new Error("Object is too deep to check equality for serialization. Please use a simpler example.");
     }
     if (Array.isArray(obj)) {
-        const arrayHash = obj.map((item) => objectHash(item, depth + 1)).join(",");
+        const arrayHash = obj.map((item) => _objectHash(item, depth + 1)).join(",");
         return crypto.createHash("sha256").update(arrayHash).digest("hex");
     }
     if (obj && typeof obj === "object") {
         const sortedHash = Object.keys(obj)
             .sort()
-            .map((key) => `${key}:${objectHash(obj[key], depth + 1)}`)
+            .map((key) => `${key}:${_objectHash(obj[key], depth + 1)}`)
             .join(",");
         return crypto.createHash("sha256").update(sortedHash).digest("hex");
     }
@@ -111,8 +105,8 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
     function getExampleId(datasetId, inputs, outputs) {
         const identifier = JSON.stringify({
             datasetId,
-            inputsHash: objectHash(inputs),
-            outputsHash: objectHash(outputs ?? {}),
+            inputsHash: _objectHash(inputs),
+            outputsHash: _objectHash(outputs ?? {}),
         });
         return v5(identifier, UUID5_NAMESPACE);
     }
@@ -121,8 +115,8 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         let example;
         try {
             example = await client.readExample(exampleId);
-            if (objectHash(example.inputs) !== objectHash(inputs) ||
-                objectHash(example.outputs ?? {}) !== objectHash(outputs ?? {}) ||
+            if (_objectHash(example.inputs) !== _objectHash(inputs) ||
+                _objectHash(example.outputs ?? {}) !== _objectHash(outputs ?? {}) ||
                 example.dataset_id !== datasetId) {
                 await client.updateExample(exampleId, {
                     inputs,
@@ -551,3 +545,5 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
         toBeSemanticCloseTo,
     };
 }
+export { wrapEvaluator } from "./vendor/evaluatedBy.js";
+export * from "./types.js";

package/dist/utils/jestlike/reporter.cjs

@@ -32,7 +32,7 @@ const chalk_1 = __importDefault(require("chalk"));
 const os = __importStar(require("node:os"));
 const path = __importStar(require("node:path"));
 const fs = __importStar(require("node:fs/promises"));
-const index_js_1 = require("./index.cjs");
+const constants_js_1 = require("./constants.cjs");
 const FEEDBACK_COLLAPSE_THRESHOLD = 48;
 const MAX_TEST_PARAMS_LENGTH = 18;
 const RESERVED_KEYS = [
@@ -105,13 +105,13 @@ async function printReporterTable(testSuiteName, results, testStatus, failureMes
     let experimentUrl;
     for (const result of results) {
         const { title, duration, status } = result;
-        const titleComponents = title.split(index_js_1.TEST_ID_DELIMITER);
+        const titleComponents = title.split(constants_js_1.TEST_ID_DELIMITER);
         const testId = titleComponents.length > 1 && titleComponents.at(-1) !== undefined
             ? titleComponents.at(-1)
             : undefined;
         const testName = testId !== undefined
-            ? titleComponents.slice(0, -1).join(index_js_1.TEST_ID_DELIMITER).trim()
-            : titleComponents.join(index_js_1.TEST_ID_DELIMITER);
+            ? titleComponents.slice(0, -1).join(constants_js_1.TEST_ID_DELIMITER).trim()
+            : titleComponents.join(constants_js_1.TEST_ID_DELIMITER);
         // Non-LangSmith test
         if (testId === undefined) {
             rows.push([
@@ -245,7 +245,7 @@ async function printReporterTable(testSuiteName, results, testStatus, failureMes
     if (collapseFeedbackColumn) {
         const feedbackColumnLength = rows.reduce((max, [row]) => {
             const maxFeedbackLineLength = row.Feedback?.split("\n").reduce((max, feedbackLine) => {
-                return Math.max(max, feedbackLine.replace(index_js_1.STRIP_ANSI_REGEX, "").length);
+                return Math.max(max, feedbackLine.replace(constants_js_1.STRIP_ANSI_REGEX, "").length);
             }, 0) ?? 0;
             return Math.max(max, maxFeedbackLineLength);
         }, 0);

package/dist/utils/jestlike/reporter.js

@@ -3,7 +3,7 @@ import chalk from "chalk";
 import * as os from "node:os";
 import * as path from "node:path";
 import * as fs from "node:fs/promises";
-import { STRIP_ANSI_REGEX, TEST_ID_DELIMITER } from "./index.js";
+import { STRIP_ANSI_REGEX, TEST_ID_DELIMITER } from "./constants.js";
 const FEEDBACK_COLLAPSE_THRESHOLD = 48;
 const MAX_TEST_PARAMS_LENGTH = 18;
 const RESERVED_KEYS = [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.3.7",
+  "version": "0.3.9-rc.0",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [
@@ -73,6 +73,10 @@
     "singletons/traceable.js",
     "singletons/traceable.d.ts",
     "singletons/traceable.d.cts",
+    "utils/jestlike.cjs",
+    "utils/jestlike.js",
+    "utils/jestlike.d.ts",
+    "utils/jestlike.d.cts",
     "index.cjs",
     "index.js",
     "index.d.ts",
@@ -162,7 +166,7 @@
     "typedoc": "^0.27.6",
     "typedoc-plugin-expand-object-like-types": "^0.1.2",
     "typescript": "^5.4.5",
-    "vitest": "^2.1.8",
+    "vitest": "^3.0.5",
     "zod": "^3.23.8"
   },
   "peerDependencies": {
@@ -342,6 +346,15 @@
       "import": "./singletons/traceable.js",
       "require": "./singletons/traceable.cjs"
     },
+    "./utils/jestlike": {
+      "types": {
+        "import": "./utils/jestlike.d.ts",
+        "require": "./utils/jestlike.d.cts",
+        "default": "./utils/jestlike.d.ts"
+      },
+      "import": "./utils/jestlike.js",
+      "require": "./utils/jestlike.cjs"
+    },
     "./package.json": "./package.json"
   }
 }

package/utils/jestlike.cjs

@@ -0,0 +1 @@
+module.exports = require('../dist/utils/jestlike/index.cjs');

package/utils/jestlike.d.cts

@@ -0,0 +1 @@
+export * from '../dist/utils/jestlike/index.js'

package/utils/jestlike.d.ts

@@ -0,0 +1 @@
+export * from '../dist/utils/jestlike/index.js'

package/utils/jestlike.js

@@ -0,0 +1 @@
+export * from '../dist/utils/jestlike/index.js'