langsmith 0.2.14-rc.2 → 0.2.14-rc.4

package/dist/index.cjs

@@ -8,4 +8,4 @@ Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () {
 var fetch_js_1 = require("./singletons/fetch.cjs");
 Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true, get: function () { return fetch_js_1.overrideFetchImplementation; } });
 // Update using yarn bump-version
-exports.__version__ = "0.2.14-rc.2";
+exports.__version__ = "0.2.14-rc.4";

package/dist/index.d.ts

@@ -2,4 +2,4 @@ export { Client, type ClientConfig, type LangSmithTracingClientInterface, } from
 export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
-export declare const __version__ = "0.2.14-rc.2";
+export declare const __version__ = "0.2.14-rc.4";

package/dist/index.js

@@ -2,4 +2,4 @@ export { Client, } from "./client.js";
 export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 // Update using yarn bump-version
-export const __version__ = "0.2.14-rc.2";
+export const __version__ = "0.2.14-rc.4";

package/dist/jest/index.cjs

@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.expect = exports.describe = exports.it = exports.test = void 0;
 const globals_1 = require("@jest/globals");
 const crypto_1 = __importDefault(require("crypto"));
 const uuid_1 = require("uuid");
@@ -14,11 +15,30 @@ const _random_name_js_1 = require("../evaluation/_random_name.cjs");
 const matchers_js_1 = require("./matchers.cjs");
 const globals_js_1 = require("./globals.cjs");
 const chain_js_1 = __importDefault(require("./vendor/chain.cjs"));
+exports.expect = chain_js_1.default;
 globals_1.expect.extend({
     toBeRelativeCloseTo: matchers_js_1.toBeRelativeCloseTo,
     toBeAbsoluteCloseTo: matchers_js_1.toBeAbsoluteCloseTo,
     toBeSemanticCloseTo: matchers_js_1.toBeSemanticCloseTo,
 });
+const objectHash = (obj, depth = 0) => {
+    // Prevent infinite recursion
+    if (depth > 50) {
+        return "[Max Depth Exceeded]";
+    }
+    if (Array.isArray(obj)) {
+        return obj.map((item) => objectHash(item, depth + 1));
+    }
+    if (obj && typeof obj === "object") {
+        return Object.keys(obj)
+            .sort()
+            .reduce((result, key) => {
+            result[key] = objectHash(obj[key], depth + 1);
+            return result;
+        }, {});
+    }
+    return crypto_1.default.createHash("sha256").update(JSON.stringify(obj)).digest("hex");
+};
 async function _createProject(client, datasetId) {
     // Create the project, updating the experimentName until we find a unique one.
     let project;
@@ -76,14 +96,8 @@ async function runDatasetSetup(testClient, datasetName) {
         });
         const examples = [];
         for await (const example of examplesList) {
-            const inputHash = crypto_1.default
-                .createHash("sha256")
-                .update(JSON.stringify(example.inputs))
-                .digest("hex");
-            const outputHash = crypto_1.default
-                .createHash("sha256")
-                .update(JSON.stringify(example.inputs))
-                .digest("hex");
+            const inputHash = objectHash(example.inputs);
+            const outputHash = objectHash(example.outputs ?? {});
             examples.push({ ...example, inputHash, outputHash });
         }
         const project = await _createProject(testClient, dataset.id);
@@ -122,8 +136,12 @@ const lsDescribe = Object.assign(wrapDescribeMethod(globals_1.describe), {
     only: wrapDescribeMethod(globals_1.describe.only),
     skip: wrapDescribeMethod(globals_1.describe.skip),
 });
+exports.describe = lsDescribe;
 function wrapTestMethod(method) {
     return function (params, config) {
+        // Due to https://github.com/jestjs/jest/issues/13653,
+        // we must access the local store value here before
+        // entering an async context
         const context = globals_js_1.jestAsyncLocalStorageInstance.getStore();
         // This typing is wrong, but necessary to avoid lint errors
         // eslint-disable-next-line @typescript-eslint/no-misused-promises
@@ -140,14 +158,8 @@ function wrapTestMethod(method) {
                 const { examples, dataset, createdAt, project, client } = await setupPromises.get(context.suiteUuid);
                 const testInput = typeof params === "string" ? {} : params.inputs;
                 const testOutput = typeof params === "string" ? {} : params.outputs;
-                const inputHash = crypto_1.default
-                    .createHash("sha256")
-                    .update(JSON.stringify(testInput))
-                    .digest("hex");
-                const outputHash = crypto_1.default
-                    .createHash("sha256")
-                    .update(JSON.stringify(testOutput))
-                    .digest("hex");
+                const inputHash = objectHash(testInput);
+                const outputHash = objectHash(testOutput ?? {});
                 if ((0, globals_js_1.trackingEnabled)()) {
                     const missingFields = [];
                     if (examples === undefined) {
@@ -224,13 +236,18 @@ function wrapTestMethod(method) {
         };
     };
 }
+function eachMethod(table) {
+    return function (name, fn, timeout) {
+        for (let i = 0; i < table.length; i += 1) {
+            const example = table[i];
+            wrapTestMethod(globals_1.test)(example)(`${name} ${i}`, fn, timeout);
+        }
+    };
+}
 const lsTest = Object.assign(wrapTestMethod(globals_1.test), {
     only: wrapTestMethod(globals_1.test.only),
     skip: wrapTestMethod(globals_1.test.skip),
+    each: eachMethod,
 });
-exports.default = {
-    test: lsTest,
-    it: lsTest,
-    describe: lsDescribe,
-    expect: chain_js_1.default,
-};
+exports.test = lsTest;
+exports.it = lsTest;

package/dist/jest/index.d.ts

@@ -1,4 +1,5 @@
 import { RunTreeConfig } from "../run_trees.js";
+import { KVMap } from "../schemas.js";
 import expectWithGradedBy from "./vendor/chain.js";
 import type { SimpleEvaluator } from "./vendor/gradedBy.js";
 declare global {
@@ -21,42 +22,34 @@ declare global {
     }
 }
 export type LangSmithJestDescribeWrapper = (name: string, fn: () => void | Promise<void>, config?: Partial<RunTreeConfig>) => void;
+declare const lsDescribe: LangSmithJestDescribeWrapper & {
+    only: LangSmithJestDescribeWrapper;
+    skip: LangSmithJestDescribeWrapper;
+};
 export type LangSmithJestTestWrapper<I, O> = (name: string, fn: (params: {
     inputs: I;
     outputs: O;
 }) => unknown | Promise<unknown>, timeout?: number) => void;
-declare const _default: {
-    test: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: string | {
+declare function eachMethod<I extends KVMap, O extends KVMap>(table: {
+    inputs: I;
+    outputs: O;
+}[]): (name: string, fn: (params: {
+    inputs: I;
+    outputs: O;
+}) => unknown | Promise<unknown>, timeout?: number) => void;
+declare const lsTest: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: {
+    inputs: I;
+    outputs: O;
+} | string, config?: Partial<RunTreeConfig>) => LangSmithJestTestWrapper<I, O>) & {
+    only: <I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: {
         inputs: I;
         outputs: O;
-    }, config?: Partial<RunTreeConfig> | undefined) => LangSmithJestTestWrapper<I, O>) & {
-        only: <I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: string | {
-            inputs: I;
-            outputs: O;
-        }, config?: Partial<RunTreeConfig> | undefined) => LangSmithJestTestWrapper<I, O>;
-        skip: <I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: string | {
-            inputs: I;
-            outputs: O;
-        }, config?: Partial<RunTreeConfig> | undefined) => LangSmithJestTestWrapper<I, O>;
-    };
-    it: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: string | {
+    } | string, config?: Partial<RunTreeConfig>) => LangSmithJestTestWrapper<I, O>;
+    skip: <I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: {
         inputs: I;
         outputs: O;
-    }, config?: Partial<RunTreeConfig> | undefined) => LangSmithJestTestWrapper<I, O>) & {
-        only: <I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: string | {
-            inputs: I;
-            outputs: O;
-        }, config?: Partial<RunTreeConfig> | undefined) => LangSmithJestTestWrapper<I, O>;
-        skip: <I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: string | {
-            inputs: I;
-            outputs: O;
-        }, config?: Partial<RunTreeConfig> | undefined) => LangSmithJestTestWrapper<I, O>;
-    };
-    describe: LangSmithJestDescribeWrapper & {
-        only: LangSmithJestDescribeWrapper;
-        skip: LangSmithJestDescribeWrapper;
-    };
-    expect: typeof expectWithGradedBy;
+    } | string, config?: Partial<RunTreeConfig>) => LangSmithJestTestWrapper<I, O>;
+    each: typeof eachMethod;
 };
-export default _default;
+export { lsTest as test, lsTest as it, lsDescribe as describe, expectWithGradedBy as expect, };
 export { type SimpleEvaluator };

package/dist/jest/index.js

@@ -14,6 +14,24 @@ expect.extend({
     toBeAbsoluteCloseTo,
     toBeSemanticCloseTo,
 });
+const objectHash = (obj, depth = 0) => {
+    // Prevent infinite recursion
+    if (depth > 50) {
+        return "[Max Depth Exceeded]";
+    }
+    if (Array.isArray(obj)) {
+        return obj.map((item) => objectHash(item, depth + 1));
+    }
+    if (obj && typeof obj === "object") {
+        return Object.keys(obj)
+            .sort()
+            .reduce((result, key) => {
+            result[key] = objectHash(obj[key], depth + 1);
+            return result;
+        }, {});
+    }
+    return crypto.createHash("sha256").update(JSON.stringify(obj)).digest("hex");
+};
 async function _createProject(client, datasetId) {
     // Create the project, updating the experimentName until we find a unique one.
     let project;
@@ -71,14 +89,8 @@ async function runDatasetSetup(testClient, datasetName) {
         });
         const examples = [];
         for await (const example of examplesList) {
-            const inputHash = crypto
-                .createHash("sha256")
-                .update(JSON.stringify(example.inputs))
-                .digest("hex");
-            const outputHash = crypto
-                .createHash("sha256")
-                .update(JSON.stringify(example.inputs))
-                .digest("hex");
+            const inputHash = objectHash(example.inputs);
+            const outputHash = objectHash(example.outputs ?? {});
             examples.push({ ...example, inputHash, outputHash });
         }
         const project = await _createProject(testClient, dataset.id);
@@ -119,6 +131,9 @@ const lsDescribe = Object.assign(wrapDescribeMethod(describe), {
 });
 function wrapTestMethod(method) {
     return function (params, config) {
+        // Due to https://github.com/jestjs/jest/issues/13653,
+        // we must access the local store value here before
+        // entering an async context
         const context = jestAsyncLocalStorageInstance.getStore();
         // This typing is wrong, but necessary to avoid lint errors
         // eslint-disable-next-line @typescript-eslint/no-misused-promises
@@ -135,14 +150,8 @@ function wrapTestMethod(method) {
                 const { examples, dataset, createdAt, project, client } = await setupPromises.get(context.suiteUuid);
                 const testInput = typeof params === "string" ? {} : params.inputs;
                 const testOutput = typeof params === "string" ? {} : params.outputs;
-                const inputHash = crypto
-                    .createHash("sha256")
-                    .update(JSON.stringify(testInput))
-                    .digest("hex");
-                const outputHash = crypto
-                    .createHash("sha256")
-                    .update(JSON.stringify(testOutput))
-                    .digest("hex");
+                const inputHash = objectHash(testInput);
+                const outputHash = objectHash(testOutput ?? {});
                 if (trackingEnabled()) {
                     const missingFields = [];
                     if (examples === undefined) {
@@ -219,13 +228,17 @@ function wrapTestMethod(method) {
         };
     };
 }
+function eachMethod(table) {
+    return function (name, fn, timeout) {
+        for (let i = 0; i < table.length; i += 1) {
+            const example = table[i];
+            wrapTestMethod(test)(example)(`${name} ${i}`, fn, timeout);
+        }
+    };
+}
 const lsTest = Object.assign(wrapTestMethod(test), {
     only: wrapTestMethod(test.only),
     skip: wrapTestMethod(test.skip),
+    each: eachMethod,
 });
-export default {
-    test: lsTest,
-    it: lsTest,
-    describe: lsDescribe,
-    expect: expectWithGradedBy,
-};
+export { lsTest as test, lsTest as it, lsDescribe as describe, expectWithGradedBy as expect, };

package/dist/jest/matchers.cjs

@@ -99,51 +99,3 @@ async function toBeSemanticCloseTo(received, expected, options) {
     };
 }
 exports.toBeSemanticCloseTo = toBeSemanticCloseTo;
-// export async function toPassEvaluator(
-//   this: MatcherContext,
-//   actual: KVMap,
-//   evaluator: SimpleEvaluator,
-//   _expected?: KVMap
-// ) {
-//   const runTree = getCurrentRunTree();
-//   const context = localStorage.getStore();
-//   if (context === undefined || context.currentExample === undefined) {
-//     throw new Error(
-//       `Could not identify example context from current context.\nPlease ensure you are calling this matcher within "ls.test()"`
-//     );
-//   }
-//   const wrappedEvaluator = traceable(evaluator, {
-//     reference_example_id: context.currentExample.id,
-//     metadata: {
-//       example_version: context.currentExample.modified_at
-//         ? new Date(context.currentExample.modified_at).toISOString()
-//         : new Date(context.currentExample.created_at).toISOString(),
-//     },
-//     client: context.client,
-//     tracingEnabled: true,
-//   });
-//   const evalResult = await wrappedEvaluator({
-//     input: runTree.inputs,
-//     expected: context.currentExample.outputs ?? {},
-//     actual,
-//   });
-//   await context.client.logEvaluationFeedback(evalResult, runTree);
-//   if (!("results" in evalResult) && !evalResult.score) {
-//     return {
-//       pass: false,
-//       message: () =>
-//         `expected ${this.utils.printReceived(
-//           actual
-//         )} to pass evaluator. Failed with ${JSON.stringify(
-//           evalResult,
-//           null,
-//           2
-//         )}`,
-//     };
-//   }
-//   return {
-//     pass: true,
-//     message: () =>
-//       `evaluator passed with score ${JSON.stringify(evalResult, null, 2)}`,
-//   };
-// }

package/dist/jest/matchers.js

@@ -93,51 +93,3 @@ export async function toBeSemanticCloseTo(received, expected, options) {
             : `Expected "${received}" to be semantically close to "${expected}" (threshold: ${threshold}, similarity: ${similarity})`,
     };
 }
-// export async function toPassEvaluator(
-//   this: MatcherContext,
-//   actual: KVMap,
-//   evaluator: SimpleEvaluator,
-//   _expected?: KVMap
-// ) {
-//   const runTree = getCurrentRunTree();
-//   const context = localStorage.getStore();
-//   if (context === undefined || context.currentExample === undefined) {
-//     throw new Error(
-//       `Could not identify example context from current context.\nPlease ensure you are calling this matcher within "ls.test()"`
-//     );
-//   }
-//   const wrappedEvaluator = traceable(evaluator, {
-//     reference_example_id: context.currentExample.id,
-//     metadata: {
-//       example_version: context.currentExample.modified_at
-//         ? new Date(context.currentExample.modified_at).toISOString()
-//         : new Date(context.currentExample.created_at).toISOString(),
-//     },
-//     client: context.client,
-//     tracingEnabled: true,
-//   });
-//   const evalResult = await wrappedEvaluator({
-//     input: runTree.inputs,
-//     expected: context.currentExample.outputs ?? {},
-//     actual,
-//   });
-//   await context.client.logEvaluationFeedback(evalResult, runTree);
-//   if (!("results" in evalResult) && !evalResult.score) {
-//     return {
-//       pass: false,
-//       message: () =>
-//         `expected ${this.utils.printReceived(
-//           actual
-//         )} to pass evaluator. Failed with ${JSON.stringify(
-//           evalResult,
-//           null,
-//           2
-//         )}`,
-//     };
-//   }
-//   return {
-//     pass: true,
-//     message: () =>
-//       `evaluator passed with score ${JSON.stringify(evalResult, null, 2)}`,
-//   };
-// }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.2.14-rc.2",
+  "version": "0.2.14-rc.4",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [