npm - langsmith - Versions diffs - 0.2.14-rc.2 → 0.2.14-rc.3 - Mend

langsmith 0.2.14-rc.2 → 0.2.14-rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (9) hide show

package/dist/index.cjs CHANGED Viewed

@@ -8,4 +8,4 @@ Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () {
 var fetch_js_1 = require("./singletons/fetch.cjs");
 Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true, get: function () { return fetch_js_1.overrideFetchImplementation; } });
 // Update using yarn bump-version
-exports.__version__ = "0.2.14-rc.2";
+exports.__version__ = "0.2.14-rc.3";

package/dist/index.d.ts CHANGED Viewed

@@ -2,4 +2,4 @@ export { Client, type ClientConfig, type LangSmithTracingClientInterface, } from
 export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
-export declare const __version__ = "0.2.14-rc.2";
+export declare const __version__ = "0.2.14-rc.3";

package/dist/index.js CHANGED Viewed

@@ -2,4 +2,4 @@ export { Client, } from "./client.js";
 export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 // Update using yarn bump-version
-export const __version__ = "0.2.14-rc.2";
+export const __version__ = "0.2.14-rc.3";

package/dist/jest/index.cjs CHANGED Viewed

@@ -19,6 +19,24 @@ globals_1.expect.extend({
     toBeAbsoluteCloseTo: matchers_js_1.toBeAbsoluteCloseTo,
     toBeSemanticCloseTo: matchers_js_1.toBeSemanticCloseTo,
 });
+const objectHash = (obj, depth = 0) => {
+    // Prevent infinite recursion
+    if (depth > 50) {
+        return "[Max Depth Exceeded]";
+    }
+    if (Array.isArray(obj)) {
+        return obj.map((item) => objectHash(item, depth + 1));
+    }
+    if (obj && typeof obj === "object") {
+        return Object.keys(obj)
+            .sort()
+            .reduce((result, key) => {
+            result[key] = objectHash(obj[key], depth + 1);
+            return result;
+        }, {});
+    }
+    return crypto_1.default.createHash("sha256").update(JSON.stringify(obj)).digest("hex");
+};
 async function _createProject(client, datasetId) {
     // Create the project, updating the experimentName until we find a unique one.
     let project;
@@ -76,14 +94,8 @@ async function runDatasetSetup(testClient, datasetName) {
         });
         const examples = [];
         for await (const example of examplesList) {
-            const inputHash = crypto_1.default
-                .createHash("sha256")
-                .update(JSON.stringify(example.inputs))
-                .digest("hex");
-            const outputHash = crypto_1.default
-                .createHash("sha256")
-                .update(JSON.stringify(example.inputs))
-                .digest("hex");
+            const inputHash = objectHash(example.inputs);
+            const outputHash = objectHash(example.outputs ?? {});
             examples.push({ ...example, inputHash, outputHash });
         }
         const project = await _createProject(testClient, dataset.id);
@@ -124,6 +136,9 @@ const lsDescribe = Object.assign(wrapDescribeMethod(globals_1.describe), {
 });
 function wrapTestMethod(method) {
     return function (params, config) {
+        // Due to https://github.com/jestjs/jest/issues/13653,
+        // we must access the local store value here before
+        // entering an async context
         const context = globals_js_1.jestAsyncLocalStorageInstance.getStore();
         // This typing is wrong, but necessary to avoid lint errors
         // eslint-disable-next-line @typescript-eslint/no-misused-promises
@@ -140,14 +155,8 @@ function wrapTestMethod(method) {
                 const { examples, dataset, createdAt, project, client } = await setupPromises.get(context.suiteUuid);
                 const testInput = typeof params === "string" ? {} : params.inputs;
                 const testOutput = typeof params === "string" ? {} : params.outputs;
-                const inputHash = crypto_1.default
-                    .createHash("sha256")
-                    .update(JSON.stringify(testInput))
-                    .digest("hex");
-                const outputHash = crypto_1.default
-                    .createHash("sha256")
-                    .update(JSON.stringify(testOutput))
-                    .digest("hex");
+                const inputHash = objectHash(testInput);
+                const outputHash = objectHash(testOutput ?? {});
                 if ((0, globals_js_1.trackingEnabled)()) {
                     const missingFields = [];
                     if (examples === undefined) {
@@ -224,9 +233,18 @@ function wrapTestMethod(method) {
         };
     };
 }
+function eachMethod(table) {
+    return function (name, fn, timeout) {
+        for (let i = 0; i < table.length; i += 1) {
+            const example = table[i];
+            wrapTestMethod(globals_1.test)(example)(`${name} ${i}`, fn, timeout);
+        }
+    };
+}
 const lsTest = Object.assign(wrapTestMethod(globals_1.test), {
     only: wrapTestMethod(globals_1.test.only),
     skip: wrapTestMethod(globals_1.test.skip),
+    each: eachMethod,
 });
 exports.default = {
     test: lsTest,

package/dist/jest/index.d.ts CHANGED Viewed

@@ -1,4 +1,5 @@
 import { RunTreeConfig } from "../run_trees.js";
+import { KVMap } from "../schemas.js";
 import expectWithGradedBy from "./vendor/chain.js";
 import type { SimpleEvaluator } from "./vendor/gradedBy.js";
 declare global {
@@ -25,6 +26,13 @@ export type LangSmithJestTestWrapper<I, O> = (name: string, fn: (params: {
     inputs: I;
     outputs: O;
 }) => unknown | Promise<unknown>, timeout?: number) => void;
+declare function eachMethod<I extends KVMap, O extends KVMap>(table: {
+    inputs: I;
+    outputs: O;
+}[]): (name: string, fn: (params: {
+    inputs: I;
+    outputs: O;
+}) => unknown | Promise<unknown>, timeout?: number) => void;
 declare const _default: {
     test: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: string | {
         inputs: I;
@@ -38,6 +46,7 @@ declare const _default: {
             inputs: I;
             outputs: O;
         }, config?: Partial<RunTreeConfig> | undefined) => LangSmithJestTestWrapper<I, O>;
+        each: typeof eachMethod;
     };
     it: (<I extends Record<string, any> = Record<string, any>, O extends Record<string, any> = Record<string, any>>(params: string | {
         inputs: I;
@@ -51,6 +60,7 @@ declare const _default: {
             inputs: I;
             outputs: O;
         }, config?: Partial<RunTreeConfig> | undefined) => LangSmithJestTestWrapper<I, O>;
+        each: typeof eachMethod;
     };
     describe: LangSmithJestDescribeWrapper & {
         only: LangSmithJestDescribeWrapper;

package/dist/jest/index.js CHANGED Viewed

@@ -14,6 +14,24 @@ expect.extend({
     toBeAbsoluteCloseTo,
     toBeSemanticCloseTo,
 });
+const objectHash = (obj, depth = 0) => {
+    // Prevent infinite recursion
+    if (depth > 50) {
+        return "[Max Depth Exceeded]";
+    }
+    if (Array.isArray(obj)) {
+        return obj.map((item) => objectHash(item, depth + 1));
+    }
+    if (obj && typeof obj === "object") {
+        return Object.keys(obj)
+            .sort()
+            .reduce((result, key) => {
+            result[key] = objectHash(obj[key], depth + 1);
+            return result;
+        }, {});
+    }
+    return crypto.createHash("sha256").update(JSON.stringify(obj)).digest("hex");
+};
 async function _createProject(client, datasetId) {
     // Create the project, updating the experimentName until we find a unique one.
     let project;
@@ -71,14 +89,8 @@ async function runDatasetSetup(testClient, datasetName) {
         });
         const examples = [];
         for await (const example of examplesList) {
-            const inputHash = crypto
-                .createHash("sha256")
-                .update(JSON.stringify(example.inputs))
-                .digest("hex");
-            const outputHash = crypto
-                .createHash("sha256")
-                .update(JSON.stringify(example.inputs))
-                .digest("hex");
+            const inputHash = objectHash(example.inputs);
+            const outputHash = objectHash(example.outputs ?? {});
             examples.push({ ...example, inputHash, outputHash });
         }
         const project = await _createProject(testClient, dataset.id);
@@ -119,6 +131,9 @@ const lsDescribe = Object.assign(wrapDescribeMethod(describe), {
 });
 function wrapTestMethod(method) {
     return function (params, config) {
+        // Due to https://github.com/jestjs/jest/issues/13653,
+        // we must access the local store value here before
+        // entering an async context
         const context = jestAsyncLocalStorageInstance.getStore();
         // This typing is wrong, but necessary to avoid lint errors
         // eslint-disable-next-line @typescript-eslint/no-misused-promises
@@ -135,14 +150,8 @@ function wrapTestMethod(method) {
                 const { examples, dataset, createdAt, project, client } = await setupPromises.get(context.suiteUuid);
                 const testInput = typeof params === "string" ? {} : params.inputs;
                 const testOutput = typeof params === "string" ? {} : params.outputs;
-                const inputHash = crypto
-                    .createHash("sha256")
-                    .update(JSON.stringify(testInput))
-                    .digest("hex");
-                const outputHash = crypto
-                    .createHash("sha256")
-                    .update(JSON.stringify(testOutput))
-                    .digest("hex");
+                const inputHash = objectHash(testInput);
+                const outputHash = objectHash(testOutput ?? {});
                 if (trackingEnabled()) {
                     const missingFields = [];
                     if (examples === undefined) {
@@ -219,9 +228,18 @@ function wrapTestMethod(method) {
         };
     };
 }
+function eachMethod(table) {
+    return function (name, fn, timeout) {
+        for (let i = 0; i < table.length; i += 1) {
+            const example = table[i];
+            wrapTestMethod(test)(example)(`${name} ${i}`, fn, timeout);
+        }
+    };
+}
 const lsTest = Object.assign(wrapTestMethod(test), {
     only: wrapTestMethod(test.only),
     skip: wrapTestMethod(test.skip),
+    each: eachMethod,
 });
 export default {
     test: lsTest,

package/dist/jest/matchers.cjs CHANGED Viewed

@@ -99,51 +99,3 @@ async function toBeSemanticCloseTo(received, expected, options) {
     };
 }
 exports.toBeSemanticCloseTo = toBeSemanticCloseTo;
-// export async function toPassEvaluator(
-//   this: MatcherContext,
-//   actual: KVMap,
-//   evaluator: SimpleEvaluator,
-//   _expected?: KVMap
-// ) {
-//   const runTree = getCurrentRunTree();
-//   const context = localStorage.getStore();
-//   if (context === undefined || context.currentExample === undefined) {
-//     throw new Error(
-//       `Could not identify example context from current context.\nPlease ensure you are calling this matcher within "ls.test()"`
-//     );
-//   }
-//   const wrappedEvaluator = traceable(evaluator, {
-//     reference_example_id: context.currentExample.id,
-//     metadata: {
-//       example_version: context.currentExample.modified_at
-//         ? new Date(context.currentExample.modified_at).toISOString()
-//         : new Date(context.currentExample.created_at).toISOString(),
-//     },
-//     client: context.client,
-//     tracingEnabled: true,
-//   });
-//   const evalResult = await wrappedEvaluator({
-//     input: runTree.inputs,
-//     expected: context.currentExample.outputs ?? {},
-//     actual,
-//   });
-//   await context.client.logEvaluationFeedback(evalResult, runTree);
-//   if (!("results" in evalResult) && !evalResult.score) {
-//     return {
-//       pass: false,
-//       message: () =>
-//         `expected ${this.utils.printReceived(
-//           actual
-//         )} to pass evaluator. Failed with ${JSON.stringify(
-//           evalResult,
-//           null,
-//           2
-//         )}`,
-//     };
-//   }
-//   return {
-//     pass: true,
-//     message: () =>
-//       `evaluator passed with score ${JSON.stringify(evalResult, null, 2)}`,
-//   };
-// }

package/dist/jest/matchers.js CHANGED Viewed

@@ -93,51 +93,3 @@ export async function toBeSemanticCloseTo(received, expected, options) {
             : `Expected "${received}" to be semantically close to "${expected}" (threshold: ${threshold}, similarity: ${similarity})`,
     };
 }
-// export async function toPassEvaluator(
-//   this: MatcherContext,
-//   actual: KVMap,
-//   evaluator: SimpleEvaluator,
-//   _expected?: KVMap
-// ) {
-//   const runTree = getCurrentRunTree();
-//   const context = localStorage.getStore();
-//   if (context === undefined || context.currentExample === undefined) {
-//     throw new Error(
-//       `Could not identify example context from current context.\nPlease ensure you are calling this matcher within "ls.test()"`
-//     );
-//   }
-//   const wrappedEvaluator = traceable(evaluator, {
-//     reference_example_id: context.currentExample.id,
-//     metadata: {
-//       example_version: context.currentExample.modified_at
-//         ? new Date(context.currentExample.modified_at).toISOString()
-//         : new Date(context.currentExample.created_at).toISOString(),
-//     },
-//     client: context.client,
-//     tracingEnabled: true,
-//   });
-//   const evalResult = await wrappedEvaluator({
-//     input: runTree.inputs,
-//     expected: context.currentExample.outputs ?? {},
-//     actual,
-//   });
-//   await context.client.logEvaluationFeedback(evalResult, runTree);
-//   if (!("results" in evalResult) && !evalResult.score) {
-//     return {
-//       pass: false,
-//       message: () =>
-//         `expected ${this.utils.printReceived(
-//           actual
-//         )} to pass evaluator. Failed with ${JSON.stringify(
-//           evalResult,
-//           null,
-//           2
-//         )}`,
-//     };
-//   }
-//   return {
-//     pass: true,
-//     message: () =>
-//       `evaluator passed with score ${JSON.stringify(evalResult, null, 2)}`,
-//   };
-// }

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.2.14-rc.2",
+  "version": "0.2.14-rc.3",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [