langsmith 0.3.0 → 0.3.2

package/dist/index.cjs

@@ -8,4 +8,4 @@ Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () {
 var fetch_js_1 = require("./singletons/fetch.cjs");
 Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true, get: function () { return fetch_js_1.overrideFetchImplementation; } });
 // Update using yarn bump-version
-exports.__version__ = "0.3.0";
+exports.__version__ = "0.3.2";

package/dist/index.d.ts

@@ -2,4 +2,4 @@ export { Client, type ClientConfig, type LangSmithTracingClientInterface, } from
 export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
-export declare const __version__ = "0.3.0";
+export declare const __version__ = "0.3.2";

package/dist/index.js

@@ -2,4 +2,4 @@ export { Client, } from "./client.js";
 export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 // Update using yarn bump-version
-export const __version__ = "0.3.0";
+export const __version__ = "0.3.2";

package/dist/jest/reporter.cjs

@@ -5,8 +5,19 @@ const reporters_1 = require("@jest/reporters");
 const reporter_js_1 = require("../utils/jestlike/reporter.cjs");
 class LangSmithEvalReporter extends reporters_1.DefaultReporter {
     async onTestResult(test, testResult, aggregatedResults) {
+        const groupedTestResults = testResult.testResults.reduce((groups, testResult) => {
+            const ancestorTitle = testResult.ancestorTitles.join(" > ");
+            if (groups[ancestorTitle] === undefined) {
+                groups[ancestorTitle] = [];
+            }
+            groups[ancestorTitle].push(testResult);
+            return groups;
+        }, {});
         try {
-            await (0, reporter_js_1.printReporterTable)(testResult.testResults, testResult.failureMessage);
+            for (const testGroupName of Object.keys(groupedTestResults)) {
+                const resultGroup = groupedTestResults[testGroupName];
+                await (0, reporter_js_1.printReporterTable)(resultGroup, testResult.failureMessage);
+            }
         }
         catch (e) {
             console.log("Failed to display LangSmith eval results:", e.message);

package/dist/jest/reporter.js

@@ -3,8 +3,19 @@ import { DefaultReporter } from "@jest/reporters";
 import { printReporterTable } from "../utils/jestlike/reporter.js";
 class LangSmithEvalReporter extends DefaultReporter {
     async onTestResult(test, testResult, aggregatedResults) {
+        const groupedTestResults = testResult.testResults.reduce((groups, testResult) => {
+            const ancestorTitle = testResult.ancestorTitles.join(" > ");
+            if (groups[ancestorTitle] === undefined) {
+                groups[ancestorTitle] = [];
+            }
+            groups[ancestorTitle].push(testResult);
+            return groups;
+        }, {});
         try {
-            await printReporterTable(testResult.testResults, testResult.failureMessage);
+            for (const testGroupName of Object.keys(groupedTestResults)) {
+                const resultGroup = groupedTestResults[testGroupName];
+                await printReporterTable(resultGroup, testResult.failureMessage);
+            }
         }
         catch (e) {
             console.log("Failed to display LangSmith eval results:", e.message);

package/dist/utils/jestlike/globals.d.ts

@@ -2,8 +2,8 @@
 import { AsyncLocalStorage } from "node:async_hooks";
 import { Dataset, TracerSession, Example } from "../../schemas.js";
 import { Client, CreateProjectParams } from "../../client.js";
-import { EvaluationResult } from "../../evaluation/evaluator.js";
 import { RunTree } from "../../run_trees.js";
+import { SimpleEvaluationResult } from "./types.js";
 export declare const DEFAULT_TEST_CLIENT: Client;
 export type TestWrapperAsyncLocalStorageData = {
     enableTestTracking?: boolean;
@@ -12,7 +12,7 @@ export type TestWrapperAsyncLocalStorageData = {
     projectConfig?: Partial<CreateProjectParams>;
     project?: TracerSession;
     setLoggedOutput?: (value: Record<string, unknown>) => void;
-    onFeedbackLogged?: (feedback: EvaluationResult) => void;
+    onFeedbackLogged?: (feedback: SimpleEvaluationResult) => void;
     currentExample?: Partial<Example> & {
         syncPromise?: Promise<Example>;
     };
@@ -26,7 +26,7 @@ export declare const evaluatorLogFeedbackPromises: Set<unknown>;
 export declare const syncExamplePromises: Map<any, any>;
 export declare function _logTestFeedback(params: {
     exampleId?: string;
-    feedback: EvaluationResult;
+    feedback: SimpleEvaluationResult;
     context: TestWrapperAsyncLocalStorageData;
     runTree?: RunTree;
     client: Client;

package/dist/utils/jestlike/index.cjs

@@ -139,9 +139,9 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
             " Please try again.");
     }
     const datasetSetupInfo = new Map();
-    function getExampleId(datasetName, inputs, outputs) {
+    function getExampleId(datasetId, inputs, outputs) {
         const identifier = JSON.stringify({
-            datasetName,
+            datasetId,
             inputsHash: objectHash(inputs),
             outputsHash: objectHash(outputs ?? {}),
         });
@@ -432,7 +432,7 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                                     .map((field) => `"${field}"`)
                                     .join(", ")} while syncing to LangSmith. Please contact us for help.`);
                             }
-                            exampleId = getExampleId(dataset.name, inputs, referenceOutputs);
+                            exampleId = getExampleId(dataset.id, inputs, referenceOutputs);
                             // TODO: Create or update the example in the background
                             // Currently run end time has to be after example modified time
                             // for examples to render properly, so we must modify the example

package/dist/utils/jestlike/index.d.ts

@@ -1,11 +1,11 @@
 /// <reference types="jest" />
 import { KVMap } from "../../schemas.js";
 import { toBeRelativeCloseTo, toBeAbsoluteCloseTo, toBeSemanticCloseTo } from "./matchers.js";
-import { EvaluationResult } from "../../evaluation/evaluator.js";
+import { SimpleEvaluationResult } from "./types.js";
 import type { LangSmithJestlikeWrapperConfig, LangSmithJestlikeWrapperParams, LangSmithJestDescribeWrapper } from "./types.js";
 export declare const STRIP_ANSI_REGEX: RegExp;
 export declare const TEST_ID_DELIMITER = ", test_id=";
-export declare function logFeedback(feedback: EvaluationResult, config?: {
+export declare function logFeedback(feedback: SimpleEvaluationResult, config?: {
     sourceRunId?: string;
 }): void;
 export declare function logOutputs(output: Record<string, unknown>): void;

package/dist/utils/jestlike/index.js

@@ -108,9 +108,9 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
             " Please try again.");
     }
     const datasetSetupInfo = new Map();
-    function getExampleId(datasetName, inputs, outputs) {
+    function getExampleId(datasetId, inputs, outputs) {
         const identifier = JSON.stringify({
-            datasetName,
+            datasetId,
             inputsHash: objectHash(inputs),
             outputsHash: objectHash(outputs ?? {}),
         });
@@ -401,7 +401,7 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                                     .map((field) => `"${field}"`)
                                     .join(", ")} while syncing to LangSmith. Please contact us for help.`);
                             }
-                            exampleId = getExampleId(dataset.name, inputs, referenceOutputs);
+                            exampleId = getExampleId(dataset.id, inputs, referenceOutputs);
                             // TODO: Create or update the example in the background
                             // Currently run end time has to be after example modified time
                             // for examples to render properly, so we must modify the example

package/dist/utils/jestlike/reporter.cjs

@@ -34,6 +34,7 @@ const path = __importStar(require("node:path"));
 const fs = __importStar(require("node:fs/promises"));
 const index_js_1 = require("./index.cjs");
 const FEEDBACK_COLLAPSE_THRESHOLD = 48;
+const MAX_TEST_PARAMS_LENGTH = 18;
 const RESERVED_KEYS = [
     "Name",
     "Result",
@@ -86,7 +87,9 @@ function formatValue(value) {
             .map(([k, v]) => {
             const rawValue = typeof v === "string" ? v : JSON.stringify(v);
             const rawEntry = `${k}: ${rawValue}`;
-            const entry = rawEntry.length > 24 ? rawEntry.slice(0, 21) + "..." : rawEntry;
+            const entry = rawEntry.length > MAX_TEST_PARAMS_LENGTH
+                ? rawEntry.slice(0, MAX_TEST_PARAMS_LENGTH - 3) + "..."
+                : rawEntry;
             return entry;
         })
             .join("\n");
@@ -230,9 +233,13 @@ async function printReporterTable(results, failureMessage) {
     }
     const defaultColumns = [
         { name: "Test", alignment: "left", maxLen: 36 },
-        { name: "Inputs", alignment: "left", minLen: 24 },
-        { name: "Reference Outputs", alignment: "left", minLen: 24 },
-        { name: "Outputs", alignment: "left", minLen: 24 },
+        { name: "Inputs", alignment: "left", minLen: MAX_TEST_PARAMS_LENGTH },
+        {
+            name: "Reference Outputs",
+            alignment: "left",
+            minLen: MAX_TEST_PARAMS_LENGTH,
+        },
+        { name: "Outputs", alignment: "left", minLen: MAX_TEST_PARAMS_LENGTH },
         { name: "Status", alignment: "left" },
     ];
     if (collapseFeedbackColumn) {
@@ -245,7 +252,7 @@ async function printReporterTable(results, failureMessage) {
         defaultColumns.push({
             name: "Feedback",
             alignment: "left",
-            minLen: feedbackColumnLength + 10,
+            minLen: feedbackColumnLength + 8,
         });
     }
     console.log();

package/dist/utils/jestlike/reporter.js

@@ -5,6 +5,7 @@ import * as path from "node:path";
 import * as fs from "node:fs/promises";
 import { STRIP_ANSI_REGEX, TEST_ID_DELIMITER } from "./index.js";
 const FEEDBACK_COLLAPSE_THRESHOLD = 48;
+const MAX_TEST_PARAMS_LENGTH = 18;
 const RESERVED_KEYS = [
     "Name",
     "Result",
@@ -57,7 +58,9 @@ function formatValue(value) {
             .map(([k, v]) => {
             const rawValue = typeof v === "string" ? v : JSON.stringify(v);
             const rawEntry = `${k}: ${rawValue}`;
-            const entry = rawEntry.length > 24 ? rawEntry.slice(0, 21) + "..." : rawEntry;
+            const entry = rawEntry.length > MAX_TEST_PARAMS_LENGTH
+                ? rawEntry.slice(0, MAX_TEST_PARAMS_LENGTH - 3) + "..."
+                : rawEntry;
             return entry;
         })
             .join("\n");
@@ -201,9 +204,13 @@ export async function printReporterTable(results, failureMessage) {
     }
     const defaultColumns = [
         { name: "Test", alignment: "left", maxLen: 36 },
-        { name: "Inputs", alignment: "left", minLen: 24 },
-        { name: "Reference Outputs", alignment: "left", minLen: 24 },
-        { name: "Outputs", alignment: "left", minLen: 24 },
+        { name: "Inputs", alignment: "left", minLen: MAX_TEST_PARAMS_LENGTH },
+        {
+            name: "Reference Outputs",
+            alignment: "left",
+            minLen: MAX_TEST_PARAMS_LENGTH,
+        },
+        { name: "Outputs", alignment: "left", minLen: MAX_TEST_PARAMS_LENGTH },
         { name: "Status", alignment: "left" },
     ];
     if (collapseFeedbackColumn) {
@@ -216,7 +223,7 @@ export async function printReporterTable(results, failureMessage) {
         defaultColumns.push({
             name: "Feedback",
             alignment: "left",
-            minLen: feedbackColumnLength + 10,
+            minLen: feedbackColumnLength + 8,
         });
     }
     console.log();

package/dist/utils/jestlike/types.d.ts

@@ -1,3 +1,4 @@
+import { EvaluationResult } from "../../evaluation/evaluator.js";
 import type { RunTreeConfig } from "../../run_trees.js";
 import type { SimpleEvaluator } from "./vendor/evaluatedBy.js";
 export { type SimpleEvaluator };
@@ -11,3 +12,8 @@ export type LangSmithJestlikeWrapperParams<I, O> = {
     config?: LangSmithJestlikeWrapperConfig;
 };
 export type LangSmithJestDescribeWrapper = (name: string, fn: () => void | Promise<void>, config?: Partial<RunTreeConfig>) => void;
+export type SimpleEvaluationResult = {
+    key: EvaluationResult["key"];
+    score: NonNullable<EvaluationResult["score"]>;
+    comment?: EvaluationResult["comment"];
+};

package/dist/utils/jestlike/vendor/evaluatedBy.d.ts

@@ -1,12 +1,12 @@
-import { EvaluationResult } from "../../../evaluation/evaluator.js";
+import { SimpleEvaluationResult } from "../types.js";
 import { RunTreeConfig } from "../../../run_trees.js";
 export type SimpleEvaluatorParams = {
     inputs: Record<string, any>;
     referenceOutputs: Record<string, any>;
     outputs: Record<string, any>;
 };
-export type SimpleEvaluator = (params: SimpleEvaluatorParams) => EvaluationResult | Promise<EvaluationResult>;
-export declare function wrapEvaluator<I, O>(evaluator: (input: I) => O | Promise<O>): (input: I, config?: Partial<RunTreeConfig> & {
+export type SimpleEvaluator = (params: SimpleEvaluatorParams) => SimpleEvaluationResult | Promise<SimpleEvaluationResult>;
+export declare function wrapEvaluator<I>(evaluator: (input: I) => SimpleEvaluationResult | Promise<SimpleEvaluationResult>): (input: I, config?: Partial<RunTreeConfig> & {
     runId?: string;
-}) => Promise<O>;
-export declare function evaluatedBy(outputs: any, evaluator: SimpleEvaluator): Promise<import("../../../schemas.js").ScoreType | undefined>;
+}) => Promise<SimpleEvaluationResult>;
+export declare function evaluatedBy(outputs: any, evaluator: SimpleEvaluator): Promise<NonNullable<import("../../../schemas.js").ScoreType | undefined>>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [