langsmith 0.3.3 → 0.3.4

package/README.md

@@ -50,7 +50,7 @@ yarn add langchain
 Tracing can be activated by setting the following environment variables or by manually specifying the LangChainTracer.
 
 ```typescript
-process.env.LANGSMITH_TRACING_V2 = "true";
+process.env.LANGSMITH_TRACING = "true";
 process.env.LANGSMITH_ENDPOINT = "https://api.smith.langchain.com";
 // process.env.LANGSMITH_ENDPOINT = "https://eu.api.smith.langchain.com"; // If signed up in the EU region
 process.env.LANGSMITH_API_KEY = "<YOUR-LANGSMITH-API-KEY>";
@@ -82,7 +82,7 @@ or by directly specifying the connection information in the RunTree.
 1. **Copy the environment variables from the Settings Page and add them to your application.**
 
 ```shell
-export LANGSMITH_TRACING_V2="true";
+export LANGSMITH_TRACING="true";
 export LANGSMITH_API_KEY=<YOUR-LANGSMITH-API-KEY>
 # export LANGSMITH_PROJECT="My Project Name" #  Optional: "default" is used if not set
 # export LANGSMITH_ENDPOINT=https://api.smith.langchain.com # or your own server
@@ -102,7 +102,7 @@ is using the `wrapOpenAI` wrapper function available in LangSmith 0.1.3 and up.
 In order to use, you first need to set your LangSmith API key:
 
 ```shell
-export LANGSMITH_TRACING_V2="true";
+export LANGSMITH_TRACING="true";
 export LANGSMITH_API_KEY=<your-api-key>
 ```
 

package/dist/client.cjs

@@ -371,6 +371,10 @@ class Client {
             this.webUrl = "http://localhost:3000";
             return this.webUrl;
         }
+        else if (this.apiUrl.endsWith("/api/v1")) {
+            this.webUrl = this.apiUrl.replace("/api/v1", "");
+            return this.webUrl;
+        }
         else if (this.apiUrl.includes("/api") &&
             !this.apiUrl.split(".", 1)[0].endsWith("api")) {
             this.webUrl = this.apiUrl.replace("/api", "");
@@ -384,6 +388,10 @@ class Client {
             this.webUrl = "https://eu.smith.langchain.com";
             return this.webUrl;
         }
+        else if (this.apiUrl.split(".", 1)[0].includes("beta")) {
+            this.webUrl = "https://beta.smith.langchain.com";
+            return this.webUrl;
+        }
         else {
             this.webUrl = "https://smith.langchain.com";
             return this.webUrl;

package/dist/client.js

@@ -343,6 +343,10 @@ export class Client {
             this.webUrl = "http://localhost:3000";
             return this.webUrl;
         }
+        else if (this.apiUrl.endsWith("/api/v1")) {
+            this.webUrl = this.apiUrl.replace("/api/v1", "");
+            return this.webUrl;
+        }
         else if (this.apiUrl.includes("/api") &&
             !this.apiUrl.split(".", 1)[0].endsWith("api")) {
             this.webUrl = this.apiUrl.replace("/api", "");
@@ -356,6 +360,10 @@ export class Client {
             this.webUrl = "https://eu.smith.langchain.com";
             return this.webUrl;
         }
+        else if (this.apiUrl.split(".", 1)[0].includes("beta")) {
+            this.webUrl = "https://beta.smith.langchain.com";
+            return this.webUrl;
+        }
         else {
             this.webUrl = "https://smith.langchain.com";
             return this.webUrl;

package/dist/index.cjs

@@ -8,4 +8,4 @@ Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () {
 var fetch_js_1 = require("./singletons/fetch.cjs");
 Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true, get: function () { return fetch_js_1.overrideFetchImplementation; } });
 // Update using yarn bump-version
-exports.__version__ = "0.3.3";
+exports.__version__ = "0.3.4";

package/dist/index.d.ts

@@ -2,4 +2,4 @@ export { Client, type ClientConfig, type LangSmithTracingClientInterface, } from
 export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
-export declare const __version__ = "0.3.3";
+export declare const __version__ = "0.3.4";

package/dist/index.js

@@ -2,4 +2,4 @@ export { Client, } from "./client.js";
 export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 // Update using yarn bump-version
-export const __version__ = "0.3.3";
+export const __version__ = "0.3.4";

package/dist/jest/reporter.cjs

@@ -5,6 +5,9 @@ const reporters_1 = require("@jest/reporters");
 const reporter_js_1 = require("../utils/jestlike/reporter.cjs");
 class LangSmithEvalReporter extends reporters_1.DefaultReporter {
     async onTestResult(test, testResult, aggregatedResults) {
+        if (testResult.failureMessage) {
+            console.log(testResult.failureMessage);
+        }
         const groupedTestResults = testResult.testResults.reduce((groups, testResult) => {
             const ancestorTitle = testResult.ancestorTitles.join(" > ");
             if (groups[ancestorTitle] === undefined) {
@@ -16,7 +19,13 @@ class LangSmithEvalReporter extends reporters_1.DefaultReporter {
         try {
             for (const testGroupName of Object.keys(groupedTestResults)) {
                 const resultGroup = groupedTestResults[testGroupName];
-                await (0, reporter_js_1.printReporterTable)(resultGroup, testResult.failureMessage);
+                const unskippedTests = resultGroup.filter((result) => result.status !== "pending");
+                const overallResult = unskippedTests.length === 0
+                    ? "skip"
+                    : unskippedTests.every((result) => result.status === "passed")
+                        ? "pass"
+                        : "fail";
+                await (0, reporter_js_1.printReporterTable)(testGroupName, resultGroup, overallResult);
             }
         }
         catch (e) {

package/dist/jest/reporter.js

@@ -3,6 +3,9 @@ import { DefaultReporter } from "@jest/reporters";
 import { printReporterTable } from "../utils/jestlike/reporter.js";
 class LangSmithEvalReporter extends DefaultReporter {
     async onTestResult(test, testResult, aggregatedResults) {
+        if (testResult.failureMessage) {
+            console.log(testResult.failureMessage);
+        }
         const groupedTestResults = testResult.testResults.reduce((groups, testResult) => {
             const ancestorTitle = testResult.ancestorTitles.join(" > ");
             if (groups[ancestorTitle] === undefined) {
@@ -14,7 +17,13 @@ class LangSmithEvalReporter extends DefaultReporter {
         try {
             for (const testGroupName of Object.keys(groupedTestResults)) {
                 const resultGroup = groupedTestResults[testGroupName];
-                await printReporterTable(resultGroup, testResult.failureMessage);
+                const unskippedTests = resultGroup.filter((result) => result.status !== "pending");
+                const overallResult = unskippedTests.length === 0
+                    ? "skip"
+                    : unskippedTests.every((result) => result.status === "passed")
+                        ? "pass"
+                        : "fail";
+                await printReporterTable(testGroupName, resultGroup, overallResult);
             }
         }
         catch (e) {

package/dist/utils/jestlike/reporter.cjs

@@ -53,13 +53,13 @@ function formatTestName(name, duration) {
 }
 function getFormattedStatus(status) {
     const s = status.toLowerCase();
-    if (s === "pending") {
+    if (s === "pending" || s === "skipped") {
         return chalk_1.default.yellow("○ Skipped");
     }
-    else if (s === "passed") {
+    else if (s.includes("pass")) {
         return chalk_1.default.green("✓ Passed");
     }
-    else if (s === "failed") {
+    else if (s.includes("fail")) {
         return chalk_1.default.red("✕ Failed");
     }
     else {
@@ -68,13 +68,13 @@ function getFormattedStatus(status) {
 }
 function getColorParam(status) {
     const s = status.toLowerCase();
-    if (s === "pending") {
+    if (s === "pending" || s === "skipped") {
         return { color: "yellow" };
     }
-    else if (s === "passed") {
+    else if (s.includes("pass")) {
         return { color: "grey" };
     }
-    else if (s === "failed") {
+    else if (s.includes("fail")) {
         return { color: "red" };
     }
     else {
@@ -99,7 +99,7 @@ function formatValue(value) {
     }
     return String(value);
 }
-async function printReporterTable(results, failureMessage) {
+async function printReporterTable(testSuiteName, results, testStatus, failureMessage) {
     const rows = [];
     const feedbackKeys = new Set();
     let experimentUrl;
@@ -122,7 +122,7 @@ async function printReporterTable(results, failureMessage) {
                 getColorParam(status),
             ]);
         }
-        else if (status === "pending") {
+        else if (status === "pending" || status === "skipped") {
             // Skipped
             rows.push([
                 {
@@ -265,6 +265,14 @@ async function printReporterTable(results, failureMessage) {
     for (const row of rows) {
         table.addRow(row[0], row[1]);
     }
+    const testStatusColor = testStatus.includes("pass")
+        ? chalk_1.default.green
+        : testStatus.includes("fail")
+            ? chalk_1.default.red
+            : chalk_1.default.yellow;
+    if (testSuiteName) {
+        console.log(testStatusColor(`› ${testSuiteName}`));
+    }
     if (failureMessage) {
         console.log(failureMessage);
     }

package/dist/utils/jestlike/reporter.d.ts

@@ -1,5 +1,5 @@
-export declare function printReporterTable(results: {
+export declare function printReporterTable(testSuiteName: string, results: {
     title: string;
     duration: number;
-    status: string;
-}[], failureMessage?: string): Promise<void>;
+    status: "pass" | "passed" | "fail" | "failed" | "pending" | "skipped";
+}[], testStatus: "pass" | "skip" | "fail", failureMessage?: string): Promise<void>;

package/dist/utils/jestlike/reporter.js

@@ -24,13 +24,13 @@ function formatTestName(name, duration) {
 }
 function getFormattedStatus(status) {
     const s = status.toLowerCase();
-    if (s === "pending") {
+    if (s === "pending" || s === "skipped") {
         return chalk.yellow("○ Skipped");
     }
-    else if (s === "passed") {
+    else if (s.includes("pass")) {
         return chalk.green("✓ Passed");
     }
-    else if (s === "failed") {
+    else if (s.includes("fail")) {
         return chalk.red("✕ Failed");
     }
     else {
@@ -39,13 +39,13 @@ function getFormattedStatus(status) {
 }
 function getColorParam(status) {
     const s = status.toLowerCase();
-    if (s === "pending") {
+    if (s === "pending" || s === "skipped") {
         return { color: "yellow" };
     }
-    else if (s === "passed") {
+    else if (s.includes("pass")) {
         return { color: "grey" };
     }
-    else if (s === "failed") {
+    else if (s.includes("fail")) {
         return { color: "red" };
     }
     else {
@@ -70,7 +70,7 @@ function formatValue(value) {
     }
     return String(value);
 }
-export async function printReporterTable(results, failureMessage) {
+export async function printReporterTable(testSuiteName, results, testStatus, failureMessage) {
     const rows = [];
     const feedbackKeys = new Set();
     let experimentUrl;
@@ -93,7 +93,7 @@ export async function printReporterTable(results, failureMessage) {
                 getColorParam(status),
             ]);
         }
-        else if (status === "pending") {
+        else if (status === "pending" || status === "skipped") {
             // Skipped
             rows.push([
                 {
@@ -236,6 +236,14 @@ export async function printReporterTable(results, failureMessage) {
     for (const row of rows) {
         table.addRow(row[0], row[1]);
     }
+    const testStatusColor = testStatus.includes("pass")
+        ? chalk.green
+        : testStatus.includes("fail")
+            ? chalk.red
+            : chalk.yellow;
+    if (testSuiteName) {
+        console.log(testStatusColor(`› ${testSuiteName}`));
+    }
     if (failureMessage) {
         console.log(failureMessage);
     }

package/dist/vitest/reporter.cjs

@@ -9,15 +9,20 @@ class LangSmithEvalReporter extends reporters_1.DefaultReporter {
     async onFinished(files, errors) {
         super.onFinished(files, errors);
         for (const file of files) {
-            const testModule = this.ctx.state.getReportedEntity(file);
-            const tests = [...testModule.children.allTests()].map((test) => {
-                return {
-                    title: test.name,
-                    status: test.result()?.state ?? "skipped",
-                    duration: Math.round(test.diagnostic()?.duration ?? 0),
-                };
-            });
-            await (0, reporter_js_1.printReporterTable)(tests);
+            for (const task of file.tasks) {
+                const testModule = this.ctx.state.getReportedEntity(task);
+                const tests = [...testModule.children.allTests()].map((test) => {
+                    return {
+                        title: test.name,
+                        status: test.result()?.state ?? "skipped",
+                        duration: Math.round(test.diagnostic()?.duration ?? 0),
+                    };
+                });
+                const result = ["pass", "fail", "skip"].includes(task.result?.state ?? "")
+                    ? task.result?.state
+                    : "skip";
+                await (0, reporter_js_1.printReporterTable)(task.name, tests, result);
+            }
         }
     }
 }

package/dist/vitest/reporter.js

@@ -7,15 +7,20 @@ class LangSmithEvalReporter extends DefaultReporter {
     async onFinished(files, errors) {
         super.onFinished(files, errors);
         for (const file of files) {
-            const testModule = this.ctx.state.getReportedEntity(file);
-            const tests = [...testModule.children.allTests()].map((test) => {
-                return {
-                    title: test.name,
-                    status: test.result()?.state ?? "skipped",
-                    duration: Math.round(test.diagnostic()?.duration ?? 0),
-                };
-            });
-            await printReporterTable(tests);
+            for (const task of file.tasks) {
+                const testModule = this.ctx.state.getReportedEntity(task);
+                const tests = [...testModule.children.allTests()].map((test) => {
+                    return {
+                        title: test.name,
+                        status: test.result()?.state ?? "skipped",
+                        duration: Math.round(test.diagnostic()?.duration ?? 0),
+                    };
+                });
+                const result = ["pass", "fail", "skip"].includes(task.result?.state ?? "")
+                    ? task.result?.state
+                    : "skip";
+                await printReporterTable(task.name, tests, result);
+            }
         }
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.3.3",
+  "version": "0.3.4",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [