langsmith 0.3.4-rc.0 → 0.3.5

package/dist/client.cjs

@@ -127,7 +127,7 @@ class AutoBatchQueue {
             // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/Promise
             itemPromiseResolve = resolve;
         });
-        const size = (0, index_js_2.stringify)(item.item).length;
+        const size = (0, index_js_2.serialize)(item.item).length;
         this.items.push({
             action: item.action,
             payload: item.item,
@@ -371,6 +371,10 @@ class Client {
             this.webUrl = "http://localhost:3000";
             return this.webUrl;
         }
+        else if (this.apiUrl.endsWith("/api/v1")) {
+            this.webUrl = this.apiUrl.replace("/api/v1", "");
+            return this.webUrl;
+        }
         else if (this.apiUrl.includes("/api") &&
             !this.apiUrl.split(".", 1)[0].endsWith("api")) {
             this.webUrl = this.apiUrl.replace("/api", "");
@@ -384,6 +388,10 @@ class Client {
             this.webUrl = "https://eu.smith.langchain.com";
             return this.webUrl;
         }
+        else if (this.apiUrl.split(".", 1)[0].includes("beta")) {
+            this.webUrl = "https://beta.smith.langchain.com";
+            return this.webUrl;
+        }
         else {
             this.webUrl = "https://smith.langchain.com";
             return this.webUrl;
@@ -675,7 +683,7 @@ class Client {
         const response = await this.caller.call((0, fetch_js_1._getFetchImplementation)(), `${this.apiUrl}/runs`, {
             method: "POST",
             headers,
-            body: (0, index_js_2.stringify)(mergedRunCreateParam),
+            body: (0, index_js_2.serialize)(mergedRunCreateParam),
             signal: AbortSignal.timeout(this.timeout_ms),
             ...this.fetchOptions,
         });
@@ -735,7 +743,7 @@ class Client {
             }
         }
         if (batchChunks.post.length > 0 || batchChunks.patch.length > 0) {
-            await this._postBatchIngestRuns((0, index_js_2.stringify)(batchChunks));
+            await this._postBatchIngestRuns((0, index_js_2.serialize)(batchChunks));
         }
     }
     async _postBatchIngestRuns(body) {
@@ -830,7 +838,7 @@ class Client {
                 const { inputs, outputs, events, attachments, ...payload } = originalPayload;
                 const fields = { inputs, outputs, events };
                 // encode the main run payload
-                const stringifiedPayload = (0, index_js_2.stringify)(payload);
+                const stringifiedPayload = (0, index_js_2.serialize)(payload);
                 accumulatedParts.push({
                     name: `${method}.${payload.id}`,
                     payload: new Blob([stringifiedPayload], {
@@ -842,7 +850,7 @@ class Client {
                     if (value === undefined) {
                         continue;
                     }
-                    const stringifiedValue = (0, index_js_2.stringify)(value);
+                    const stringifiedValue = (0, index_js_2.serialize)(value);
                     accumulatedParts.push({
                         name: `${method}.${payload.id}.${key}`,
                         payload: new Blob([stringifiedValue], {
@@ -907,7 +915,7 @@ class Client {
             const body = new Blob(chunks);
             // Convert Blob to ArrayBuffer for compatibility
             const arrayBuffer = await body.arrayBuffer();
-            const res = await this.batchIngestCaller.call((0, fetch_js_1._getFetchImplementation)(), `${this.apiUrl}/v1/runs/multipart`, {
+            const res = await this.batchIngestCaller.call((0, fetch_js_1._getFetchImplementation)(), `${this.apiUrl}/runs/multipart`, {
                 method: "POST",
                 headers: {
                     ...this.headers,
@@ -958,7 +966,7 @@ class Client {
         const response = await this.caller.call((0, fetch_js_1._getFetchImplementation)(), `${this.apiUrl}/runs/${runId}`, {
             method: "PATCH",
             headers,
-            body: (0, index_js_2.stringify)(run),
+            body: (0, index_js_2.serialize)(run),
             signal: AbortSignal.timeout(this.timeout_ms),
             ...this.fetchOptions,
         });
@@ -2845,14 +2853,14 @@ class Client {
                 ...(example.split && { split: example.split }),
             };
             // Add main example data
-            const stringifiedExample = (0, index_js_2.stringify)(exampleBody);
+            const stringifiedExample = (0, index_js_2.serialize)(exampleBody);
             const exampleBlob = new Blob([stringifiedExample], {
                 type: "application/json",
             });
             formData.append(exampleId, exampleBlob);
             // Add inputs
             if (example.inputs) {
-                const stringifiedInputs = (0, index_js_2.stringify)(example.inputs);
+                const stringifiedInputs = (0, index_js_2.serialize)(example.inputs);
                 const inputsBlob = new Blob([stringifiedInputs], {
                     type: "application/json",
                 });
@@ -2860,7 +2868,7 @@ class Client {
             }
             // Add outputs if present
             if (example.outputs) {
-                const stringifiedOutputs = (0, index_js_2.stringify)(example.outputs);
+                const stringifiedOutputs = (0, index_js_2.serialize)(example.outputs);
                 const outputsBlob = new Blob([stringifiedOutputs], {
                     type: "application/json",
                 });
@@ -2885,7 +2893,7 @@ class Client {
                 }
             }
             if (example.attachments_operations) {
-                const stringifiedAttachmentsOperations = (0, index_js_2.stringify)(example.attachments_operations);
+                const stringifiedAttachmentsOperations = (0, index_js_2.serialize)(example.attachments_operations);
                 const attachmentsOperationsBlob = new Blob([stringifiedAttachmentsOperations], {
                     type: "application/json",
                 });
@@ -2919,20 +2927,20 @@ class Client {
                 ...(example.split && { split: example.split }),
             };
             // Add main example data
-            const stringifiedExample = (0, index_js_2.stringify)(exampleBody);
+            const stringifiedExample = (0, index_js_2.serialize)(exampleBody);
             const exampleBlob = new Blob([stringifiedExample], {
                 type: "application/json",
             });
             formData.append(exampleId, exampleBlob);
             // Add inputs
-            const stringifiedInputs = (0, index_js_2.stringify)(example.inputs);
+            const stringifiedInputs = (0, index_js_2.serialize)(example.inputs);
             const inputsBlob = new Blob([stringifiedInputs], {
                 type: "application/json",
             });
             formData.append(`${exampleId}.inputs`, inputsBlob);
             // Add outputs if present
             if (example.outputs) {
-                const stringifiedOutputs = (0, index_js_2.stringify)(example.outputs);
+                const stringifiedOutputs = (0, index_js_2.serialize)(example.outputs);
                 const outputsBlob = new Blob([stringifiedOutputs], {
                     type: "application/json",
                 });

package/dist/client.js

@@ -8,7 +8,7 @@ import { warnOnce } from "./utils/warn.js";
 import { parsePromptIdentifier } from "./utils/prompts.js";
 import { raiseForStatus } from "./utils/error.js";
 import { _getFetchImplementation } from "./singletons/fetch.js";
-import { stringify as stringifyForTracing } from "./utils/fast-safe-stringify/index.js";
+import { serialize as serializePayloadForTracing } from "./utils/fast-safe-stringify/index.js";
 export function mergeRuntimeEnvIntoRunCreate(run) {
     const runtimeEnv = getRuntimeEnvironment();
     const envVars = getLangChainEnvVarsMetadata();
@@ -100,7 +100,7 @@ export class AutoBatchQueue {
             // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/Promise
             itemPromiseResolve = resolve;
         });
-        const size = stringifyForTracing(item.item).length;
+        const size = serializePayloadForTracing(item.item).length;
         this.items.push({
             action: item.action,
             payload: item.item,
@@ -343,6 +343,10 @@ export class Client {
             this.webUrl = "http://localhost:3000";
             return this.webUrl;
         }
+        else if (this.apiUrl.endsWith("/api/v1")) {
+            this.webUrl = this.apiUrl.replace("/api/v1", "");
+            return this.webUrl;
+        }
         else if (this.apiUrl.includes("/api") &&
             !this.apiUrl.split(".", 1)[0].endsWith("api")) {
             this.webUrl = this.apiUrl.replace("/api", "");
@@ -356,6 +360,10 @@ export class Client {
             this.webUrl = "https://eu.smith.langchain.com";
             return this.webUrl;
         }
+        else if (this.apiUrl.split(".", 1)[0].includes("beta")) {
+            this.webUrl = "https://beta.smith.langchain.com";
+            return this.webUrl;
+        }
         else {
             this.webUrl = "https://smith.langchain.com";
             return this.webUrl;
@@ -647,7 +655,7 @@ export class Client {
         const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/runs`, {
             method: "POST",
             headers,
-            body: stringifyForTracing(mergedRunCreateParam),
+            body: serializePayloadForTracing(mergedRunCreateParam),
             signal: AbortSignal.timeout(this.timeout_ms),
             ...this.fetchOptions,
         });
@@ -707,7 +715,7 @@ export class Client {
             }
         }
         if (batchChunks.post.length > 0 || batchChunks.patch.length > 0) {
-            await this._postBatchIngestRuns(stringifyForTracing(batchChunks));
+            await this._postBatchIngestRuns(serializePayloadForTracing(batchChunks));
         }
     }
     async _postBatchIngestRuns(body) {
@@ -802,7 +810,7 @@ export class Client {
                 const { inputs, outputs, events, attachments, ...payload } = originalPayload;
                 const fields = { inputs, outputs, events };
                 // encode the main run payload
-                const stringifiedPayload = stringifyForTracing(payload);
+                const stringifiedPayload = serializePayloadForTracing(payload);
                 accumulatedParts.push({
                     name: `${method}.${payload.id}`,
                     payload: new Blob([stringifiedPayload], {
@@ -814,7 +822,7 @@ export class Client {
                     if (value === undefined) {
                         continue;
                     }
-                    const stringifiedValue = stringifyForTracing(value);
+                    const stringifiedValue = serializePayloadForTracing(value);
                     accumulatedParts.push({
                         name: `${method}.${payload.id}.${key}`,
                         payload: new Blob([stringifiedValue], {
@@ -879,7 +887,7 @@ export class Client {
             const body = new Blob(chunks);
             // Convert Blob to ArrayBuffer for compatibility
             const arrayBuffer = await body.arrayBuffer();
-            const res = await this.batchIngestCaller.call(_getFetchImplementation(), `${this.apiUrl}/v1/runs/multipart`, {
+            const res = await this.batchIngestCaller.call(_getFetchImplementation(), `${this.apiUrl}/runs/multipart`, {
                 method: "POST",
                 headers: {
                     ...this.headers,
@@ -930,7 +938,7 @@ export class Client {
         const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/runs/${runId}`, {
             method: "PATCH",
             headers,
-            body: stringifyForTracing(run),
+            body: serializePayloadForTracing(run),
             signal: AbortSignal.timeout(this.timeout_ms),
             ...this.fetchOptions,
         });
@@ -2817,14 +2825,14 @@ export class Client {
                 ...(example.split && { split: example.split }),
             };
             // Add main example data
-            const stringifiedExample = stringifyForTracing(exampleBody);
+            const stringifiedExample = serializePayloadForTracing(exampleBody);
             const exampleBlob = new Blob([stringifiedExample], {
                 type: "application/json",
             });
             formData.append(exampleId, exampleBlob);
             // Add inputs
             if (example.inputs) {
-                const stringifiedInputs = stringifyForTracing(example.inputs);
+                const stringifiedInputs = serializePayloadForTracing(example.inputs);
                 const inputsBlob = new Blob([stringifiedInputs], {
                     type: "application/json",
                 });
@@ -2832,7 +2840,7 @@ export class Client {
             }
             // Add outputs if present
             if (example.outputs) {
-                const stringifiedOutputs = stringifyForTracing(example.outputs);
+                const stringifiedOutputs = serializePayloadForTracing(example.outputs);
                 const outputsBlob = new Blob([stringifiedOutputs], {
                     type: "application/json",
                 });
@@ -2857,7 +2865,7 @@ export class Client {
                 }
             }
             if (example.attachments_operations) {
-                const stringifiedAttachmentsOperations = stringifyForTracing(example.attachments_operations);
+                const stringifiedAttachmentsOperations = serializePayloadForTracing(example.attachments_operations);
                 const attachmentsOperationsBlob = new Blob([stringifiedAttachmentsOperations], {
                     type: "application/json",
                 });
@@ -2891,20 +2899,20 @@ export class Client {
                 ...(example.split && { split: example.split }),
             };
             // Add main example data
-            const stringifiedExample = stringifyForTracing(exampleBody);
+            const stringifiedExample = serializePayloadForTracing(exampleBody);
             const exampleBlob = new Blob([stringifiedExample], {
                 type: "application/json",
             });
             formData.append(exampleId, exampleBlob);
             // Add inputs
-            const stringifiedInputs = stringifyForTracing(example.inputs);
+            const stringifiedInputs = serializePayloadForTracing(example.inputs);
             const inputsBlob = new Blob([stringifiedInputs], {
                 type: "application/json",
             });
             formData.append(`${exampleId}.inputs`, inputsBlob);
             // Add outputs if present
             if (example.outputs) {
-                const stringifiedOutputs = stringifyForTracing(example.outputs);
+                const stringifiedOutputs = serializePayloadForTracing(example.outputs);
                 const outputsBlob = new Blob([stringifiedOutputs], {
                     type: "application/json",
                 });

package/dist/evaluation/_runner.cjs

@@ -574,7 +574,6 @@ class _ExperimentManager {
             projectMetadata["revision_id"] = await (0, _git_js_1.getDefaultRevisionId)();
         }
         await this.client.updateProject(experiment.id, {
-            endTime: new Date().toISOString(),
             metadata: projectMetadata,
         });
     }

package/dist/evaluation/_runner.js

@@ -570,7 +570,6 @@ export class _ExperimentManager {
             projectMetadata["revision_id"] = await getDefaultRevisionId();
         }
         await this.client.updateProject(experiment.id, {
-            endTime: new Date().toISOString(),
             metadata: projectMetadata,
         });
     }

package/dist/index.cjs

@@ -8,4 +8,4 @@ Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () {
 var fetch_js_1 = require("./singletons/fetch.cjs");
 Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true, get: function () { return fetch_js_1.overrideFetchImplementation; } });
 // Update using yarn bump-version
-exports.__version__ = "0.3.4-rc.0";
+exports.__version__ = "0.3.5";

package/dist/index.d.ts

@@ -2,4 +2,4 @@ export { Client, type ClientConfig, type LangSmithTracingClientInterface, } from
 export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
-export declare const __version__ = "0.3.4-rc.0";
+export declare const __version__ = "0.3.5";

package/dist/index.js

@@ -2,4 +2,4 @@ export { Client, } from "./client.js";
 export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 // Update using yarn bump-version
-export const __version__ = "0.3.4-rc.0";
+export const __version__ = "0.3.5";

package/dist/jest/reporter.cjs

@@ -5,6 +5,9 @@ const reporters_1 = require("@jest/reporters");
 const reporter_js_1 = require("../utils/jestlike/reporter.cjs");
 class LangSmithEvalReporter extends reporters_1.DefaultReporter {
     async onTestResult(test, testResult, aggregatedResults) {
+        if (testResult.failureMessage) {
+            console.log(testResult.failureMessage);
+        }
         const groupedTestResults = testResult.testResults.reduce((groups, testResult) => {
             const ancestorTitle = testResult.ancestorTitles.join(" > ");
             if (groups[ancestorTitle] === undefined) {
@@ -16,7 +19,13 @@ class LangSmithEvalReporter extends reporters_1.DefaultReporter {
         try {
             for (const testGroupName of Object.keys(groupedTestResults)) {
                 const resultGroup = groupedTestResults[testGroupName];
-                await (0, reporter_js_1.printReporterTable)(resultGroup, testResult.failureMessage);
+                const unskippedTests = resultGroup.filter((result) => result.status !== "pending");
+                const overallResult = unskippedTests.length === 0
+                    ? "skip"
+                    : unskippedTests.every((result) => result.status === "passed")
+                        ? "pass"
+                        : "fail";
+                await (0, reporter_js_1.printReporterTable)(testGroupName, resultGroup, overallResult);
             }
         }
         catch (e) {

package/dist/jest/reporter.js

@@ -3,6 +3,9 @@ import { DefaultReporter } from "@jest/reporters";
 import { printReporterTable } from "../utils/jestlike/reporter.js";
 class LangSmithEvalReporter extends DefaultReporter {
     async onTestResult(test, testResult, aggregatedResults) {
+        if (testResult.failureMessage) {
+            console.log(testResult.failureMessage);
+        }
         const groupedTestResults = testResult.testResults.reduce((groups, testResult) => {
             const ancestorTitle = testResult.ancestorTitles.join(" > ");
             if (groups[ancestorTitle] === undefined) {
@@ -14,7 +17,13 @@ class LangSmithEvalReporter extends DefaultReporter {
         try {
             for (const testGroupName of Object.keys(groupedTestResults)) {
                 const resultGroup = groupedTestResults[testGroupName];
-                await printReporterTable(resultGroup, testResult.failureMessage);
+                const unskippedTests = resultGroup.filter((result) => result.status !== "pending");
+                const overallResult = unskippedTests.length === 0
+                    ? "skip"
+                    : unskippedTests.every((result) => result.status === "passed")
+                        ? "pass"
+                        : "fail";
+                await printReporterTable(testGroupName, resultGroup, overallResult);
             }
         }
         catch (e) {

package/dist/utils/fast-safe-stringify/index.cjs

@@ -1,35 +1,40 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.stringify = void 0;
+exports.serialize = void 0;
 /* eslint-disable */
 // @ts-nocheck
 var LIMIT_REPLACE_NODE = "[...]";
 var CIRCULAR_REPLACE_NODE = { result: "[Circular]" };
 var arr = [];
 var replacerStack = [];
+const encoder = new TextEncoder();
 function defaultOptions() {
     return {
         depthLimit: Number.MAX_SAFE_INTEGER,
         edgesLimit: Number.MAX_SAFE_INTEGER,
     };
 }
+function encodeString(str) {
+    return encoder.encode(str);
+}
 // Regular stringify
-function stringify(obj, replacer, spacer, options) {
+function serialize(obj, replacer, spacer, options) {
     try {
-        return JSON.stringify(obj, replacer, spacer);
+        const str = JSON.stringify(obj, replacer, spacer);
+        return encodeString(str);
     }
     catch (e) {
         // Fall back to more complex stringify if circular reference
         if (!e.message?.includes("Converting circular structure to JSON")) {
             console.warn("[WARNING]: LangSmith received unserializable value.");
-            return "[Unserializable]";
+            return encodeString("[Unserializable]");
         }
         console.warn("[WARNING]: LangSmith received circular JSON. This will decrease tracer performance.");
         if (typeof options === "undefined") {
             options = defaultOptions();
         }
         decirc(obj, "", 0, [], undefined, 0, options);
-        var res;
+        let res;
         try {
             if (replacerStack.length === 0) {
                 res = JSON.stringify(obj, replacer, spacer);
@@ -39,11 +44,11 @@ function stringify(obj, replacer, spacer, options) {
             }
         }
         catch (_) {
-            return JSON.stringify("[unable to serialize, circular reference is too complex to analyze]");
+            return encodeString("[unable to serialize, circular reference is too complex to analyze]");
         }
         finally {
             while (arr.length !== 0) {
-                var part = arr.pop();
+                const part = arr.pop();
                 if (part.length === 4) {
                     Object.defineProperty(part[0], part[1], part[3]);
                 }
@@ -52,10 +57,10 @@ function stringify(obj, replacer, spacer, options) {
                 }
             }
         }
-        return res;
+        return encodeString(res);
     }
 }
-exports.stringify = stringify;
+exports.serialize = serialize;
 function setReplace(replace, val, k, parent) {
     var propertyDescriptor = Object.getOwnPropertyDescriptor(parent, k);
     if (propertyDescriptor.get !== undefined) {

package/dist/utils/fast-safe-stringify/index.d.ts

@@ -1 +1 @@
-export declare function stringify(obj: any, replacer?: any, spacer?: any, options?: any): string;
+export declare function serialize(obj: any, replacer?: any, spacer?: any, options?: any): Uint8Array;

package/dist/utils/fast-safe-stringify/index.js

@@ -4,29 +4,34 @@ var LIMIT_REPLACE_NODE = "[...]";
 var CIRCULAR_REPLACE_NODE = { result: "[Circular]" };
 var arr = [];
 var replacerStack = [];
+const encoder = new TextEncoder();
 function defaultOptions() {
     return {
         depthLimit: Number.MAX_SAFE_INTEGER,
         edgesLimit: Number.MAX_SAFE_INTEGER,
     };
 }
+function encodeString(str) {
+    return encoder.encode(str);
+}
 // Regular stringify
-export function stringify(obj, replacer, spacer, options) {
+export function serialize(obj, replacer, spacer, options) {
     try {
-        return JSON.stringify(obj, replacer, spacer);
+        const str = JSON.stringify(obj, replacer, spacer);
+        return encodeString(str);
     }
     catch (e) {
         // Fall back to more complex stringify if circular reference
         if (!e.message?.includes("Converting circular structure to JSON")) {
             console.warn("[WARNING]: LangSmith received unserializable value.");
-            return "[Unserializable]";
+            return encodeString("[Unserializable]");
         }
         console.warn("[WARNING]: LangSmith received circular JSON. This will decrease tracer performance.");
         if (typeof options === "undefined") {
             options = defaultOptions();
         }
         decirc(obj, "", 0, [], undefined, 0, options);
-        var res;
+        let res;
         try {
             if (replacerStack.length === 0) {
                 res = JSON.stringify(obj, replacer, spacer);
@@ -36,11 +41,11 @@ export function stringify(obj, replacer, spacer, options) {
             }
         }
         catch (_) {
-            return JSON.stringify("[unable to serialize, circular reference is too complex to analyze]");
+            return encodeString("[unable to serialize, circular reference is too complex to analyze]");
         }
         finally {
             while (arr.length !== 0) {
-                var part = arr.pop();
+                const part = arr.pop();
                 if (part.length === 4) {
                     Object.defineProperty(part[0], part[1], part[3]);
                 }
@@ -49,7 +54,7 @@ export function stringify(obj, replacer, spacer, options) {
                 }
             }
         }
-        return res;
+        return encodeString(res);
     }
 }
 function setReplace(replace, val, k, parent) {

package/dist/utils/jestlike/reporter.cjs

@@ -53,13 +53,13 @@ function formatTestName(name, duration) {
 }
 function getFormattedStatus(status) {
     const s = status.toLowerCase();
-    if (s === "pending") {
+    if (s === "pending" || s === "skipped") {
         return chalk_1.default.yellow("○ Skipped");
     }
-    else if (s === "passed") {
+    else if (s.includes("pass")) {
         return chalk_1.default.green("✓ Passed");
     }
-    else if (s === "failed") {
+    else if (s.includes("fail")) {
         return chalk_1.default.red("✕ Failed");
     }
     else {
@@ -68,13 +68,13 @@ function getFormattedStatus(status) {
 }
 function getColorParam(status) {
     const s = status.toLowerCase();
-    if (s === "pending") {
+    if (s === "pending" || s === "skipped") {
         return { color: "yellow" };
     }
-    else if (s === "passed") {
+    else if (s.includes("pass")) {
         return { color: "grey" };
     }
-    else if (s === "failed") {
+    else if (s.includes("fail")) {
         return { color: "red" };
     }
     else {
@@ -99,7 +99,7 @@ function formatValue(value) {
     }
     return String(value);
 }
-async function printReporterTable(results, failureMessage) {
+async function printReporterTable(testSuiteName, results, testStatus, failureMessage) {
     const rows = [];
     const feedbackKeys = new Set();
     let experimentUrl;
@@ -122,7 +122,7 @@ async function printReporterTable(results, failureMessage) {
                 getColorParam(status),
             ]);
         }
-        else if (status === "pending") {
+        else if (status === "pending" || status === "skipped") {
             // Skipped
             rows.push([
                 {
@@ -265,6 +265,14 @@ async function printReporterTable(results, failureMessage) {
     for (const row of rows) {
         table.addRow(row[0], row[1]);
     }
+    const testStatusColor = testStatus.includes("pass")
+        ? chalk_1.default.green
+        : testStatus.includes("fail")
+            ? chalk_1.default.red
+            : chalk_1.default.yellow;
+    if (testSuiteName) {
+        console.log(testStatusColor(`› ${testSuiteName}`));
+    }
     if (failureMessage) {
         console.log(failureMessage);
     }

package/dist/utils/jestlike/reporter.d.ts

@@ -1,5 +1,5 @@
-export declare function printReporterTable(results: {
+export declare function printReporterTable(testSuiteName: string, results: {
     title: string;
     duration: number;
-    status: string;
-}[], failureMessage?: string): Promise<void>;
+    status: "pass" | "passed" | "fail" | "failed" | "pending" | "skipped";
+}[], testStatus: "pass" | "skip" | "fail", failureMessage?: string): Promise<void>;

package/dist/utils/jestlike/reporter.js

@@ -24,13 +24,13 @@ function formatTestName(name, duration) {
 }
 function getFormattedStatus(status) {
     const s = status.toLowerCase();
-    if (s === "pending") {
+    if (s === "pending" || s === "skipped") {
         return chalk.yellow("○ Skipped");
     }
-    else if (s === "passed") {
+    else if (s.includes("pass")) {
         return chalk.green("✓ Passed");
     }
-    else if (s === "failed") {
+    else if (s.includes("fail")) {
         return chalk.red("✕ Failed");
     }
     else {
@@ -39,13 +39,13 @@ function getFormattedStatus(status) {
 }
 function getColorParam(status) {
     const s = status.toLowerCase();
-    if (s === "pending") {
+    if (s === "pending" || s === "skipped") {
         return { color: "yellow" };
     }
-    else if (s === "passed") {
+    else if (s.includes("pass")) {
         return { color: "grey" };
     }
-    else if (s === "failed") {
+    else if (s.includes("fail")) {
         return { color: "red" };
     }
     else {
@@ -70,7 +70,7 @@ function formatValue(value) {
     }
     return String(value);
 }
-export async function printReporterTable(results, failureMessage) {
+export async function printReporterTable(testSuiteName, results, testStatus, failureMessage) {
     const rows = [];
     const feedbackKeys = new Set();
     let experimentUrl;
@@ -93,7 +93,7 @@ export async function printReporterTable(results, failureMessage) {
                 getColorParam(status),
             ]);
         }
-        else if (status === "pending") {
+        else if (status === "pending" || status === "skipped") {
             // Skipped
             rows.push([
                 {
@@ -236,6 +236,14 @@ export async function printReporterTable(results, failureMessage) {
     for (const row of rows) {
         table.addRow(row[0], row[1]);
     }
+    const testStatusColor = testStatus.includes("pass")
+        ? chalk.green
+        : testStatus.includes("fail")
+            ? chalk.red
+            : chalk.yellow;
+    if (testSuiteName) {
+        console.log(testStatusColor(`› ${testSuiteName}`));
+    }
     if (failureMessage) {
         console.log(failureMessage);
     }

package/dist/vitest/reporter.cjs

@@ -9,15 +9,20 @@ class LangSmithEvalReporter extends reporters_1.DefaultReporter {
     async onFinished(files, errors) {
         super.onFinished(files, errors);
         for (const file of files) {
-            const testModule = this.ctx.state.getReportedEntity(file);
-            const tests = [...testModule.children.allTests()].map((test) => {
-                return {
-                    title: test.name,
-                    status: test.result()?.state ?? "skipped",
-                    duration: Math.round(test.diagnostic()?.duration ?? 0),
-                };
-            });
-            await (0, reporter_js_1.printReporterTable)(tests);
+            for (const task of file.tasks) {
+                const testModule = this.ctx.state.getReportedEntity(task);
+                const tests = [...testModule.children.allTests()].map((test) => {
+                    return {
+                        title: test.name,
+                        status: test.result()?.state ?? "skipped",
+                        duration: Math.round(test.diagnostic()?.duration ?? 0),
+                    };
+                });
+                const result = ["pass", "fail", "skip"].includes(task.result?.state ?? "")
+                    ? task.result?.state
+                    : "skip";
+                await (0, reporter_js_1.printReporterTable)(task.name, tests, result);
+            }
         }
     }
 }

package/dist/vitest/reporter.js

@@ -7,15 +7,20 @@ class LangSmithEvalReporter extends DefaultReporter {
     async onFinished(files, errors) {
         super.onFinished(files, errors);
         for (const file of files) {
-            const testModule = this.ctx.state.getReportedEntity(file);
-            const tests = [...testModule.children.allTests()].map((test) => {
-                return {
-                    title: test.name,
-                    status: test.result()?.state ?? "skipped",
-                    duration: Math.round(test.diagnostic()?.duration ?? 0),
-                };
-            });
-            await printReporterTable(tests);
+            for (const task of file.tasks) {
+                const testModule = this.ctx.state.getReportedEntity(task);
+                const tests = [...testModule.children.allTests()].map((test) => {
+                    return {
+                        title: test.name,
+                        status: test.result()?.state ?? "skipped",
+                        duration: Math.round(test.diagnostic()?.duration ?? 0),
+                    };
+                });
+                const result = ["pass", "fail", "skip"].includes(task.result?.state ?? "")
+                    ? task.result?.state
+                    : "skip";
+                await printReporterTable(task.name, tests, result);
+            }
         }
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.3.4-rc.0",
+  "version": "0.3.5",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [