langsmith 0.1.14 → 0.1.19

package/client.d.cts

@@ -0,0 +1 @@
+export * from './dist/client.js'

package/dist/client.cjs

@@ -777,7 +777,7 @@ class Client {
      * @param traceId - The ID of the trace to filter by.
      * @param referenceExampleId - The ID of the reference example to filter by.
      * @param startTime - The start time to filter by.
-     * @param executionOrder - The execution order to filter by.
+     * @param isRoot - Indicates whether to only return root runs.
      * @param runType - The run type to filter by.
      * @param error - Indicates whether to filter by error runs.
      * @param id - The ID of the run to filter by.
@@ -851,7 +851,7 @@ class Client {
      * });
      */
     async *listRuns(props) {
-        const { projectId, projectName, parentRunId, traceId, referenceExampleId, startTime, executionOrder, runType, error, id, query, filter, traceFilter, treeFilter, limit, } = props;
+        const { projectId, projectName, parentRunId, traceId, referenceExampleId, startTime, executionOrder, isRoot, runType, error, id, query, filter, traceFilter, treeFilter, limit, select, } = props;
         let projectIds = [];
         if (projectId) {
             projectIds = Array.isArray(projectId) ? projectId : [projectId];
@@ -863,6 +863,36 @@ class Client {
             const projectIds_ = await Promise.all(projectNames.map((name) => this.readProject({ projectName: name }).then((project) => project.id)));
             projectIds.push(...projectIds_);
         }
+        const default_select = [
+            "app_path",
+            "child_run_ids",
+            "completion_cost",
+            "completion_tokens",
+            "dotted_order",
+            "end_time",
+            "error",
+            "events",
+            "extra",
+            "feedback_stats",
+            "first_token_time",
+            "id",
+            "inputs",
+            "name",
+            "outputs",
+            "parent_run_id",
+            "parent_run_ids",
+            "prompt_cost",
+            "prompt_tokens",
+            "reference_example_id",
+            "run_type",
+            "session_id",
+            "start_time",
+            "status",
+            "tags",
+            "total_cost",
+            "total_tokens",
+            "trace_id",
+        ];
         const body = {
             session: projectIds.length ? projectIds : null,
             run_type: runType,
@@ -878,6 +908,8 @@ class Client {
             id,
             limit,
             trace: traceId,
+            select: select ? select : default_select,
+            is_root: isRoot,
         };
         for await (const runs of this._getCursorPaginatedList("/runs/query", body)) {
             yield* runs;
@@ -1377,7 +1409,7 @@ class Client {
         }
         await response.json();
     }
-    async createExample(inputs, outputs, { datasetId, datasetName, createdAt, exampleId }) {
+    async createExample(inputs, outputs, { datasetId, datasetName, createdAt, exampleId, metadata, }) {
         let datasetId_ = datasetId;
         if (datasetId_ === undefined && datasetName === undefined) {
             throw new Error("Must provide either datasetName or datasetId");
@@ -1396,6 +1428,7 @@ class Client {
             outputs,
             created_at: createdAt_?.toISOString(),
             id: exampleId,
+            metadata,
         };
         const response = await this.caller.call(fetch, `${this.apiUrl}/examples`, {
             method: "POST",
@@ -1411,7 +1444,7 @@ class Client {
         return result;
     }
     async createExamples(props) {
-        const { inputs, outputs, sourceRunIds, exampleIds, datasetId, datasetName, } = props;
+        const { inputs, outputs, metadata, sourceRunIds, exampleIds, datasetId, datasetName, } = props;
         let datasetId_ = datasetId;
         if (datasetId_ === undefined && datasetName === undefined) {
             throw new Error("Must provide either datasetName or datasetId");
@@ -1428,6 +1461,7 @@ class Client {
                 dataset_id: datasetId_,
                 inputs: input,
                 outputs: outputs ? outputs[idx] : undefined,
+                metadata: metadata ? metadata[idx] : undefined,
                 id: exampleIds ? exampleIds[idx] : undefined,
                 source_run_id: sourceRunIds ? sourceRunIds[idx] : undefined,
             };

package/dist/client.d.ts

@@ -29,6 +29,10 @@ interface ListRunsParams {
      * The ID of the trace to filter by.
      */
     traceId?: string;
+    /**
+     * isRoot - Whether to only include root runs.
+     *  */
+    isRoot?: boolean;
     /**
      * The execution order to filter by.
      */
@@ -96,6 +100,11 @@ interface ListRunsParams {
      * conjunction with the regular `filter` parameter to let you filter runs by attributes of any run within a trace.
      */
     treeFilter?: string;
+    /**
+     * The values to include in the response.
+     *
+     */
+    select?: string[];
 }
 interface UploadCSVParams {
     csvFile: Blob;
@@ -135,6 +144,7 @@ export type CreateExampleOptions = {
     datasetName?: string;
     createdAt?: Date;
     exampleId?: string;
+    metadata?: KVMap;
 };
 export declare class Queue<T> {
     items: [T, () => void][];
@@ -214,7 +224,7 @@ export declare class Client {
      * @param traceId - The ID of the trace to filter by.
      * @param referenceExampleId - The ID of the reference example to filter by.
      * @param startTime - The start time to filter by.
-     * @param executionOrder - The execution order to filter by.
+     * @param isRoot - Indicates whether to only return root runs.
      * @param runType - The run type to filter by.
      * @param error - Indicates whether to filter by error runs.
      * @param id - The ID of the run to filter by.
@@ -367,10 +377,11 @@ export declare class Client {
         datasetId?: string;
         datasetName?: string;
     }): Promise<void>;
-    createExample(inputs: KVMap, outputs: KVMap, { datasetId, datasetName, createdAt, exampleId }: CreateExampleOptions): Promise<Example>;
+    createExample(inputs: KVMap, outputs: KVMap, { datasetId, datasetName, createdAt, exampleId, metadata, }: CreateExampleOptions): Promise<Example>;
     createExamples(props: {
         inputs: Array<KVMap>;
         outputs?: Array<KVMap>;
+        metadata?: Array<KVMap>;
         sourceRunIds?: Array<string>;
         exampleIds?: Array<string>;
         datasetId?: string;

package/dist/client.js

@@ -750,7 +750,7 @@ export class Client {
      * @param traceId - The ID of the trace to filter by.
      * @param referenceExampleId - The ID of the reference example to filter by.
      * @param startTime - The start time to filter by.
-     * @param executionOrder - The execution order to filter by.
+     * @param isRoot - Indicates whether to only return root runs.
      * @param runType - The run type to filter by.
      * @param error - Indicates whether to filter by error runs.
      * @param id - The ID of the run to filter by.
@@ -824,7 +824,7 @@ export class Client {
      * });
      */
     async *listRuns(props) {
-        const { projectId, projectName, parentRunId, traceId, referenceExampleId, startTime, executionOrder, runType, error, id, query, filter, traceFilter, treeFilter, limit, } = props;
+        const { projectId, projectName, parentRunId, traceId, referenceExampleId, startTime, executionOrder, isRoot, runType, error, id, query, filter, traceFilter, treeFilter, limit, select, } = props;
         let projectIds = [];
         if (projectId) {
             projectIds = Array.isArray(projectId) ? projectId : [projectId];
@@ -836,6 +836,36 @@ export class Client {
             const projectIds_ = await Promise.all(projectNames.map((name) => this.readProject({ projectName: name }).then((project) => project.id)));
             projectIds.push(...projectIds_);
         }
+        const default_select = [
+            "app_path",
+            "child_run_ids",
+            "completion_cost",
+            "completion_tokens",
+            "dotted_order",
+            "end_time",
+            "error",
+            "events",
+            "extra",
+            "feedback_stats",
+            "first_token_time",
+            "id",
+            "inputs",
+            "name",
+            "outputs",
+            "parent_run_id",
+            "parent_run_ids",
+            "prompt_cost",
+            "prompt_tokens",
+            "reference_example_id",
+            "run_type",
+            "session_id",
+            "start_time",
+            "status",
+            "tags",
+            "total_cost",
+            "total_tokens",
+            "trace_id",
+        ];
         const body = {
             session: projectIds.length ? projectIds : null,
             run_type: runType,
@@ -851,6 +881,8 @@ export class Client {
             id,
             limit,
             trace: traceId,
+            select: select ? select : default_select,
+            is_root: isRoot,
         };
         for await (const runs of this._getCursorPaginatedList("/runs/query", body)) {
             yield* runs;
@@ -1350,7 +1382,7 @@ export class Client {
         }
         await response.json();
     }
-    async createExample(inputs, outputs, { datasetId, datasetName, createdAt, exampleId }) {
+    async createExample(inputs, outputs, { datasetId, datasetName, createdAt, exampleId, metadata, }) {
         let datasetId_ = datasetId;
         if (datasetId_ === undefined && datasetName === undefined) {
             throw new Error("Must provide either datasetName or datasetId");
@@ -1369,6 +1401,7 @@ export class Client {
             outputs,
             created_at: createdAt_?.toISOString(),
             id: exampleId,
+            metadata,
         };
         const response = await this.caller.call(fetch, `${this.apiUrl}/examples`, {
             method: "POST",
@@ -1384,7 +1417,7 @@ export class Client {
         return result;
     }
     async createExamples(props) {
-        const { inputs, outputs, sourceRunIds, exampleIds, datasetId, datasetName, } = props;
+        const { inputs, outputs, metadata, sourceRunIds, exampleIds, datasetId, datasetName, } = props;
         let datasetId_ = datasetId;
         if (datasetId_ === undefined && datasetName === undefined) {
             throw new Error("Must provide either datasetName or datasetId");
@@ -1401,6 +1434,7 @@ export class Client {
                 dataset_id: datasetId_,
                 inputs: input,
                 outputs: outputs ? outputs[idx] : undefined,
+                metadata: metadata ? metadata[idx] : undefined,
                 id: exampleIds ? exampleIds[idx] : undefined,
                 source_run_id: sourceRunIds ? sourceRunIds[idx] : undefined,
             };

package/dist/evaluation/evaluator.d.ts

@@ -1,4 +1,42 @@
 import { Example, Run, ScoreType, ValueType } from "../schemas.js";
+/**
+ * Represents a categorical class.
+ */
+export type Category = {
+    /**
+     * The value of the category.
+     */
+    value?: number;
+    /**
+     * The label of the category.
+     */
+    label: string;
+};
+/**
+ * Configuration for feedback.
+ */
+export type FeedbackConfig = {
+    /**
+     * The type of feedback.
+     * - "continuous": Feedback with a continuous numeric.
+     * - "categorical": Feedback with a categorical value (classes)
+     * - "freeform": Feedback with a freeform text value (notes).
+     */
+    type: "continuous" | "categorical" | "freeform";
+    /**
+     * The minimum value for continuous feedback.
+     */
+    min?: number;
+    /**
+     * The maximum value for continuous feedback.
+     */
+    max?: number;
+    /**
+     * The categories for categorical feedback.
+     * Each category can be a string or an object with additional properties.
+     */
+    categories?: (Category | Record<string, unknown>)[];
+};
 /**
  * Represents the result of an evaluation.
  */
@@ -38,6 +76,12 @@ export type EvaluationResult = {
      * the root of the trace.
      */
     targetRunId?: string;
+    /**
+     * The feedback config associated with the evaluation result.
+     * If set, this will be used to define how a feedback key
+     * should be interpreted.
+     */
+    feedbackConfig?: FeedbackConfig;
 };
 export interface RunEvaluator {
     evaluateRun(run: Run, example?: Example): Promise<EvaluationResult>;

package/dist/index.cjs

@@ -6,4 +6,4 @@ Object.defineProperty(exports, "Client", { enumerable: true, get: function () {
 var run_trees_js_1 = require("./run_trees.cjs");
 Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () { return run_trees_js_1.RunTree; } });
 // Update using yarn bump-version
-exports.__version__ = "0.1.14";
+exports.__version__ = "0.1.19";

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 export { Client } from "./client.js";
 export type { Dataset, Example, TracerSession, Run, Feedback, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
-export declare const __version__ = "0.1.14";
+export declare const __version__ = "0.1.19";

package/dist/index.js

@@ -1,4 +1,4 @@
 export { Client } from "./client.js";
 export { RunTree } from "./run_trees.js";
 // Update using yarn bump-version
-export const __version__ = "0.1.14";
+export const __version__ = "0.1.19";

package/dist/run_trees.cjs

@@ -43,7 +43,7 @@ function convertToDottedOrderFormat(epoch, runId) {
 }
 exports.convertToDottedOrderFormat = convertToDottedOrderFormat;
 class RunTree {
-    constructor(config) {
+    constructor(originalConfig) {
         Object.defineProperty(this, "id", {
             enumerable: true,
             configurable: true,
@@ -159,7 +159,13 @@ class RunTree {
             value: void 0
         });
         const defaultConfig = RunTree.getDefaultConfig();
+        const { metadata, ...config } = originalConfig;
         const client = config.client ?? new client_js_1.Client();
+        const dedupedMetadata = {
+            ...metadata,
+            ...config?.extra?.metadata,
+        };
+        config.extra = { ...config.extra, metadata: dedupedMetadata };
         Object.assign(this, { ...defaultConfig, ...config, client });
         if (!this.trace_id) {
             if (this.parent_run) {
@@ -191,7 +197,7 @@ class RunTree {
             parentRun = langChainTracer?.getRun?.(parentRunId);
             projectName = langChainTracer?.projectName;
         }
-        const deduppedTags = [
+        const dedupedTags = [
             ...new Set((parentRun?.tags ?? []).concat(config?.tags ?? [])),
         ];
         const dedupedMetadata = {
@@ -201,7 +207,7 @@ class RunTree {
         const rt = new RunTree({
             name: props?.name ?? "<lambda>",
             parent_run: parentRun,
-            tags: deduppedTags,
+            tags: dedupedTags,
             extra: {
                 metadata: dedupedMetadata,
             },

package/dist/run_trees.d.ts

@@ -12,6 +12,7 @@ export interface RunTreeConfig {
     start_time?: number;
     end_time?: number;
     extra?: KVMap;
+    metadata?: KVMap;
     tags?: string[];
     error?: string;
     serialized?: object;
@@ -57,7 +58,7 @@ export declare class RunTree implements BaseRun {
     events?: KVMap[] | undefined;
     trace_id: string;
     dotted_order: string;
-    constructor(config: RunTreeConfig);
+    constructor(originalConfig: RunTreeConfig);
     static fromRunnableConfig(config: RunnableConfigLike, props: {
         name: string;
         tags?: string[];

package/dist/run_trees.js

@@ -16,7 +16,7 @@ export function convertToDottedOrderFormat(epoch, runId) {
         runId);
 }
 export class RunTree {
-    constructor(config) {
+    constructor(originalConfig) {
         Object.defineProperty(this, "id", {
             enumerable: true,
             configurable: true,
@@ -132,7 +132,13 @@ export class RunTree {
             value: void 0
         });
         const defaultConfig = RunTree.getDefaultConfig();
+        const { metadata, ...config } = originalConfig;
         const client = config.client ?? new Client();
+        const dedupedMetadata = {
+            ...metadata,
+            ...config?.extra?.metadata,
+        };
+        config.extra = { ...config.extra, metadata: dedupedMetadata };
         Object.assign(this, { ...defaultConfig, ...config, client });
         if (!this.trace_id) {
             if (this.parent_run) {
@@ -164,7 +170,7 @@ export class RunTree {
             parentRun = langChainTracer?.getRun?.(parentRunId);
             projectName = langChainTracer?.projectName;
         }
-        const deduppedTags = [
+        const dedupedTags = [
             ...new Set((parentRun?.tags ?? []).concat(config?.tags ?? [])),
         ];
         const dedupedMetadata = {
@@ -174,7 +180,7 @@ export class RunTree {
         const rt = new RunTree({
             name: props?.name ?? "<lambda>",
             parent_run: parentRun,
-            tags: deduppedTags,
+            tags: dedupedTags,
             extra: {
                 metadata: dedupedMetadata,
             },

package/dist/schemas.d.ts

@@ -15,7 +15,7 @@ export interface TracerSessionResult extends TracerSession {
     completion_tokens?: number;
     last_run_start_time?: number;
     feedback_stats?: Record<string, unknown>;
-    reference_dataset_ids?: string[];
+    reference_dataset_id?: string;
     run_facets?: KVMap[];
 }
 export type KVMap = Record<string, any>;
@@ -27,6 +27,7 @@ export interface BaseExample {
     dataset_id: string;
     inputs: KVMap;
     outputs?: KVMap;
+    metadata?: KVMap;
 }
 /**
  * A run can represent either a trace (root run)
@@ -108,6 +109,8 @@ export interface Run extends BaseRun {
     first_token_time?: number;
     /** IDs of parent runs, if multiple exist. */
     parent_run_ids?: string[];
+    /** Whether the run is included in a dataset. */
+    in_dataset?: boolean;
 }
 export interface RunCreate extends BaseRun {
     revision_id?: string;
@@ -157,6 +160,7 @@ export interface ExampleUpdate {
     dataset_id?: string;
     inputs?: KVMap;
     outputs?: KVMap;
+    metadata?: KVMap;
 }
 export interface BaseDataset {
     name: string;

package/dist/traceable.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isTraceableFunction = exports.traceable = void 0;
+exports.isTraceableFunction = exports.getCurrentRunTree = exports.traceable = void 0;
 const async_hooks_1 = require("async_hooks");
 const run_trees_js_1 = require("./run_trees.cjs");
 const asyncLocalStorage = new async_hooks_1.AsyncLocalStorage();
@@ -24,12 +24,13 @@ const isAsyncIterable = (x) => x != null &&
  */
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 function traceable(wrappedFunc, config) {
+    const { aggregator, ...runTreeConfig } = config ?? {};
     const traceableFunc = async (...args) => {
         let currentRunTree;
         let rawInputs;
         const ensuredConfig = {
             name: wrappedFunc.name || "<lambda>",
-            ...config,
+            ...runTreeConfig,
         };
         const previousRunTree = asyncLocalStorage.getStore();
         if ((0, run_trees_js_1.isRunTree)(args[0])) {
@@ -79,7 +80,26 @@ function traceable(wrappedFunc, config) {
                                 chunks.push(chunk);
                                 yield chunk;
                             }
-                            await currentRunTree.end({ outputs: chunks });
+                            let finalOutputs;
+                            if (aggregator !== undefined) {
+                                try {
+                                    finalOutputs = await aggregator(chunks);
+                                }
+                                catch (e) {
+                                    console.error(`[ERROR]: LangSmith aggregation failed: `, e);
+                                    finalOutputs = chunks;
+                                }
+                            }
+                            else {
+                                finalOutputs = chunks;
+                            }
+                            if (typeof finalOutputs === "object" &&
+                                !Array.isArray(finalOutputs)) {
+                                await currentRunTree.end(finalOutputs);
+                            }
+                            else {
+                                await currentRunTree.end({ outputs: finalOutputs });
+                            }
                             await currentRunTree.patchRun();
                         }
                         return resolve(wrapOutputForTracing());
@@ -112,11 +132,29 @@ function traceable(wrappedFunc, config) {
         });
     };
     Object.defineProperty(traceableFunc, "langsmith:traceable", {
-        value: config,
+        value: runTreeConfig,
     });
     return traceableFunc;
 }
 exports.traceable = traceable;
+/**
+ * Return the current run tree from within a traceable-wrapped function.
+ * Will throw an error if called outside of a traceable function.
+ *
+ * @returns The run tree for the given context.
+ */
+function getCurrentRunTree() {
+    const runTree = asyncLocalStorage.getStore();
+    if (runTree === undefined) {
+        throw new Error([
+            "Could not get the current run tree.",
+            "",
+            "Please make sure you are calling this method within a traceable function.",
+        ].join("\n"));
+    }
+    return runTree;
+}
+exports.getCurrentRunTree = getCurrentRunTree;
 function isTraceableFunction(x
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 ) {

package/dist/traceable.d.ts

@@ -44,6 +44,15 @@ export type TraceableFunction<Func extends (...args: any[]) => any> = Func exten
  * @param config Additional metadata such as name, tags or providing
  *     a custom LangSmith client instance
  */
-export declare function traceable<Func extends (...args: any[]) => any>(wrappedFunc: Func, config?: Partial<RunTreeConfig>): TraceableFunction<Func>;
+export declare function traceable<Func extends (...args: any[]) => any>(wrappedFunc: Func, config?: Partial<RunTreeConfig> & {
+    aggregator?: (args: any[]) => any;
+}): TraceableFunction<Func>;
+/**
+ * Return the current run tree from within a traceable-wrapped function.
+ * Will throw an error if called outside of a traceable function.
+ *
+ * @returns The run tree for the given context.
+ */
+export declare function getCurrentRunTree(): RunTree;
 export declare function isTraceableFunction(x: unknown): x is TraceableFunction<any>;
 export {};

package/dist/traceable.js

@@ -21,12 +21,13 @@ const isAsyncIterable = (x) => x != null &&
  */
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export function traceable(wrappedFunc, config) {
+    const { aggregator, ...runTreeConfig } = config ?? {};
     const traceableFunc = async (...args) => {
         let currentRunTree;
         let rawInputs;
         const ensuredConfig = {
             name: wrappedFunc.name || "<lambda>",
-            ...config,
+            ...runTreeConfig,
         };
         const previousRunTree = asyncLocalStorage.getStore();
         if (isRunTree(args[0])) {
@@ -76,7 +77,26 @@ export function traceable(wrappedFunc, config) {
                                 chunks.push(chunk);
                                 yield chunk;
                             }
-                            await currentRunTree.end({ outputs: chunks });
+                            let finalOutputs;
+                            if (aggregator !== undefined) {
+                                try {
+                                    finalOutputs = await aggregator(chunks);
+                                }
+                                catch (e) {
+                                    console.error(`[ERROR]: LangSmith aggregation failed: `, e);
+                                    finalOutputs = chunks;
+                                }
+                            }
+                            else {
+                                finalOutputs = chunks;
+                            }
+                            if (typeof finalOutputs === "object" &&
+                                !Array.isArray(finalOutputs)) {
+                                await currentRunTree.end(finalOutputs);
+                            }
+                            else {
+                                await currentRunTree.end({ outputs: finalOutputs });
+                            }
                             await currentRunTree.patchRun();
                         }
                         return resolve(wrapOutputForTracing());
@@ -109,10 +129,27 @@ export function traceable(wrappedFunc, config) {
         });
     };
     Object.defineProperty(traceableFunc, "langsmith:traceable", {
-        value: config,
+        value: runTreeConfig,
     });
     return traceableFunc;
 }
+/**
+ * Return the current run tree from within a traceable-wrapped function.
+ * Will throw an error if called outside of a traceable function.
+ *
+ * @returns The run tree for the given context.
+ */
+export function getCurrentRunTree() {
+    const runTree = asyncLocalStorage.getStore();
+    if (runTree === undefined) {
+        throw new Error([
+            "Could not get the current run tree.",
+            "",
+            "Please make sure you are calling this method within a traceable function.",
+        ].join("\n"));
+    }
+    return runTree;
+}
 export function isTraceableFunction(x
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 ) {

package/dist/wrappers/index.cjs

@@ -0,0 +1,17 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./openai.cjs"), exports);

package/dist/wrappers/index.d.ts

@@ -0,0 +1 @@
+export * from "./openai.js";

package/dist/wrappers/index.js

@@ -0,0 +1 @@
+export * from "./openai.js";