langsmith 0.1.28 → 0.1.30

package/dist/evaluation/_runner.cjs

@@ -48,7 +48,16 @@ class _ExperimentManager {
             for await (const example of unresolvedData) {
                 exs.push(example);
             }
-            this.setExamples(exs);
+            if (this._numRepetitions && this._numRepetitions > 0) {
+                const repeatedExamples = [];
+                for (let i = 0; i < this._numRepetitions; i++) {
+                    repeatedExamples.push(...exs);
+                }
+                this.setExamples(repeatedExamples);
+            }
+            else {
+                this.setExamples(exs);
+            }
         }
         return this._examples;
     }
@@ -120,6 +129,12 @@ class _ExperimentManager {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "_numRepetitions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "_runsArray", {
             enumerable: true,
             configurable: true,
@@ -188,6 +203,7 @@ class _ExperimentManager {
         this._runs = args.runs;
         this._evaluationResults = args.evaluationResults;
         this._summaryResults = args.summaryResults;
+        this._numRepetitions = args.numRepetitions;
     }
     _getExperiment() {
         if (!this._experiment) {
@@ -255,6 +271,7 @@ class _ExperimentManager {
         const firstExample = examples[0];
         const project = await this._getProject(firstExample);
         await this._printExperimentStart();
+        this._metadata["num_repetitions"] = this._numRepetitions;
         return new _ExperimentManager({
             examples,
             experiment: project,
@@ -614,6 +631,7 @@ async function _evaluate(target, fields) {
         metadata: fields.metadata,
         experiment: experiment_ ?? fields.experimentPrefix,
         runs: newRuns ?? undefined,
+        numRepetitions: fields.numRepetitions ?? 1,
     }).start();
     if (_isCallable(target)) {
         manager = await manager.withPredictions(convertInvokeToTopLevel(target), { maxConcurrency: fields.maxConcurrency });

package/dist/evaluation/_runner.d.ts

@@ -23,6 +23,7 @@ interface _ExperimentManagerArgs {
     evaluationResults?: AsyncGenerator<EvaluationResults>;
     summaryResults?: AsyncGenerator<(runsArray: Run[]) => AsyncGenerator<EvaluationResults, any, unknown>, any, unknown>;
     examples?: Example[];
+    numRepetitions?: number;
     _runsArray?: Run[];
 }
 export interface EvaluateOptions {
@@ -65,6 +66,12 @@ export interface EvaluateOptions {
      * @default undefined
      */
     client?: Client;
+    /**
+     * The number of repetitions to perform. Each example
+     * will be run this many times.
+     * @default 1
+     */
+    numRepetitions?: number;
 }
 export declare function evaluate(
 /**
@@ -88,6 +95,7 @@ declare class _ExperimentManager {
     _evaluationResults?: AsyncGenerator<EvaluationResults>;
     _summaryResults?: AsyncGenerator<(runsArray: Run[]) => AsyncGenerator<EvaluationResults, any, unknown>, any, unknown>;
     _examples?: Example[];
+    _numRepetitions?: number;
     _runsArray?: Run[];
     client: Client;
     _experiment?: TracerSession;

package/dist/evaluation/_runner.js

@@ -44,7 +44,16 @@ class _ExperimentManager {
             for await (const example of unresolvedData) {
                 exs.push(example);
             }
-            this.setExamples(exs);
+            if (this._numRepetitions && this._numRepetitions > 0) {
+                const repeatedExamples = [];
+                for (let i = 0; i < this._numRepetitions; i++) {
+                    repeatedExamples.push(...exs);
+                }
+                this.setExamples(repeatedExamples);
+            }
+            else {
+                this.setExamples(exs);
+            }
         }
         return this._examples;
     }
@@ -116,6 +125,12 @@ class _ExperimentManager {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "_numRepetitions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "_runsArray", {
             enumerable: true,
             configurable: true,
@@ -184,6 +199,7 @@ class _ExperimentManager {
         this._runs = args.runs;
         this._evaluationResults = args.evaluationResults;
         this._summaryResults = args.summaryResults;
+        this._numRepetitions = args.numRepetitions;
     }
     _getExperiment() {
         if (!this._experiment) {
@@ -251,6 +267,7 @@ class _ExperimentManager {
         const firstExample = examples[0];
         const project = await this._getProject(firstExample);
         await this._printExperimentStart();
+        this._metadata["num_repetitions"] = this._numRepetitions;
         return new _ExperimentManager({
             examples,
             experiment: project,
@@ -610,6 +627,7 @@ async function _evaluate(target, fields) {
         metadata: fields.metadata,
         experiment: experiment_ ?? fields.experimentPrefix,
         runs: newRuns ?? undefined,
+        numRepetitions: fields.numRepetitions ?? 1,
     }).start();
     if (_isCallable(target)) {
         manager = await manager.withPredictions(convertInvokeToTopLevel(target), { maxConcurrency: fields.maxConcurrency });

package/dist/index.cjs

@@ -6,4 +6,4 @@ Object.defineProperty(exports, "Client", { enumerable: true, get: function () {
 var run_trees_js_1 = require("./run_trees.cjs");
 Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () { return run_trees_js_1.RunTree; } });
 // Update using yarn bump-version
-exports.__version__ = "0.1.28";
+exports.__version__ = "0.1.30";

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 export { Client } from "./client.js";
 export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
-export declare const __version__ = "0.1.28";
+export declare const __version__ = "0.1.30";

package/dist/index.js

@@ -1,4 +1,4 @@
 export { Client } from "./client.js";
 export { RunTree } from "./run_trees.js";
 // Update using yarn bump-version
-export const __version__ = "0.1.28";
+export const __version__ = "0.1.30";

package/dist/run_trees.cjs

@@ -39,6 +39,53 @@ function convertToDottedOrderFormat(epoch, runId, executionOrder = 1) {
     return (stripNonAlphanumeric(`${new Date(epoch).toISOString().slice(0, -1)}${paddedOrder}Z`) + runId);
 }
 exports.convertToDottedOrderFormat = convertToDottedOrderFormat;
+/**
+ * Baggage header information
+ */
+class Baggage {
+    constructor(metadata, tags) {
+        Object.defineProperty(this, "metadata", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "tags", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.metadata = metadata;
+        this.tags = tags;
+    }
+    static fromHeader(value) {
+        const items = value.split(",");
+        let metadata = {};
+        let tags = [];
+        for (const item of items) {
+            const [key, uriValue] = item.split("=");
+            const value = decodeURIComponent(uriValue);
+            if (key === "langsmith-metadata") {
+                metadata = JSON.parse(value);
+            }
+            else if (key === "langsmith-tags") {
+                tags = value.split(",");
+            }
+        }
+        return new Baggage(metadata, tags);
+    }
+    toHeader() {
+        const items = [];
+        if (this.metadata && Object.keys(this.metadata).length > 0) {
+            items.push(`langsmith-metadata=${encodeURIComponent(JSON.stringify(this.metadata))}`);
+        }
+        if (this.tags && this.tags.length > 0) {
+            items.push(`langsmith-tags=${encodeURIComponent(this.tags.join(","))}`);
+        }
+        return items.join(",");
+    }
+}
 class RunTree {
     constructor(originalConfig) {
         Object.defineProperty(this, "id", {
@@ -203,53 +250,6 @@ class RunTree {
             }
         }
     }
-    static fromRunnableConfig(parentConfig, props) {
-        // We only handle the callback manager case for now
-        const callbackManager = parentConfig?.callbacks;
-        let parentRun;
-        let projectName;
-        let client;
-        let tracingEnabled = (0, env_js_2.isTracingEnabled)();
-        if (callbackManager) {
-            const parentRunId = callbackManager?.getParentRunId?.() ?? "";
-            const langChainTracer = callbackManager?.handlers?.find((handler) => handler?.name == "langchain_tracer");
-            parentRun = langChainTracer?.getRun?.(parentRunId);
-            projectName = langChainTracer?.projectName;
-            client = langChainTracer?.client;
-            tracingEnabled = tracingEnabled || !!langChainTracer;
-        }
-        if (!parentRun) {
-            return new RunTree({
-                client,
-                tracingEnabled,
-                project_name: projectName,
-                name: props.name,
-                tags: props.tags,
-                metadata: props.metadata,
-            });
-        }
-        const parentRunTree = new RunTree({
-            name: parentRun.name,
-            id: parentRun.id,
-            client,
-            tracingEnabled,
-            project_name: projectName,
-            tags: [
-                ...new Set((parentRun?.tags ?? []).concat(parentConfig?.tags ?? [])),
-            ],
-            extra: {
-                metadata: {
-                    ...parentRun?.extra?.metadata,
-                    ...parentConfig?.metadata,
-                },
-            },
-        });
-        return parentRunTree.createChild({
-            name: props.name,
-            tags: props.tags,
-            metadata: props.metadata,
-        });
-    }
     static getDefaultConfig() {
         return {
             id: uuid.v4(),
@@ -367,6 +367,82 @@ class RunTree {
     toJSON() {
         return this._convertToCreate(this, undefined, false);
     }
+    static fromRunnableConfig(parentConfig, props) {
+        // We only handle the callback manager case for now
+        const callbackManager = parentConfig?.callbacks;
+        let parentRun;
+        let projectName;
+        let client;
+        let tracingEnabled = (0, env_js_2.isTracingEnabled)();
+        if (callbackManager) {
+            const parentRunId = callbackManager?.getParentRunId?.() ?? "";
+            const langChainTracer = callbackManager?.handlers?.find((handler) => handler?.name == "langchain_tracer");
+            parentRun = langChainTracer?.getRun?.(parentRunId);
+            projectName = langChainTracer?.projectName;
+            client = langChainTracer?.client;
+            tracingEnabled = tracingEnabled || !!langChainTracer;
+        }
+        if (!parentRun) {
+            return new RunTree({
+                ...props,
+                client,
+                tracingEnabled,
+                project_name: projectName,
+            });
+        }
+        const parentRunTree = new RunTree({
+            name: parentRun.name,
+            id: parentRun.id,
+            client,
+            tracingEnabled,
+            project_name: projectName,
+            tags: [
+                ...new Set((parentRun?.tags ?? []).concat(parentConfig?.tags ?? [])),
+            ],
+            extra: {
+                metadata: {
+                    ...parentRun?.extra?.metadata,
+                    ...parentConfig?.metadata,
+                },
+            },
+        });
+        return parentRunTree.createChild(props);
+    }
+    static fromDottedOrder(dottedOrder) {
+        return this.fromHeaders({ "langsmith-trace": dottedOrder });
+    }
+    static fromHeaders(headers, inheritArgs) {
+        const headerTrace = headers["langsmith-trace"];
+        if (!headerTrace || typeof headerTrace !== "string")
+            return undefined;
+        const parentDottedOrder = headerTrace.trim();
+        const parsedDottedOrder = parentDottedOrder.split(".").map((part) => {
+            const [strTime, uuid] = part.split("Z");
+            return { strTime, time: Date.parse(strTime + "Z"), uuid };
+        });
+        const traceId = parsedDottedOrder[0].uuid;
+        const config = {
+            ...inheritArgs,
+            name: inheritArgs?.["name"] ?? "parent",
+            run_type: inheritArgs?.["run_type"] ?? "chain",
+            start_time: inheritArgs?.["start_time"] ?? Date.now(),
+            id: parsedDottedOrder.at(-1)?.uuid,
+            trace_id: traceId,
+            dotted_order: parentDottedOrder,
+        };
+        if (headers["baggage"]) {
+            const baggage = Baggage.fromHeader(headers["baggage"]);
+            config.metadata = baggage.metadata;
+            config.tags = baggage.tags;
+        }
+        return new RunTree(config);
+    }
+    toHeaders() {
+        return {
+            "langsmith-trace": this.dotted_order,
+            baggage: new Baggage(this.extra?.metadata, this.tags).toHeader(),
+        };
+    }
 }
 exports.RunTree = RunTree;
 function isRunTree(x) {

package/dist/run_trees.d.ts

@@ -24,6 +24,8 @@ export interface RunTreeConfig {
     on_end?: (runTree: RunTree) => void;
     execution_order?: number;
     child_execution_order?: number;
+    trace_id?: string;
+    dotted_order?: string;
 }
 export interface RunnableConfigLike {
     /**
@@ -66,11 +68,6 @@ export declare class RunTree implements BaseRun {
     execution_order: number;
     child_execution_order: number;
     constructor(originalConfig: RunTreeConfig);
-    static fromRunnableConfig(parentConfig: RunnableConfigLike, props: {
-        name: string;
-        tags?: string[];
-        metadata?: KVMap;
-    }): RunTree;
     private static getDefaultConfig;
     createChild(config: RunTreeConfig): RunTree;
     end(outputs?: KVMap, error?: string, endTime?: number): Promise<void>;
@@ -78,6 +75,13 @@ export declare class RunTree implements BaseRun {
     postRun(excludeChildRuns?: boolean): Promise<void>;
     patchRun(): Promise<void>;
     toJSON(): RunCreate;
+    static fromRunnableConfig(parentConfig: RunnableConfigLike, props: RunTreeConfig): RunTree;
+    static fromDottedOrder(dottedOrder: string): RunTree | undefined;
+    static fromHeaders(headers: Record<string, string>, inheritArgs?: RunTreeConfig): RunTree | undefined;
+    toHeaders(): {
+        "langsmith-trace": string;
+        baggage: string;
+    };
 }
 export declare function isRunTree(x?: unknown): x is RunTree;
 export declare function isRunnableConfigLike(x?: unknown): x is RunnableConfigLike;

package/dist/run_trees.js

@@ -12,6 +12,53 @@ export function convertToDottedOrderFormat(epoch, runId, executionOrder = 1) {
     const paddedOrder = executionOrder.toFixed(0).slice(0, 3).padStart(3, "0");
     return (stripNonAlphanumeric(`${new Date(epoch).toISOString().slice(0, -1)}${paddedOrder}Z`) + runId);
 }
+/**
+ * Baggage header information
+ */
+class Baggage {
+    constructor(metadata, tags) {
+        Object.defineProperty(this, "metadata", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "tags", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.metadata = metadata;
+        this.tags = tags;
+    }
+    static fromHeader(value) {
+        const items = value.split(",");
+        let metadata = {};
+        let tags = [];
+        for (const item of items) {
+            const [key, uriValue] = item.split("=");
+            const value = decodeURIComponent(uriValue);
+            if (key === "langsmith-metadata") {
+                metadata = JSON.parse(value);
+            }
+            else if (key === "langsmith-tags") {
+                tags = value.split(",");
+            }
+        }
+        return new Baggage(metadata, tags);
+    }
+    toHeader() {
+        const items = [];
+        if (this.metadata && Object.keys(this.metadata).length > 0) {
+            items.push(`langsmith-metadata=${encodeURIComponent(JSON.stringify(this.metadata))}`);
+        }
+        if (this.tags && this.tags.length > 0) {
+            items.push(`langsmith-tags=${encodeURIComponent(this.tags.join(","))}`);
+        }
+        return items.join(",");
+    }
+}
 export class RunTree {
     constructor(originalConfig) {
         Object.defineProperty(this, "id", {
@@ -176,53 +223,6 @@ export class RunTree {
             }
         }
     }
-    static fromRunnableConfig(parentConfig, props) {
-        // We only handle the callback manager case for now
-        const callbackManager = parentConfig?.callbacks;
-        let parentRun;
-        let projectName;
-        let client;
-        let tracingEnabled = isTracingEnabled();
-        if (callbackManager) {
-            const parentRunId = callbackManager?.getParentRunId?.() ?? "";
-            const langChainTracer = callbackManager?.handlers?.find((handler) => handler?.name == "langchain_tracer");
-            parentRun = langChainTracer?.getRun?.(parentRunId);
-            projectName = langChainTracer?.projectName;
-            client = langChainTracer?.client;
-            tracingEnabled = tracingEnabled || !!langChainTracer;
-        }
-        if (!parentRun) {
-            return new RunTree({
-                client,
-                tracingEnabled,
-                project_name: projectName,
-                name: props.name,
-                tags: props.tags,
-                metadata: props.metadata,
-            });
-        }
-        const parentRunTree = new RunTree({
-            name: parentRun.name,
-            id: parentRun.id,
-            client,
-            tracingEnabled,
-            project_name: projectName,
-            tags: [
-                ...new Set((parentRun?.tags ?? []).concat(parentConfig?.tags ?? [])),
-            ],
-            extra: {
-                metadata: {
-                    ...parentRun?.extra?.metadata,
-                    ...parentConfig?.metadata,
-                },
-            },
-        });
-        return parentRunTree.createChild({
-            name: props.name,
-            tags: props.tags,
-            metadata: props.metadata,
-        });
-    }
     static getDefaultConfig() {
         return {
             id: uuid.v4(),
@@ -340,6 +340,82 @@ export class RunTree {
     toJSON() {
         return this._convertToCreate(this, undefined, false);
     }
+    static fromRunnableConfig(parentConfig, props) {
+        // We only handle the callback manager case for now
+        const callbackManager = parentConfig?.callbacks;
+        let parentRun;
+        let projectName;
+        let client;
+        let tracingEnabled = isTracingEnabled();
+        if (callbackManager) {
+            const parentRunId = callbackManager?.getParentRunId?.() ?? "";
+            const langChainTracer = callbackManager?.handlers?.find((handler) => handler?.name == "langchain_tracer");
+            parentRun = langChainTracer?.getRun?.(parentRunId);
+            projectName = langChainTracer?.projectName;
+            client = langChainTracer?.client;
+            tracingEnabled = tracingEnabled || !!langChainTracer;
+        }
+        if (!parentRun) {
+            return new RunTree({
+                ...props,
+                client,
+                tracingEnabled,
+                project_name: projectName,
+            });
+        }
+        const parentRunTree = new RunTree({
+            name: parentRun.name,
+            id: parentRun.id,
+            client,
+            tracingEnabled,
+            project_name: projectName,
+            tags: [
+                ...new Set((parentRun?.tags ?? []).concat(parentConfig?.tags ?? [])),
+            ],
+            extra: {
+                metadata: {
+                    ...parentRun?.extra?.metadata,
+                    ...parentConfig?.metadata,
+                },
+            },
+        });
+        return parentRunTree.createChild(props);
+    }
+    static fromDottedOrder(dottedOrder) {
+        return this.fromHeaders({ "langsmith-trace": dottedOrder });
+    }
+    static fromHeaders(headers, inheritArgs) {
+        const headerTrace = headers["langsmith-trace"];
+        if (!headerTrace || typeof headerTrace !== "string")
+            return undefined;
+        const parentDottedOrder = headerTrace.trim();
+        const parsedDottedOrder = parentDottedOrder.split(".").map((part) => {
+            const [strTime, uuid] = part.split("Z");
+            return { strTime, time: Date.parse(strTime + "Z"), uuid };
+        });
+        const traceId = parsedDottedOrder[0].uuid;
+        const config = {
+            ...inheritArgs,
+            name: inheritArgs?.["name"] ?? "parent",
+            run_type: inheritArgs?.["run_type"] ?? "chain",
+            start_time: inheritArgs?.["start_time"] ?? Date.now(),
+            id: parsedDottedOrder.at(-1)?.uuid,
+            trace_id: traceId,
+            dotted_order: parentDottedOrder,
+        };
+        if (headers["baggage"]) {
+            const baggage = Baggage.fromHeader(headers["baggage"]);
+            config.metadata = baggage.metadata;
+            config.tags = baggage.tags;
+        }
+        return new RunTree(config);
+    }
+    toHeaders() {
+        return {
+            "langsmith-trace": this.dotted_order,
+            baggage: new Baggage(this.extra?.metadata, this.tags).toHeader(),
+        };
+    }
 }
 export function isRunTree(x) {
     return (x !== undefined &&

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.1.28",
+  "version": "0.1.30",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [