langsmith 0.1.24 → 0.1.26

package/dist/client.cjs

@@ -907,8 +907,23 @@ class Client {
             select: select ? select : default_select,
             is_root: isRoot,
         };
+        let runsYielded = 0;
         for await (const runs of this._getCursorPaginatedList("/runs/query", body)) {
-            yield* runs;
+            if (limit) {
+                if (runsYielded >= limit) {
+                    break;
+                }
+                if (runs.length + runsYielded > limit) {
+                    const newRuns = runs.slice(0, limit - runsYielded);
+                    yield* newRuns;
+                    break;
+                }
+                runsYielded += runs.length;
+                yield* runs;
+            }
+            else {
+                yield* runs;
+            }
         }
     }
     async shareRun(runId, { shareId } = {}) {
@@ -1170,6 +1185,14 @@ class Client {
         const tenantId = await this._getTenantId();
         return `${this.getHostUrl()}/o/${tenantId}/projects/p/${project.id}`;
     }
+    async getDatasetUrl({ datasetId, datasetName, }) {
+        if (datasetId === undefined && datasetName === undefined) {
+            throw new Error("Must provide either datasetName or datasetId");
+        }
+        const dataset = await this.readDataset({ datasetId, datasetName });
+        const tenantId = await this._getTenantId();
+        return `${this.getHostUrl()}/o/${tenantId}/datasets/${dataset.id}`;
+    }
     async _getTenantId() {
         if (this._tenantId !== null) {
             return this._tenantId;
@@ -1428,7 +1451,7 @@ class Client {
         }
         await response.json();
     }
-    async createExample(inputs, outputs, { datasetId, datasetName, createdAt, exampleId, metadata, }) {
+    async createExample(inputs, outputs, { datasetId, datasetName, createdAt, exampleId, metadata, split, }) {
         let datasetId_ = datasetId;
         if (datasetId_ === undefined && datasetName === undefined) {
             throw new Error("Must provide either datasetName or datasetId");
@@ -1448,6 +1471,7 @@ class Client {
             created_at: createdAt_?.toISOString(),
             id: exampleId,
             metadata,
+            split,
         };
         const response = await this.caller.call(fetch, `${this.apiUrl}/examples`, {
             method: "POST",
@@ -1481,6 +1505,7 @@ class Client {
                 inputs: input,
                 outputs: outputs ? outputs[idx] : undefined,
                 metadata: metadata ? metadata[idx] : undefined,
+                split: props.splits ? props.splits[idx] : undefined,
                 id: exampleIds ? exampleIds[idx] : undefined,
                 source_run_id: sourceRunIds ? sourceRunIds[idx] : undefined,
             };
@@ -1518,7 +1543,7 @@ class Client {
         const path = `/examples/${exampleId}`;
         return await this._get(path);
     }
-    async *listExamples({ datasetId, datasetName, exampleIds, asOf, inlineS3Urls, metadata, } = {}) {
+    async *listExamples({ datasetId, datasetName, exampleIds, asOf, splits, inlineS3Urls, metadata, } = {}) {
         let datasetId_;
         if (datasetId !== undefined && datasetName !== undefined) {
             throw new Error("Must provide either datasetName or datasetId, not both");
@@ -1549,6 +1574,11 @@ class Client {
                 params.append("id", id_);
             }
         }
+        if (splits !== undefined) {
+            for (const split of splits) {
+                params.append("splits", split);
+            }
+        }
         if (metadata !== undefined) {
             const serializedMetadata = JSON.stringify(metadata);
             params.append("metadata", serializedMetadata);

package/dist/client.d.ts

@@ -145,6 +145,7 @@ export type CreateExampleOptions = {
     createdAt?: Date;
     exampleId?: string;
     metadata?: KVMap;
+    split?: string | string[];
 };
 export declare class Queue<T> {
     items: [T, () => void][];
@@ -338,6 +339,10 @@ export declare class Client {
         projectId?: string;
         projectName?: string;
     }): Promise<string>;
+    getDatasetUrl({ datasetId, datasetName, }: {
+        datasetId?: string;
+        datasetName?: string;
+    }): Promise<string>;
     private _getTenantId;
     listProjects({ projectIds, name, nameContains, referenceDatasetId, referenceDatasetName, referenceFree, }?: {
         projectIds?: string[];
@@ -385,11 +390,12 @@ export declare class Client {
         datasetId?: string;
         datasetName?: string;
     }): Promise<void>;
-    createExample(inputs: KVMap, outputs: KVMap, { datasetId, datasetName, createdAt, exampleId, metadata, }: CreateExampleOptions): Promise<Example>;
+    createExample(inputs: KVMap, outputs: KVMap, { datasetId, datasetName, createdAt, exampleId, metadata, split, }: CreateExampleOptions): Promise<Example>;
     createExamples(props: {
         inputs: Array<KVMap>;
         outputs?: Array<KVMap>;
         metadata?: Array<KVMap>;
+        splits?: Array<string | Array<string>>;
         sourceRunIds?: Array<string>;
         exampleIds?: Array<string>;
         datasetId?: string;
@@ -398,11 +404,12 @@ export declare class Client {
     createLLMExample(input: string, generation: string | undefined, options: CreateExampleOptions): Promise<Example>;
     createChatExample(input: KVMap[] | LangChainBaseMessage[], generations: KVMap | LangChainBaseMessage | undefined, options: CreateExampleOptions): Promise<Example>;
     readExample(exampleId: string): Promise<Example>;
-    listExamples({ datasetId, datasetName, exampleIds, asOf, inlineS3Urls, metadata, }?: {
+    listExamples({ datasetId, datasetName, exampleIds, asOf, splits, inlineS3Urls, metadata, }?: {
         datasetId?: string;
         datasetName?: string;
         exampleIds?: string[];
         asOf?: string | Date;
+        splits?: string[];
         inlineS3Urls?: boolean;
         metadata?: KVMap;
     }): AsyncIterable<Example>;

package/dist/client.js

@@ -880,8 +880,23 @@ export class Client {
             select: select ? select : default_select,
             is_root: isRoot,
         };
+        let runsYielded = 0;
         for await (const runs of this._getCursorPaginatedList("/runs/query", body)) {
-            yield* runs;
+            if (limit) {
+                if (runsYielded >= limit) {
+                    break;
+                }
+                if (runs.length + runsYielded > limit) {
+                    const newRuns = runs.slice(0, limit - runsYielded);
+                    yield* newRuns;
+                    break;
+                }
+                runsYielded += runs.length;
+                yield* runs;
+            }
+            else {
+                yield* runs;
+            }
         }
     }
     async shareRun(runId, { shareId } = {}) {
@@ -1143,6 +1158,14 @@ export class Client {
         const tenantId = await this._getTenantId();
         return `${this.getHostUrl()}/o/${tenantId}/projects/p/${project.id}`;
     }
+    async getDatasetUrl({ datasetId, datasetName, }) {
+        if (datasetId === undefined && datasetName === undefined) {
+            throw new Error("Must provide either datasetName or datasetId");
+        }
+        const dataset = await this.readDataset({ datasetId, datasetName });
+        const tenantId = await this._getTenantId();
+        return `${this.getHostUrl()}/o/${tenantId}/datasets/${dataset.id}`;
+    }
     async _getTenantId() {
         if (this._tenantId !== null) {
             return this._tenantId;
@@ -1401,7 +1424,7 @@ export class Client {
         }
         await response.json();
     }
-    async createExample(inputs, outputs, { datasetId, datasetName, createdAt, exampleId, metadata, }) {
+    async createExample(inputs, outputs, { datasetId, datasetName, createdAt, exampleId, metadata, split, }) {
         let datasetId_ = datasetId;
         if (datasetId_ === undefined && datasetName === undefined) {
             throw new Error("Must provide either datasetName or datasetId");
@@ -1421,6 +1444,7 @@ export class Client {
             created_at: createdAt_?.toISOString(),
             id: exampleId,
             metadata,
+            split,
         };
         const response = await this.caller.call(fetch, `${this.apiUrl}/examples`, {
             method: "POST",
@@ -1454,6 +1478,7 @@ export class Client {
                 inputs: input,
                 outputs: outputs ? outputs[idx] : undefined,
                 metadata: metadata ? metadata[idx] : undefined,
+                split: props.splits ? props.splits[idx] : undefined,
                 id: exampleIds ? exampleIds[idx] : undefined,
                 source_run_id: sourceRunIds ? sourceRunIds[idx] : undefined,
             };
@@ -1491,7 +1516,7 @@ export class Client {
         const path = `/examples/${exampleId}`;
         return await this._get(path);
     }
-    async *listExamples({ datasetId, datasetName, exampleIds, asOf, inlineS3Urls, metadata, } = {}) {
+    async *listExamples({ datasetId, datasetName, exampleIds, asOf, splits, inlineS3Urls, metadata, } = {}) {
         let datasetId_;
         if (datasetId !== undefined && datasetName !== undefined) {
             throw new Error("Must provide either datasetName or datasetId, not both");
@@ -1522,6 +1547,11 @@ export class Client {
                 params.append("id", id_);
             }
         }
+        if (splits !== undefined) {
+            for (const split of splits) {
+                params.append("splits", split);
+            }
+        }
         if (metadata !== undefined) {
             const serializedMetadata = JSON.stringify(metadata);
             params.append("metadata", serializedMetadata);

package/dist/evaluation/_runner.cjs

@@ -8,6 +8,7 @@ const _uuid_js_1 = require("../utils/_uuid.cjs");
 const async_caller_js_1 = require("../utils/async_caller.cjs");
 const atee_js_1 = require("../utils/atee.cjs");
 const env_js_1 = require("../utils/env.cjs");
+const error_js_1 = require("../utils/error.cjs");
 const _random_name_js_1 = require("./_random_name.cjs");
 const evaluator_js_1 = require("./evaluator.cjs");
 const uuid_1 = require("uuid");
@@ -238,15 +239,21 @@ class _ExperimentManager {
         }
         return project;
     }
-    _printExperimentStart() {
-        // @TODO log with experiment URL
+    async _printExperimentStart() {
         console.log(`Starting evaluation of experiment: ${this.experimentName}`);
+        const firstExample = this._examples?.[0];
+        const datasetId = firstExample?.dataset_id;
+        if (!datasetId || !this._experiment)
+            return;
+        const datasetUrl = await this.client.getDatasetUrl({ datasetId });
+        const compareUrl = `${datasetUrl}/compare?selectedSessions=${this._experiment.id}`;
+        console.log(`View results at ${compareUrl}`);
     }
     async start() {
         const examples = await this.getExamples();
         const firstExample = examples[0];
         const project = await this._getProject(firstExample);
-        this._printExperimentStart();
+        await this._printExperimentStart();
         return new _ExperimentManager({
             examples,
             experiment: project,
@@ -377,7 +384,7 @@ class _ExperimentManager {
             try {
                 const options = {
                     reference_example_id: example.id,
-                    project_name: fields.experimentName,
+                    project_name: "evaluators",
                     metadata: {
                         example_version: example.modified_at
                             ? new Date(example.modified_at).toISOString()
@@ -390,6 +397,7 @@ class _ExperimentManager {
             }
             catch (e) {
                 console.error(`Error running evaluator ${evaluator.evaluateRun.name} on run ${run.id}: ${e}`);
+                (0, error_js_1.printErrorStackTrace)(e);
             }
         }
         return {
@@ -410,7 +418,6 @@ class _ExperimentManager {
         if (maxConcurrency === 0) {
             for await (const currentResults of this.getResults()) {
                 yield this._runEvaluators(evaluators, currentResults, {
-                    experimentName: this.experimentName,
                     client: this.client,
                 });
             }
@@ -422,7 +429,6 @@ class _ExperimentManager {
             const futures = [];
             for await (const currentResults of this.getResults()) {
                 futures.push(caller.call(this._runEvaluators, evaluators, currentResults, {
-                    experimentName: this.experimentName,
                     client: this.client,
                 }));
             }
@@ -460,6 +466,7 @@ class _ExperimentManager {
                 }
                 catch (e) {
                     console.error(`Error running summary evaluator ${evaluator.name}: ${JSON.stringify(e, null, 2)}`);
+                    (0, error_js_1.printErrorStackTrace)(e);
                 }
             }
             yield {
@@ -495,6 +502,21 @@ class _ExperimentManager {
             return undefined;
         return modifiedAtTime.reduce((max, current) => (current.time > max.time ? current : max), modifiedAtTime[0]).date;
     }
+    async _getDatasetSplits() {
+        const examples = await this.getExamples();
+        const allSplits = examples.reduce((acc, ex) => {
+            if (ex.metadata && ex.metadata.dataset_split) {
+                if (Array.isArray(ex.metadata.dataset_split)) {
+                    ex.metadata.dataset_split.forEach((split) => acc.add(split));
+                }
+                else if (typeof ex.metadata.dataset_split === "string") {
+                    acc.add(ex.metadata.dataset_split);
+                }
+            }
+            return acc;
+        }, new Set());
+        return allSplits.size ? Array.from(allSplits) : undefined;
+    }
     async _end() {
         const experiment = this._experiment;
         if (!experiment) {
@@ -502,6 +524,7 @@ class _ExperimentManager {
         }
         const projectMetadata = await this._getExperimentMetadata();
         projectMetadata["dataset_version"] = await this._getDatasetVersion();
+        projectMetadata["dataset_splits"] = await this._getDatasetSplits();
         // Update revision_id if not already set
         if (!projectMetadata["revision_id"]) {
             projectMetadata["revision_id"] = await (0, _git_js_1.getDefaultRevisionId)();
@@ -633,6 +656,7 @@ async function _forward(fn, example, experimentName, metadata, client) {
     }
     catch (e) {
         console.error(`Error running target function: ${e}`);
+        (0, error_js_1.printErrorStackTrace)(e);
     }
     if (!run) {
         throw new Error(`Run not created by target function.

package/dist/evaluation/_runner.d.ts

@@ -104,7 +104,7 @@ declare class _ExperimentManager {
     _getExperiment(): TracerSession;
     _getExperimentMetadata(): Promise<KVMap>;
     _getProject(firstExample: Example): Promise<TracerSession>;
-    _printExperimentStart(): void;
+    protected _printExperimentStart(): Promise<void>;
     start(): Promise<_ExperimentManager>;
     withPredictions(target: TargetNoInvoke, options?: {
         maxConcurrency?: number;
@@ -125,7 +125,6 @@ declare class _ExperimentManager {
         maxConcurrency?: number;
     }): AsyncGenerator<_ForwardResults>;
     _runEvaluators(evaluators: Array<RunEvaluator>, currentResults: ExperimentResultRow, fields: {
-        experimentName: string;
         client: Client;
     }): Promise<ExperimentResultRow>;
     /**
@@ -140,6 +139,7 @@ declare class _ExperimentManager {
     }): AsyncGenerator<ExperimentResultRow>;
     _applySummaryEvaluators(summaryEvaluators: Array<SummaryEvaluatorT>): AsyncGenerator<(runsArray: Run[]) => AsyncGenerator<EvaluationResults>>;
     _getDatasetVersion(): Promise<string | undefined>;
+    _getDatasetSplits(): Promise<string[] | undefined>;
     _end(): Promise<void>;
 }
 /**

package/dist/evaluation/_runner.js

@@ -5,6 +5,7 @@ import { assertUuid } from "../utils/_uuid.js";
 import { AsyncCaller } from "../utils/async_caller.js";
 import { atee } from "../utils/atee.js";
 import { getLangChainEnvVarsMetadata } from "../utils/env.js";
+import { printErrorStackTrace } from "../utils/error.js";
 import { randomName } from "./_random_name.js";
 import { runEvaluator, } from "./evaluator.js";
 import { v4 as uuidv4 } from "uuid";
@@ -234,15 +235,21 @@ class _ExperimentManager {
         }
         return project;
     }
-    _printExperimentStart() {
-        // @TODO log with experiment URL
+    async _printExperimentStart() {
         console.log(`Starting evaluation of experiment: ${this.experimentName}`);
+        const firstExample = this._examples?.[0];
+        const datasetId = firstExample?.dataset_id;
+        if (!datasetId || !this._experiment)
+            return;
+        const datasetUrl = await this.client.getDatasetUrl({ datasetId });
+        const compareUrl = `${datasetUrl}/compare?selectedSessions=${this._experiment.id}`;
+        console.log(`View results at ${compareUrl}`);
     }
     async start() {
         const examples = await this.getExamples();
         const firstExample = examples[0];
         const project = await this._getProject(firstExample);
-        this._printExperimentStart();
+        await this._printExperimentStart();
         return new _ExperimentManager({
             examples,
             experiment: project,
@@ -373,7 +380,7 @@ class _ExperimentManager {
             try {
                 const options = {
                     reference_example_id: example.id,
-                    project_name: fields.experimentName,
+                    project_name: "evaluators",
                     metadata: {
                         example_version: example.modified_at
                             ? new Date(example.modified_at).toISOString()
@@ -386,6 +393,7 @@ class _ExperimentManager {
             }
             catch (e) {
                 console.error(`Error running evaluator ${evaluator.evaluateRun.name} on run ${run.id}: ${e}`);
+                printErrorStackTrace(e);
             }
         }
         return {
@@ -406,7 +414,6 @@ class _ExperimentManager {
         if (maxConcurrency === 0) {
             for await (const currentResults of this.getResults()) {
                 yield this._runEvaluators(evaluators, currentResults, {
-                    experimentName: this.experimentName,
                     client: this.client,
                 });
             }
@@ -418,7 +425,6 @@ class _ExperimentManager {
             const futures = [];
             for await (const currentResults of this.getResults()) {
                 futures.push(caller.call(this._runEvaluators, evaluators, currentResults, {
-                    experimentName: this.experimentName,
                     client: this.client,
                 }));
             }
@@ -456,6 +462,7 @@ class _ExperimentManager {
                 }
                 catch (e) {
                     console.error(`Error running summary evaluator ${evaluator.name}: ${JSON.stringify(e, null, 2)}`);
+                    printErrorStackTrace(e);
                 }
             }
             yield {
@@ -491,6 +498,21 @@ class _ExperimentManager {
             return undefined;
         return modifiedAtTime.reduce((max, current) => (current.time > max.time ? current : max), modifiedAtTime[0]).date;
     }
+    async _getDatasetSplits() {
+        const examples = await this.getExamples();
+        const allSplits = examples.reduce((acc, ex) => {
+            if (ex.metadata && ex.metadata.dataset_split) {
+                if (Array.isArray(ex.metadata.dataset_split)) {
+                    ex.metadata.dataset_split.forEach((split) => acc.add(split));
+                }
+                else if (typeof ex.metadata.dataset_split === "string") {
+                    acc.add(ex.metadata.dataset_split);
+                }
+            }
+            return acc;
+        }, new Set());
+        return allSplits.size ? Array.from(allSplits) : undefined;
+    }
     async _end() {
         const experiment = this._experiment;
         if (!experiment) {
@@ -498,6 +520,7 @@ class _ExperimentManager {
         }
         const projectMetadata = await this._getExperimentMetadata();
         projectMetadata["dataset_version"] = await this._getDatasetVersion();
+        projectMetadata["dataset_splits"] = await this._getDatasetSplits();
         // Update revision_id if not already set
         if (!projectMetadata["revision_id"]) {
             projectMetadata["revision_id"] = await getDefaultRevisionId();
@@ -629,6 +652,7 @@ async function _forward(fn, example, experimentName, metadata, client) {
     }
     catch (e) {
         console.error(`Error running target function: ${e}`);
+        printErrorStackTrace(e);
     }
     if (!run) {
         throw new Error(`Run not created by target function.

package/dist/evaluation/evaluator.cjs

@@ -59,7 +59,12 @@ class DynamicRunEvaluator {
         if (typeof this.func !== "function") {
             throw new Error("Target must be runnable function");
         }
-        const wrappedTraceableFunc = (0, traceable_js_1.traceable)(this.func, { project_name: "evaluators", name: "evaluator", ...options });
+        const wrappedTraceableFunc = (0, traceable_js_1.traceable)(this.func, {
+            project_name: "evaluators",
+            name: "evaluator",
+            id: sourceRunId,
+            ...options,
+        });
         const result = (await wrappedTraceableFunc(
         // Pass data via `langSmithRunAndExample` key to avoid conflicts with other
         // inputs. This key is extracted in the wrapped function, with `run` and

package/dist/evaluation/evaluator.js

@@ -56,7 +56,12 @@ export class DynamicRunEvaluator {
         if (typeof this.func !== "function") {
             throw new Error("Target must be runnable function");
         }
-        const wrappedTraceableFunc = traceable(this.func, { project_name: "evaluators", name: "evaluator", ...options });
+        const wrappedTraceableFunc = traceable(this.func, {
+            project_name: "evaluators",
+            name: "evaluator",
+            id: sourceRunId,
+            ...options,
+        });
         const result = (await wrappedTraceableFunc(
         // Pass data via `langSmithRunAndExample` key to avoid conflicts with other
         // inputs. This key is extracted in the wrapped function, with `run` and

package/dist/index.cjs

@@ -6,4 +6,4 @@ Object.defineProperty(exports, "Client", { enumerable: true, get: function () {
 var run_trees_js_1 = require("./run_trees.cjs");
 Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () { return run_trees_js_1.RunTree; } });
 // Update using yarn bump-version
-exports.__version__ = "0.1.24";
+exports.__version__ = "0.1.26";

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 export { Client } from "./client.js";
-export type { Dataset, Example, TracerSession, Run, Feedback, } from "./schemas.js";
+export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
-export declare const __version__ = "0.1.24";
+export declare const __version__ = "0.1.26";

package/dist/index.js

@@ -1,4 +1,4 @@
 export { Client } from "./client.js";
 export { RunTree } from "./run_trees.js";
 // Update using yarn bump-version
-export const __version__ = "0.1.24";
+export const __version__ = "0.1.26";

package/dist/schemas.d.ts

@@ -162,6 +162,7 @@ export interface RunUpdate {
 export interface ExampleCreate extends BaseExample {
     id?: string;
     created_at?: string;
+    split?: string | string[];
 }
 export interface Example extends BaseExample {
     id: string;
@@ -175,6 +176,7 @@ export interface ExampleUpdate {
     inputs?: KVMap;
     outputs?: KVMap;
     metadata?: KVMap;
+    split?: string | string[];
 }
 export interface BaseDataset {
     name: string;
@@ -293,4 +295,22 @@ export interface ComparativeExperiment {
     experiments_info?: Array<Record<string, unknown>>;
     feedback_stats?: Record<string, unknown>;
 }
+/**
+ * Represents the expected output schema returned by traceable
+ * or by run tree output for LangSmith to correctly display
+ * documents in the UI
+ */
+export type RetrieverOutput = Array<{
+    page_content: string;
+    type: "Document";
+    metadata?: KVMap;
+}>;
+export interface InvocationParamsSchema {
+    ls_provider?: string;
+    ls_model_name?: string;
+    ls_model_type: "chat";
+    ls_temperature?: number;
+    ls_max_tokens?: number;
+    ls_stop?: string[];
+}
 export {};

package/dist/traceable.cjs

@@ -63,12 +63,20 @@ const handleRunOutputs = (rawOutputs) => {
     }
     return { outputs: rawOutputs };
 };
-const getTracingRunTree = (runTree, inputs) => {
+const getTracingRunTree = (runTree, inputs, getInvocationParams) => {
     const tracingEnabled_ = tracingIsEnabled(runTree.tracingEnabled);
     if (!tracingEnabled_) {
         return undefined;
     }
     runTree.inputs = handleRunInputs(inputs);
+    const invocationParams = getInvocationParams?.(...inputs);
+    if (invocationParams != null) {
+        runTree.extra ??= {};
+        runTree.extra.metadata = {
+            ...runTree.extra.metadata,
+            ...invocationParams,
+        };
+    }
     return runTree;
 };
 // idea: store the state of the promise outside
@@ -279,7 +287,7 @@ function traceable(wrappedFunc, config) {
             // used for handoff between LangChain.JS and traceable functions
             if ((0, run_trees_js_1.isRunnableConfigLike)(firstArg)) {
                 return [
-                    getTracingRunTree(run_trees_js_1.RunTree.fromRunnableConfig(firstArg, ensuredConfig), restArgs),
+                    getTracingRunTree(run_trees_js_1.RunTree.fromRunnableConfig(firstArg, ensuredConfig), restArgs, config?.getInvocationParams),
                     restArgs,
                 ];
             }
@@ -295,7 +303,7 @@ function traceable(wrappedFunc, config) {
             if (firstArg === exports.ROOT || (0, run_trees_js_1.isRunTree)(firstArg)) {
                 const currentRunTree = getTracingRunTree(firstArg === exports.ROOT
                     ? new run_trees_js_1.RunTree(ensuredConfig)
-                    : firstArg.createChild(ensuredConfig), restArgs);
+                    : firstArg.createChild(ensuredConfig), restArgs, config?.getInvocationParams);
                 return [currentRunTree, [currentRunTree, ...restArgs]];
             }
             // Node.JS uses AsyncLocalStorage (ALS) and AsyncResource
@@ -303,11 +311,11 @@ function traceable(wrappedFunc, config) {
             const prevRunFromStore = asyncLocalStorage.getStore();
             if (prevRunFromStore) {
                 return [
-                    getTracingRunTree(prevRunFromStore.createChild(ensuredConfig), processedArgs),
+                    getTracingRunTree(prevRunFromStore.createChild(ensuredConfig), processedArgs, config?.getInvocationParams),
                     processedArgs,
                 ];
             }
-            const currentRunTree = getTracingRunTree(new run_trees_js_1.RunTree(ensuredConfig), processedArgs);
+            const currentRunTree = getTracingRunTree(new run_trees_js_1.RunTree(ensuredConfig), processedArgs, config?.getInvocationParams);
             return [currentRunTree, processedArgs];
         })();
         return asyncLocalStorage.run(currentRunTree, () => {

package/dist/traceable.d.ts

@@ -1,4 +1,5 @@
 import { RunTree, RunTreeConfig, RunnableConfigLike } from "./run_trees.js";
+import { InvocationParamsSchema } from "./schemas.js";
 export declare const ROOT: unique symbol;
 export type RunTreeLike = RunTree;
 type SmartPromise<T> = T extends AsyncGenerator ? T : T extends Promise<unknown> ? T : Promise<T>;
@@ -52,6 +53,15 @@ export type TraceableFunction<Func extends (...args: any[]) => any> = Func exten
 export declare function traceable<Func extends (...args: any[]) => any>(wrappedFunc: Func, config?: Partial<RunTreeConfig> & {
     aggregator?: (args: any[]) => any;
     argsConfigPath?: [number] | [number, string];
+    /**
+     * Extract invocation parameters from the arguments of the traced function.
+     * This is useful for LangSmith to properly track common metadata like
+     * provider, model name and temperature.
+     *
+     * @param args Arguments of the traced function
+     * @returns Key-value map of the invocation parameters, which will be merged with the existing metadata
+     */
+    getInvocationParams?: (...args: Parameters<Func>) => InvocationParamsSchema | undefined;
 }): TraceableFunction<Func>;
 /**
  * Return the current run tree from within a traceable-wrapped function.

package/dist/traceable.js

@@ -60,12 +60,20 @@ const handleRunOutputs = (rawOutputs) => {
     }
     return { outputs: rawOutputs };
 };
-const getTracingRunTree = (runTree, inputs) => {
+const getTracingRunTree = (runTree, inputs, getInvocationParams) => {
     const tracingEnabled_ = tracingIsEnabled(runTree.tracingEnabled);
     if (!tracingEnabled_) {
         return undefined;
     }
     runTree.inputs = handleRunInputs(inputs);
+    const invocationParams = getInvocationParams?.(...inputs);
+    if (invocationParams != null) {
+        runTree.extra ??= {};
+        runTree.extra.metadata = {
+            ...runTree.extra.metadata,
+            ...invocationParams,
+        };
+    }
     return runTree;
 };
 // idea: store the state of the promise outside
@@ -276,7 +284,7 @@ export function traceable(wrappedFunc, config) {
             // used for handoff between LangChain.JS and traceable functions
             if (isRunnableConfigLike(firstArg)) {
                 return [
-                    getTracingRunTree(RunTree.fromRunnableConfig(firstArg, ensuredConfig), restArgs),
+                    getTracingRunTree(RunTree.fromRunnableConfig(firstArg, ensuredConfig), restArgs, config?.getInvocationParams),
                     restArgs,
                 ];
             }
@@ -292,7 +300,7 @@ export function traceable(wrappedFunc, config) {
             if (firstArg === ROOT || isRunTree(firstArg)) {
                 const currentRunTree = getTracingRunTree(firstArg === ROOT
                     ? new RunTree(ensuredConfig)
-                    : firstArg.createChild(ensuredConfig), restArgs);
+                    : firstArg.createChild(ensuredConfig), restArgs, config?.getInvocationParams);
                 return [currentRunTree, [currentRunTree, ...restArgs]];
             }
             // Node.JS uses AsyncLocalStorage (ALS) and AsyncResource
@@ -300,11 +308,11 @@ export function traceable(wrappedFunc, config) {
             const prevRunFromStore = asyncLocalStorage.getStore();
             if (prevRunFromStore) {
                 return [
-                    getTracingRunTree(prevRunFromStore.createChild(ensuredConfig), processedArgs),
+                    getTracingRunTree(prevRunFromStore.createChild(ensuredConfig), processedArgs, config?.getInvocationParams),
                     processedArgs,
                 ];
             }
-            const currentRunTree = getTracingRunTree(new RunTree(ensuredConfig), processedArgs);
+            const currentRunTree = getTracingRunTree(new RunTree(ensuredConfig), processedArgs, config?.getInvocationParams);
             return [currentRunTree, processedArgs];
         })();
         return asyncLocalStorage.run(currentRunTree, () => {

package/dist/utils/error.cjs

@@ -0,0 +1,25 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.printErrorStackTrace = void 0;
+function getErrorStackTrace(e) {
+    if (typeof e !== "object" || e == null)
+        return undefined;
+    if (!("stack" in e) || typeof e.stack !== "string")
+        return undefined;
+    let stack = e.stack;
+    const prevLine = `${e}`;
+    if (stack.startsWith(prevLine)) {
+        stack = stack.slice(prevLine.length);
+    }
+    if (stack.startsWith("\n")) {
+        stack = stack.slice(1);
+    }
+    return stack;
+}
+function printErrorStackTrace(e) {
+    const stack = getErrorStackTrace(e);
+    if (stack == null)
+        return;
+    console.error(stack);
+}
+exports.printErrorStackTrace = printErrorStackTrace;

package/dist/utils/error.d.ts

@@ -0,0 +1 @@
+export declare function printErrorStackTrace(e: unknown): void;

package/dist/utils/error.js

@@ -0,0 +1,21 @@
+function getErrorStackTrace(e) {
+    if (typeof e !== "object" || e == null)
+        return undefined;
+    if (!("stack" in e) || typeof e.stack !== "string")
+        return undefined;
+    let stack = e.stack;
+    const prevLine = `${e}`;
+    if (stack.startsWith(prevLine)) {
+        stack = stack.slice(prevLine.length);
+    }
+    if (stack.startsWith("\n")) {
+        stack = stack.slice(1);
+    }
+    return stack;
+}
+export function printErrorStackTrace(e) {
+    const stack = getErrorStackTrace(e);
+    if (stack == null)
+        return;
+    console.error(stack);
+}

package/dist/wrappers/openai.cjs

@@ -147,6 +147,22 @@ const wrapOpenAI = (openai, options) => {
         run_type: "llm",
         aggregator: chatAggregator,
         argsConfigPath: [1, "langsmithExtra"],
+        getInvocationParams: (payload) => {
+            if (typeof payload !== "object" || payload == null)
+                return undefined;
+            // we can safely do so, as the types are not exported in TSC
+            const params = payload;
+            const ls_stop = (typeof params.stop === "string" ? [params.stop] : params.stop) ??
+                undefined;
+            return {
+                ls_provider: "openai",
+                ls_model_type: "chat",
+                ls_model_name: params.model,
+                ls_max_tokens: params.max_tokens ?? undefined,
+                ls_temperature: params.temperature ?? undefined,
+                ls_stop,
+            };
+        },
         ...options,
     });
     openai.completions.create = (0, traceable_js_1.traceable)(openai.completions.create.bind(openai.completions), {
@@ -154,6 +170,22 @@ const wrapOpenAI = (openai, options) => {
         run_type: "llm",
         aggregator: textAggregator,
         argsConfigPath: [1, "langsmithExtra"],
+        getInvocationParams: (payload) => {
+            if (typeof payload !== "object" || payload == null)
+                return undefined;
+            // we can safely do so, as the types are not exported in TSC
+            const params = payload;
+            const ls_stop = (typeof params.stop === "string" ? [params.stop] : params.stop) ??
+                undefined;
+            return {
+                ls_provider: "openai",
+                ls_model_type: "chat",
+                ls_model_name: params.model,
+                ls_max_tokens: params.max_tokens ?? undefined,
+                ls_temperature: params.temperature ?? undefined,
+                ls_stop,
+            };
+        },
         ...options,
     });
     return openai;

package/dist/wrappers/openai.js

@@ -144,6 +144,22 @@ export const wrapOpenAI = (openai, options) => {
         run_type: "llm",
         aggregator: chatAggregator,
         argsConfigPath: [1, "langsmithExtra"],
+        getInvocationParams: (payload) => {
+            if (typeof payload !== "object" || payload == null)
+                return undefined;
+            // we can safely do so, as the types are not exported in TSC
+            const params = payload;
+            const ls_stop = (typeof params.stop === "string" ? [params.stop] : params.stop) ??
+                undefined;
+            return {
+                ls_provider: "openai",
+                ls_model_type: "chat",
+                ls_model_name: params.model,
+                ls_max_tokens: params.max_tokens ?? undefined,
+                ls_temperature: params.temperature ?? undefined,
+                ls_stop,
+            };
+        },
         ...options,
     });
     openai.completions.create = traceable(openai.completions.create.bind(openai.completions), {
@@ -151,6 +167,22 @@ export const wrapOpenAI = (openai, options) => {
         run_type: "llm",
         aggregator: textAggregator,
         argsConfigPath: [1, "langsmithExtra"],
+        getInvocationParams: (payload) => {
+            if (typeof payload !== "object" || payload == null)
+                return undefined;
+            // we can safely do so, as the types are not exported in TSC
+            const params = payload;
+            const ls_stop = (typeof params.stop === "string" ? [params.stop] : params.stop) ??
+                undefined;
+            return {
+                ls_provider: "openai",
+                ls_model_type: "chat",
+                ls_model_name: params.model,
+                ls_max_tokens: params.max_tokens ?? undefined,
+                ls_temperature: params.temperature ?? undefined,
+                ls_stop,
+            };
+        },
         ...options,
     });
     return openai;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.1.24",
+  "version": "0.1.26",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [
@@ -85,6 +85,7 @@
   },
   "devDependencies": {
     "@babel/preset-env": "^7.22.4",
+    "@faker-js/faker": "^8.4.1",
     "@jest/globals": "^29.5.0",
     "@langchain/core": "^0.1.32",
     "@langchain/langgraph": "^0.0.8",
@@ -196,4 +197,4 @@
     },
     "./package.json": "./package.json"
   }
-}
+}