langsmith 0.5.3 → 0.5.4

package/dist/client.cjs

@@ -3229,6 +3229,115 @@ class Client {
         const [results] = await this._logEvaluationFeedback(evaluatorResponse, run, sourceInfo);
         return results;
     }
+    /**
+     * API for managing feedback configs
+     */
+    /**
+     * Create a feedback configuration on the LangSmith API.
+     *
+     * This upserts: if an identical config already exists, it returns it.
+     * If a conflicting config exists for the same key, a 400 error is raised.
+     *
+     * @param options - The options for creating a feedback config
+     * @param options.feedbackKey - The unique key for this feedback config
+     * @param options.feedbackConfig - The config specifying type, bounds, and categories
+     * @param options.isLowerScoreBetter - Whether a lower score is better
+     * @returns The created FeedbackConfigSchema object
+     */
+    async createFeedbackConfig(options) {
+        const { feedbackKey, feedbackConfig, isLowerScoreBetter = false } = options;
+        const body = {
+            feedback_key: feedbackKey,
+            feedback_config: feedbackConfig,
+            is_lower_score_better: isLowerScoreBetter,
+        };
+        const response = await this.caller.call(async () => {
+            const res = await this._fetch(`${this.apiUrl}/feedback-configs`, {
+                method: "POST",
+                headers: { ...this.headers, "Content-Type": "application/json" },
+                signal: AbortSignal.timeout(this.timeout_ms),
+                ...this.fetchOptions,
+                body: JSON.stringify(body),
+            });
+            await (0, error_js_1.raiseForStatus)(res, "create feedback config");
+            return res;
+        });
+        return response.json();
+    }
+    /**
+     * List feedback configurations on the LangSmith API.
+     * @param options - The options for listing feedback configs
+     * @param options.feedbackKeys - Filter by specific feedback keys
+     * @param options.nameContains - Filter by name substring
+     * @param options.limit - The maximum number of configs to return
+     * @returns An async iterator of FeedbackConfigSchema objects
+     */
+    async *listFeedbackConfigs(options = {}) {
+        const { feedbackKeys, nameContains, limit } = options;
+        const params = new URLSearchParams();
+        if (feedbackKeys) {
+            feedbackKeys.forEach((key) => {
+                params.append("key", key);
+            });
+        }
+        if (nameContains)
+            params.append("name_contains", nameContains);
+        params.append("limit", (limit !== undefined ? Math.min(limit, 100) : 100).toString());
+        let count = 0;
+        for await (const configs of this._getPaginated("/feedback-configs", params)) {
+            yield* configs;
+            count += configs.length;
+            if (limit !== undefined && count >= limit)
+                break;
+        }
+    }
+    /**
+     * Update a feedback configuration on the LangSmith API.
+     * @param feedbackKey - The key of the feedback config to update
+     * @param options - The options for updating the feedback config
+     * @param options.feedbackConfig - The new feedback config
+     * @param options.isLowerScoreBetter - Whether a lower score is better
+     * @returns The updated FeedbackConfigSchema object
+     */
+    async updateFeedbackConfig(feedbackKey, options = {}) {
+        const { feedbackConfig, isLowerScoreBetter } = options;
+        const body = { feedback_key: feedbackKey };
+        if (feedbackConfig !== undefined) {
+            body.feedback_config = feedbackConfig;
+        }
+        if (isLowerScoreBetter !== undefined) {
+            body.is_lower_score_better = isLowerScoreBetter;
+        }
+        const response = await this.caller.call(async () => {
+            const res = await this._fetch(`${this.apiUrl}/feedback-configs`, {
+                method: "PATCH",
+                headers: { ...this.headers, "Content-Type": "application/json" },
+                signal: AbortSignal.timeout(this.timeout_ms),
+                ...this.fetchOptions,
+                body: JSON.stringify(body),
+            });
+            await (0, error_js_1.raiseForStatus)(res, "update feedback config");
+            return res;
+        });
+        return response.json();
+    }
+    /**
+     * Delete a feedback configuration on the LangSmith API.
+     * @param feedbackKey - The key of the feedback config to delete
+     */
+    async deleteFeedbackConfig(feedbackKey) {
+        const params = new URLSearchParams({ feedback_key: feedbackKey });
+        await this.caller.call(async () => {
+            const res = await this._fetch(`${this.apiUrl}/feedback-configs?${params}`, {
+                method: "DELETE",
+                headers: this.headers,
+                signal: AbortSignal.timeout(this.timeout_ms),
+                ...this.fetchOptions,
+            });
+            await (0, error_js_1.raiseForStatus)(res, "delete feedback config", true);
+            return res;
+        });
+    }
     /**
      * API for managing annotation queues
      */
@@ -3272,12 +3381,13 @@ class Client {
      * @returns The created AnnotationQueue object
      */
     async createAnnotationQueue(options) {
-        const { name, description, queueId, rubricInstructions } = options;
+        const { name, description, queueId, rubricInstructions, rubricItems } = options;
         const body = {
             name,
             description,
             id: queueId || uuid.v4(),
             rubric_instructions: rubricInstructions,
+            rubric_items: rubricItems,
         };
         const serializedBody = JSON.stringify(Object.fromEntries(Object.entries(body).filter(([_, v]) => v !== undefined)));
         const response = await this.caller.call(async () => {
@@ -3319,12 +3429,17 @@ class Client {
      * @param options.description - The new description for the annotation queue
      */
     async updateAnnotationQueue(queueId, options) {
-        const { name, description, rubricInstructions } = options;
-        const body = JSON.stringify({
-            name,
-            description,
-            rubric_instructions: rubricInstructions,
-        });
+        const { name, description, rubricInstructions, rubricItems } = options;
+        const bodyObj = {};
+        if (name !== undefined)
+            bodyObj.name = name;
+        if (description !== undefined)
+            bodyObj.description = description;
+        if (rubricInstructions !== undefined)
+            bodyObj.rubric_instructions = rubricInstructions;
+        if (rubricItems !== undefined)
+            bodyObj.rubric_items = rubricItems;
+        const body = JSON.stringify(bodyObj);
         await this.caller.call(async () => {
             const res = await this._fetch(`${this.apiUrl}/annotation-queues/${(0, _uuid_js_1.assertUuid)(queueId, "queueId")}`, {
                 method: "PATCH",

package/dist/client.d.ts

@@ -1,6 +1,6 @@
 import type { OTELContext } from "./experimental/otel/types.js";
 import { AsyncCallerParams } from "./utils/async_caller.js";
-import { ComparativeExperiment, DataType, Dataset, DatasetDiffInfo, DatasetShareSchema, Example, ExampleCreate, ExampleUpdate, ExampleUpdateWithoutId, Feedback, FeedbackConfig, FeedbackIngestToken, KVMap, LangChainBaseMessage, LangSmithSettings, LikePromptResponse, Prompt, PromptCommit, PromptSortField, Run, RunCreate, RunUpdate, ScoreType, ExampleSearch, TimeDelta, TracerSession, TracerSessionResult, ValueType, AnnotationQueue, RunWithAnnotationQueueInfo, Attachments, UploadExamplesResponse, UpdateExamplesResponse, DatasetVersion, AnnotationQueueWithDetails } from "./schemas.js";
+import { ComparativeExperiment, DataType, Dataset, DatasetDiffInfo, DatasetShareSchema, Example, ExampleCreate, ExampleUpdate, ExampleUpdateWithoutId, Feedback, FeedbackConfig, FeedbackIngestToken, KVMap, LangChainBaseMessage, LangSmithSettings, LikePromptResponse, Prompt, PromptCommit, PromptSortField, Run, RunCreate, RunUpdate, ScoreType, ExampleSearch, TimeDelta, TracerSession, TracerSessionResult, ValueType, AnnotationQueue, RunWithAnnotationQueueInfo, Attachments, UploadExamplesResponse, UpdateExamplesResponse, DatasetVersion, AnnotationQueueWithDetails, AnnotationQueueRubricItem, FeedbackConfigSchema } from "./schemas.js";
 import { EvaluationResult, EvaluationResults } from "./evaluation/evaluator.js";
 import { PromptCache } from "./utils/prompt_cache/index.js";
 export interface ClientConfig {
@@ -906,6 +906,56 @@ export declare class Client implements LangSmithTracingClientInterface {
     logEvaluationFeedback(evaluatorResponse: EvaluationResult | EvaluationResult[] | EvaluationResults, run?: Run, sourceInfo?: {
         [key: string]: any;
     }): Promise<EvaluationResult[]>;
+    /**
+     * API for managing feedback configs
+     */
+    /**
+     * Create a feedback configuration on the LangSmith API.
+     *
+     * This upserts: if an identical config already exists, it returns it.
+     * If a conflicting config exists for the same key, a 400 error is raised.
+     *
+     * @param options - The options for creating a feedback config
+     * @param options.feedbackKey - The unique key for this feedback config
+     * @param options.feedbackConfig - The config specifying type, bounds, and categories
+     * @param options.isLowerScoreBetter - Whether a lower score is better
+     * @returns The created FeedbackConfigSchema object
+     */
+    createFeedbackConfig(options: {
+        feedbackKey: string;
+        feedbackConfig: FeedbackConfig;
+        isLowerScoreBetter?: boolean;
+    }): Promise<FeedbackConfigSchema>;
+    /**
+     * List feedback configurations on the LangSmith API.
+     * @param options - The options for listing feedback configs
+     * @param options.feedbackKeys - Filter by specific feedback keys
+     * @param options.nameContains - Filter by name substring
+     * @param options.limit - The maximum number of configs to return
+     * @returns An async iterator of FeedbackConfigSchema objects
+     */
+    listFeedbackConfigs(options?: {
+        feedbackKeys?: string[];
+        nameContains?: string;
+        limit?: number;
+    }): AsyncIterableIterator<FeedbackConfigSchema>;
+    /**
+     * Update a feedback configuration on the LangSmith API.
+     * @param feedbackKey - The key of the feedback config to update
+     * @param options - The options for updating the feedback config
+     * @param options.feedbackConfig - The new feedback config
+     * @param options.isLowerScoreBetter - Whether a lower score is better
+     * @returns The updated FeedbackConfigSchema object
+     */
+    updateFeedbackConfig(feedbackKey: string, options?: {
+        feedbackConfig?: FeedbackConfig;
+        isLowerScoreBetter?: boolean;
+    }): Promise<FeedbackConfigSchema>;
+    /**
+     * Delete a feedback configuration on the LangSmith API.
+     * @param feedbackKey - The key of the feedback config to delete
+     */
+    deleteFeedbackConfig(feedbackKey: string): Promise<void>;
     /**
      * API for managing annotation queues
      */
@@ -937,6 +987,7 @@ export declare class Client implements LangSmithTracingClientInterface {
         description?: string;
         queueId?: string;
         rubricInstructions?: string;
+        rubricItems?: AnnotationQueueRubricItem[];
     }): Promise<AnnotationQueueWithDetails>;
     /**
      * Read an annotation queue with the specified queue ID.
@@ -952,9 +1003,10 @@ export declare class Client implements LangSmithTracingClientInterface {
      * @param options.description - The new description for the annotation queue
      */
     updateAnnotationQueue(queueId: string, options: {
-        name: string;
+        name?: string;
         description?: string;
         rubricInstructions?: string;
+        rubricItems?: AnnotationQueueRubricItem[];
     }): Promise<void>;
     /**
      * Delete an annotation queue with the specified queue ID.

package/dist/client.js

@@ -3191,6 +3191,115 @@ export class Client {
         const [results] = await this._logEvaluationFeedback(evaluatorResponse, run, sourceInfo);
         return results;
     }
+    /**
+     * API for managing feedback configs
+     */
+    /**
+     * Create a feedback configuration on the LangSmith API.
+     *
+     * This upserts: if an identical config already exists, it returns it.
+     * If a conflicting config exists for the same key, a 400 error is raised.
+     *
+     * @param options - The options for creating a feedback config
+     * @param options.feedbackKey - The unique key for this feedback config
+     * @param options.feedbackConfig - The config specifying type, bounds, and categories
+     * @param options.isLowerScoreBetter - Whether a lower score is better
+     * @returns The created FeedbackConfigSchema object
+     */
+    async createFeedbackConfig(options) {
+        const { feedbackKey, feedbackConfig, isLowerScoreBetter = false } = options;
+        const body = {
+            feedback_key: feedbackKey,
+            feedback_config: feedbackConfig,
+            is_lower_score_better: isLowerScoreBetter,
+        };
+        const response = await this.caller.call(async () => {
+            const res = await this._fetch(`${this.apiUrl}/feedback-configs`, {
+                method: "POST",
+                headers: { ...this.headers, "Content-Type": "application/json" },
+                signal: AbortSignal.timeout(this.timeout_ms),
+                ...this.fetchOptions,
+                body: JSON.stringify(body),
+            });
+            await raiseForStatus(res, "create feedback config");
+            return res;
+        });
+        return response.json();
+    }
+    /**
+     * List feedback configurations on the LangSmith API.
+     * @param options - The options for listing feedback configs
+     * @param options.feedbackKeys - Filter by specific feedback keys
+     * @param options.nameContains - Filter by name substring
+     * @param options.limit - The maximum number of configs to return
+     * @returns An async iterator of FeedbackConfigSchema objects
+     */
+    async *listFeedbackConfigs(options = {}) {
+        const { feedbackKeys, nameContains, limit } = options;
+        const params = new URLSearchParams();
+        if (feedbackKeys) {
+            feedbackKeys.forEach((key) => {
+                params.append("key", key);
+            });
+        }
+        if (nameContains)
+            params.append("name_contains", nameContains);
+        params.append("limit", (limit !== undefined ? Math.min(limit, 100) : 100).toString());
+        let count = 0;
+        for await (const configs of this._getPaginated("/feedback-configs", params)) {
+            yield* configs;
+            count += configs.length;
+            if (limit !== undefined && count >= limit)
+                break;
+        }
+    }
+    /**
+     * Update a feedback configuration on the LangSmith API.
+     * @param feedbackKey - The key of the feedback config to update
+     * @param options - The options for updating the feedback config
+     * @param options.feedbackConfig - The new feedback config
+     * @param options.isLowerScoreBetter - Whether a lower score is better
+     * @returns The updated FeedbackConfigSchema object
+     */
+    async updateFeedbackConfig(feedbackKey, options = {}) {
+        const { feedbackConfig, isLowerScoreBetter } = options;
+        const body = { feedback_key: feedbackKey };
+        if (feedbackConfig !== undefined) {
+            body.feedback_config = feedbackConfig;
+        }
+        if (isLowerScoreBetter !== undefined) {
+            body.is_lower_score_better = isLowerScoreBetter;
+        }
+        const response = await this.caller.call(async () => {
+            const res = await this._fetch(`${this.apiUrl}/feedback-configs`, {
+                method: "PATCH",
+                headers: { ...this.headers, "Content-Type": "application/json" },
+                signal: AbortSignal.timeout(this.timeout_ms),
+                ...this.fetchOptions,
+                body: JSON.stringify(body),
+            });
+            await raiseForStatus(res, "update feedback config");
+            return res;
+        });
+        return response.json();
+    }
+    /**
+     * Delete a feedback configuration on the LangSmith API.
+     * @param feedbackKey - The key of the feedback config to delete
+     */
+    async deleteFeedbackConfig(feedbackKey) {
+        const params = new URLSearchParams({ feedback_key: feedbackKey });
+        await this.caller.call(async () => {
+            const res = await this._fetch(`${this.apiUrl}/feedback-configs?${params}`, {
+                method: "DELETE",
+                headers: this.headers,
+                signal: AbortSignal.timeout(this.timeout_ms),
+                ...this.fetchOptions,
+            });
+            await raiseForStatus(res, "delete feedback config", true);
+            return res;
+        });
+    }
     /**
      * API for managing annotation queues
      */
@@ -3234,12 +3343,13 @@ export class Client {
      * @returns The created AnnotationQueue object
      */
     async createAnnotationQueue(options) {
-        const { name, description, queueId, rubricInstructions } = options;
+        const { name, description, queueId, rubricInstructions, rubricItems } = options;
         const body = {
             name,
             description,
             id: queueId || uuid.v4(),
             rubric_instructions: rubricInstructions,
+            rubric_items: rubricItems,
         };
         const serializedBody = JSON.stringify(Object.fromEntries(Object.entries(body).filter(([_, v]) => v !== undefined)));
         const response = await this.caller.call(async () => {
@@ -3281,12 +3391,17 @@ export class Client {
      * @param options.description - The new description for the annotation queue
      */
     async updateAnnotationQueue(queueId, options) {
-        const { name, description, rubricInstructions } = options;
-        const body = JSON.stringify({
-            name,
-            description,
-            rubric_instructions: rubricInstructions,
-        });
+        const { name, description, rubricInstructions, rubricItems } = options;
+        const bodyObj = {};
+        if (name !== undefined)
+            bodyObj.name = name;
+        if (description !== undefined)
+            bodyObj.description = description;
+        if (rubricInstructions !== undefined)
+            bodyObj.rubric_instructions = rubricInstructions;
+        if (rubricItems !== undefined)
+            bodyObj.rubric_items = rubricItems;
+        const body = JSON.stringify(bodyObj);
         await this.caller.call(async () => {
             const res = await this._fetch(`${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}`, {
                 method: "PATCH",

package/dist/evaluation/_runner.cjs

@@ -783,13 +783,13 @@ async function _evaluate(target, fields) {
     return results;
 }
 async function _forward(fn, example, experimentName, metadata, client, includeAttachments) {
-    let run = null;
+    let run = undefined;
     const _getRun = (r) => {
         run = r;
     };
     const defaultOptions = {
         reference_example_id: example.id,
-        on_end: _getRun,
+        on_start: _getRun,
         project_name: experimentName,
         metadata: {
             ...metadata,

package/dist/evaluation/_runner.js

@@ -776,13 +776,13 @@ async function _evaluate(target, fields) {
     return results;
 }
 async function _forward(fn, example, experimentName, metadata, client, includeAttachments) {
-    let run = null;
+    let run = undefined;
     const _getRun = (r) => {
         run = r;
     };
     const defaultOptions = {
         reference_example_id: example.id,
-        on_end: _getRun,
+        on_start: _getRun,
         project_name: experimentName,
         metadata: {
             ...metadata,

package/dist/index.cjs

@@ -18,4 +18,4 @@ Object.defineProperty(exports, "PromptCache", { enumerable: true, get: function
 Object.defineProperty(exports, "configureGlobalPromptCache", { enumerable: true, get: function () { return index_js_1.configureGlobalPromptCache; } });
 Object.defineProperty(exports, "promptCacheSingleton", { enumerable: true, get: function () { return index_js_1.promptCacheSingleton; } });
 // Update using yarn bump-version
-exports.__version__ = "0.5.3";
+exports.__version__ = "0.5.4";

package/dist/index.d.ts

@@ -1,8 +1,8 @@
 export { Client, type ClientConfig, type LangSmithTracingClientInterface, } from "./client.js";
-export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
+export type { Dataset, Example, TracerSession, Run, Feedback, FeedbackConfigSchema, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 export { getDefaultProjectName } from "./utils/project.js";
 export { uuid7, uuid7FromTime } from "./uuid.js";
 export { Cache, PromptCache, type CacheConfig, type CacheMetrics, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
-export declare const __version__ = "0.5.3";
+export declare const __version__ = "0.5.4";

package/dist/index.js

@@ -5,4 +5,4 @@ export { getDefaultProjectName } from "./utils/project.js";
 export { uuid7, uuid7FromTime } from "./uuid.js";
 export { Cache, PromptCache, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
 // Update using yarn bump-version
-export const __version__ = "0.5.3";
+export const __version__ = "0.5.4";

package/dist/schemas.d.ts

@@ -457,9 +457,35 @@ export interface AnnotationQueue {
     /** The ID of the tenant associated with the annotation queue. */
     tenant_id: string;
 }
+export interface AnnotationQueueRubricItem {
+    /** The feedback key this rubric item is associated with. */
+    feedback_key: string;
+    /** An optional description of the rubric item. */
+    description?: string | null;
+    /** Descriptions for categorical values. */
+    value_descriptions?: Record<string, string> | null;
+    /** Descriptions for continuous score values. */
+    score_descriptions?: Record<string, string> | null;
+    /** Whether this rubric item is required. */
+    is_required?: boolean | null;
+}
 export interface AnnotationQueueWithDetails extends AnnotationQueue {
     /** The rubric instructions for the annotation queue. */
     rubric_instructions?: string;
+    /** The rubric items for the annotation queue. */
+    rubric_items?: AnnotationQueueRubricItem[];
+}
+export interface FeedbackConfigSchema {
+    /** The unique key identifying this feedback configuration. */
+    feedback_key: string;
+    /** The configuration specifying the type, bounds, and categories. */
+    feedback_config: FeedbackConfig;
+    /** The ID of the tenant that owns this feedback configuration. */
+    tenant_id: string;
+    /** When this feedback configuration was last modified. */
+    modified_at: string;
+    /** Whether a lower score is considered better for this feedback key. */
+    is_lower_score_better?: boolean | null;
 }
 export interface RunWithAnnotationQueueInfo extends BaseRun {
     /** The last time this run was reviewed. */

package/dist/traceable.cjs

@@ -123,7 +123,7 @@ async function handleEnd(params) {
         });
     }
 }
-const _populateUsageMetadata = (processedOutputs, runTree) => {
+const _populateUsageMetadataAndOutputs = (processedOutputs, runTree) => {
     if (runTree !== undefined) {
         let usageMetadata;
         try {
@@ -139,6 +139,8 @@ const _populateUsageMetadata = (processedOutputs, runTree) => {
             };
             processedOutputs.usage_metadata = usageMetadata;
         }
+        // Set outputs on run tree as soon as available
+        runTree.outputs = processedOutputs;
     }
 };
 function isAsyncFn(fn) {
@@ -170,7 +172,7 @@ async function handleRunOutputs(params) {
         if (isAsyncFn(processOutputsFn)) {
             void outputs
                 .then(async (processedOutputs) => {
-                _populateUsageMetadata(processedOutputs, runTree);
+                _populateUsageMetadataAndOutputs(processedOutputs, runTree);
                 await childRunEndPromises;
                 await runTree?.end(processedOutputs);
             })
@@ -203,7 +205,7 @@ async function handleRunOutputs(params) {
     catch (e) {
         console.error("Error occurred during processOutputs. Sending unprocessed outputs:", e);
     }
-    _populateUsageMetadata(outputs, runTree);
+    _populateUsageMetadataAndOutputs(outputs, runTree);
     void childRunEndPromises
         .then(async () => {
         try {
@@ -441,7 +443,7 @@ const convertSerializableArg = (arg, options) => {
  */
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 function traceable(wrappedFunc, config) {
-    const { aggregator, argsConfigPath, __finalTracedIteratorKey, processInputs, processOutputs, extractAttachments, ...runTreeConfig } = config ?? {};
+    const { aggregator, argsConfigPath, __finalTracedIteratorKey, processInputs, processOutputs, extractAttachments, on_start, ...runTreeConfig } = config ?? {};
     const processInputsFn = processInputs ?? ((x) => x);
     const processOutputsFn = processOutputs ?? ((x) => x);
     const extractAttachmentsFn = extractAttachments ?? ((...x) => [undefined, runInputsToMap(x)]);
@@ -577,6 +579,7 @@ function traceable(wrappedFunc, config) {
         const currentRunTree = (0, run_trees_js_1.isRunTree)(currentContext)
             ? currentContext
             : undefined;
+        on_start?.(currentRunTree);
         const otelContextManager = maybeCreateOtelContext(currentRunTree, config?.project_name, config?.tracer);
         const otel_context = (0, otel_js_1.getOTELContext)();
         const runWithContext = () => {

package/dist/traceable.d.ts

@@ -1,4 +1,4 @@
-import { RunTreeConfig } from "./run_trees.js";
+import { RunTree, RunTreeConfig } from "./run_trees.js";
 import { Attachments, InvocationParamsSchema, KVMap } from "./schemas.js";
 import type { TraceableFunction } from "./singletons/types.js";
 import { OTELTracer } from "./experimental/otel/types.js";
@@ -18,6 +18,7 @@ export type TraceableConfig<Func extends (...args: any[]) => any> = Partial<Omit
     aggregator?: (args: any[]) => any;
     argsConfigPath?: [number] | [number, string];
     tracer?: OTELTracer;
+    on_start?: (runTree: RunTree | undefined) => void;
     __finalTracedIteratorKey?: string;
     __deferredSerializableArgOptions?: {
         depth?: number;

package/dist/traceable.js

@@ -119,7 +119,7 @@ async function handleEnd(params) {
         });
     }
 }
-const _populateUsageMetadata = (processedOutputs, runTree) => {
+const _populateUsageMetadataAndOutputs = (processedOutputs, runTree) => {
     if (runTree !== undefined) {
         let usageMetadata;
         try {
@@ -135,6 +135,8 @@ const _populateUsageMetadata = (processedOutputs, runTree) => {
             };
             processedOutputs.usage_metadata = usageMetadata;
         }
+        // Set outputs on run tree as soon as available
+        runTree.outputs = processedOutputs;
     }
 };
 function isAsyncFn(fn) {
@@ -166,7 +168,7 @@ async function handleRunOutputs(params) {
         if (isAsyncFn(processOutputsFn)) {
             void outputs
                 .then(async (processedOutputs) => {
-                _populateUsageMetadata(processedOutputs, runTree);
+                _populateUsageMetadataAndOutputs(processedOutputs, runTree);
                 await childRunEndPromises;
                 await runTree?.end(processedOutputs);
             })
@@ -199,7 +201,7 @@ async function handleRunOutputs(params) {
     catch (e) {
         console.error("Error occurred during processOutputs. Sending unprocessed outputs:", e);
     }
-    _populateUsageMetadata(outputs, runTree);
+    _populateUsageMetadataAndOutputs(outputs, runTree);
     void childRunEndPromises
         .then(async () => {
         try {
@@ -437,7 +439,7 @@ const convertSerializableArg = (arg, options) => {
  */
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export function traceable(wrappedFunc, config) {
-    const { aggregator, argsConfigPath, __finalTracedIteratorKey, processInputs, processOutputs, extractAttachments, ...runTreeConfig } = config ?? {};
+    const { aggregator, argsConfigPath, __finalTracedIteratorKey, processInputs, processOutputs, extractAttachments, on_start, ...runTreeConfig } = config ?? {};
     const processInputsFn = processInputs ?? ((x) => x);
     const processOutputsFn = processOutputs ?? ((x) => x);
     const extractAttachmentsFn = extractAttachments ?? ((...x) => [undefined, runInputsToMap(x)]);
@@ -573,6 +575,7 @@ export function traceable(wrappedFunc, config) {
         const currentRunTree = isRunTree(currentContext)
             ? currentContext
             : undefined;
+        on_start?.(currentRunTree);
         const otelContextManager = maybeCreateOtelContext(currentRunTree, config?.project_name, config?.tracer);
         const otel_context = getOTELContext();
         const runWithContext = () => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.5.3",
+  "version": "0.5.4",
   "description": "Client library to connect to the LangSmith Observability and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [