langsmith 0.8.5 → 0.8.6

package/dist/experimental/vercel/telemetry.cjs

@@ -2,7 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LangSmithTelemetry = LangSmithTelemetry;
 const run_trees_js_1 = require("../../run_trees.cjs");
-const traceable_js_1 = require("../../singletons/traceable.cjs");
+const traceable_js_1 = require("../../traceable.cjs");
 const env_js_1 = require("../../env.cjs");
 const utils_js_1 = require("./utils.cjs");
 const utils_js_2 = require("./utils.cjs");
@@ -58,6 +58,12 @@ function _getLsAgentType(parentRunTree) {
     }
     return "root";
 }
+function _hasNonzeroUsageMetadata(runTree) {
+    const usageMetadata = runTree.extra?.metadata?.usage_metadata;
+    if (!(0, types_js_1.isRecord)(usageMetadata))
+        return false;
+    return ["input_tokens", "output_tokens", "total_tokens"].some((key) => typeof usageMetadata[key] === "number" && usageMetadata[key] > 0);
+}
 /**
  * Creates a LangSmith `Telemetry` for the Vercel AI SDK.
  *
@@ -82,6 +88,31 @@ function _getLsAgentType(parentRunTree) {
  */
 function LangSmithTelemetry(config) {
     const { name: customName, runType = "chain", metadata: customMetadata, tags: customTags, client, projectName, processInputs, processOutputs, processChildLLMRunInputs, processChildLLMRunOutputs, traceResponseMetadata, traceRawHttp, tracingEnabled, extra: customExtra, } = config ?? {};
+    function formatToolResultMessage(toolCallId, toolName, toolOutput) {
+        const rawOutput = (0, types_js_1.isRecord)(toolOutput)
+            ? toolOutput.type === "tool-result"
+                ? toolOutput.output
+                : "error" in toolOutput
+                    ? toolOutput.error
+                    : toolOutput
+            : toolOutput;
+        const artifact = rawOutput instanceof Error ? rawOutput.message : rawOutput;
+        const content = typeof artifact === "string"
+            ? artifact
+            : (JSON.stringify(artifact) ?? String(artifact));
+        return {
+            role: "tool",
+            content,
+            tool_call_id: toolCallId,
+            name: toolName,
+            artifact,
+        };
+    }
+    function appendToolResultMessage(harnessState, toolCallId, toolName, toolOutput) {
+        const message = formatToolResultMessage(toolCallId, toolName, toolOutput);
+        harnessState.completedStepMessages.push(message);
+        return message;
+    }
     function getOpenStepOrRoot(state) {
         let openStep;
         state.stepRunTrees.forEach((stepRt) => {
@@ -91,6 +122,16 @@ function LangSmithTelemetry(config) {
         });
         return openStep ?? state.rootRunTree;
     }
+    async function flushPiDeferredStepRun(state) {
+        const workaround = state.experimental_harness?.piAggregateUsageWorkaround;
+        if (workaround == null)
+            return;
+        const stepRunTree = workaround.deferredStepRunTree;
+        if (stepRunTree == null)
+            return;
+        workaround.deferredStepRunTree = undefined;
+        await stepRunTree.patchRun({ excludeInputs: true });
+    }
     async function finalizeOpenToolRuns(state, opts) {
         const entries = Array.from(state.toolRunTrees.entries());
         for (let i = 0; i < entries.length; i++) {
@@ -161,7 +202,6 @@ function LangSmithTelemetry(config) {
                     ai_sdk_method: event.operationId,
                     ls_agent_type: _getLsAgentType(parentRunTree),
                     ls_model_name: event.modelId,
-                    ls_provider: event.provider,
                     ls_integration: "vercel-ai-sdk-telemetry",
                 },
             },
@@ -181,17 +221,29 @@ function LangSmithTelemetry(config) {
             rootRunTree,
             stepRunTrees: new Map(),
             toolRunTrees: new Map(),
+            experimental_harness: event.operationId === "ai.harness"
+                ? {
+                    lastStepContent: [],
+                    completedStepMessages: [],
+                    toolResults: new Map(),
+                    piAggregateUsageWorkaround: event.provider === "pi" ? {} : undefined,
+                }
+                : undefined,
         });
     };
     const onStepStart = async (event) => {
         const state = invocationsByCallId.get(event.callId);
         if (!state)
             return;
+        await flushPiDeferredStepRun(state);
         const stepNumber = event.stepNumber ?? 0;
         let inputs = {};
         if (event.recordInputs !== false) {
             if ("messages" in event && event.messages != null) {
-                inputs.messages = _formatMessages(event.messages);
+                inputs.messages = [
+                    ..._formatMessages(event.messages),
+                    ...(state.experimental_harness?.completedStepMessages ?? []),
+                ];
             }
             if ("runtimeContext" in event && event.runtimeContext != null) {
                 inputs.runtimeContext = event.runtimeContext;
@@ -212,9 +264,18 @@ function LangSmithTelemetry(config) {
             name: event.provider,
             run_type: "llm",
             inputs,
-            extra: { metadata: { step_number: stepNumber } },
+            extra: {
+                metadata: {
+                    step_number: stepNumber,
+                    ls_model_name: event.modelId,
+                },
+            },
         });
         state.stepRunTrees.set(stepNumber, stepRunTree);
+        const piUsageWorkaround = state.experimental_harness?.piAggregateUsageWorkaround;
+        if (piUsageWorkaround != null) {
+            piUsageWorkaround.latestStepRunTree = stepRunTree;
+        }
         await stepRunTree.postRun();
     };
     const onLanguageModelCallStart = async (event) => {
@@ -254,22 +315,25 @@ function LangSmithTelemetry(config) {
         const state = invocationsByCallId.get(event.callId);
         if (!state)
             return;
-        const parentRunTree = getOpenStepOrRoot(state);
+        const parentRunTree = state.rootRunTree;
         let inputs = {};
         if (event.recordInputs !== false) {
-            if ((0, types_js_1.isRecord)(event.toolCall.input)) {
-                inputs = { ...event.toolCall.input };
+            let toolCallInput = event.toolCall.input;
+            if (typeof toolCallInput === "string") {
+                try {
+                    toolCallInput = JSON.parse(toolCallInput);
+                }
+                catch {
+                    // The tool argument itself may be a plain string rather than JSON.
+                }
             }
-            else if (typeof event.toolCall.input !== "undefined") {
-                inputs = { input: event.toolCall.input };
+            if ((0, types_js_1.isRecord)(toolCallInput)) {
+                inputs = { ...toolCallInput };
             }
-            if ("toolContext" in event && event.toolContext != null) {
-                inputs.toolContext = event.toolContext;
+            else if (toolCallInput !== undefined) {
+                inputs = { input: toolCallInput };
             }
         }
-        else {
-            inputs = {};
-        }
         const toolRunTree = parentRunTree.createChild({
             name: event.toolCall.toolName,
             run_type: "tool",
@@ -291,14 +355,20 @@ function LangSmithTelemetry(config) {
         const toolRunTree = state.toolRunTrees.get(event.toolCall.toolCallId);
         if (!toolRunTree)
             return;
-        state.toolRunTrees.delete(event.toolCall.toolCallId);
+        const harnessState = state.experimental_harness;
+        if (harnessState != null) {
+            harnessState.toolResults.set(event.toolCall.toolCallId, {
+                toolName: event.toolCall.toolName,
+                output: event.toolOutput,
+            });
+        }
         let outputs;
         let error;
         if (event.recordOutputs !== false) {
             if (event.toolOutput.type === "tool-result") {
                 outputs = { output: event.toolOutput.output };
             }
-            else if (event.toolOutput.type === "tool-error") {
+            else if ((0, types_js_1.isRecord)(event.toolOutput) && "error" in event.toolOutput) {
                 const err = event.toolOutput.error;
                 error = err instanceof Error ? err.message : String(err);
             }
@@ -308,6 +378,7 @@ function LangSmithTelemetry(config) {
         }
         await toolRunTree.end(outputs, error, Math.floor(toolRunTree.start_time + event.toolExecutionMs));
         await toolRunTree.patchRun({ excludeInputs: true });
+        state.toolRunTrees.delete(event.toolCall.toolCallId);
     };
     const onStepFinish = async (event) => {
         const state = invocationsByCallId.get(event.callId);
@@ -317,8 +388,12 @@ function LangSmithTelemetry(config) {
         const stepRunTree = state.stepRunTrees.get(stepNumber);
         if (!stepRunTree)
             return;
+        const harnessState = state.experimental_harness;
         let outputs = {};
         if (event.recordOutputs !== false) {
+            if (harnessState != null && Array.isArray(event.content)) {
+                harnessState.lastStepContent = [...event.content];
+            }
             outputs = _formatStepOutput(event, traceRawHttp);
             if (processChildLLMRunOutputs) {
                 try {
@@ -329,11 +404,38 @@ function LangSmithTelemetry(config) {
                 }
             }
         }
+        if (harnessState != null && Array.isArray(event.content)) {
+            harnessState.completedStepMessages.push((0, utils_js_1.convertMessageToTracedFormat)({
+                role: "assistant",
+                content: event.content,
+            }));
+            for (const part of event.content) {
+                if (!(0, types_js_1.isRecord)(part) ||
+                    part.type !== "tool-call" ||
+                    typeof part.toolCallId !== "string") {
+                    continue;
+                }
+                const toolResult = harnessState.toolResults.get(part.toolCallId);
+                if (toolResult) {
+                    appendToolResultMessage(harnessState, part.toolCallId, toolResult.toolName, toolResult.output);
+                    harnessState.toolResults.delete(part.toolCallId);
+                }
+            }
+        }
         // Set usage metadata
         // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
         (0, utils_js_2.setUsageMetadataOnRunTree)(event, stepRunTree);
         await stepRunTree.end(outputs);
-        await stepRunTree.patchRun({ excludeInputs: true });
+        const piUsageWorkaround = harnessState?.piAggregateUsageWorkaround;
+        if (piUsageWorkaround != null && !_hasNonzeroUsageMetadata(stepRunTree)) {
+            // Pi reports zero usage for inferred step boundaries and sends the real
+            // aggregate at turn end. Defer the final PATCH so that aggregate can be
+            // attached to the last LLM run without issuing a duplicate update.
+            piUsageWorkaround.deferredStepRunTree = stepRunTree;
+        }
+        else {
+            await stepRunTree.patchRun({ excludeInputs: true });
+        }
         state.stepRunTrees.delete(stepNumber);
     };
     const onEnd = async (event) => {
@@ -343,6 +445,8 @@ function LangSmithTelemetry(config) {
         if (!state)
             return;
         const { rootRunTree } = state;
+        const harnessState = state.experimental_harness;
+        const piUsageWorkaround = harnessState?.piAggregateUsageWorkaround;
         await finalizeOpenToolRuns(state, { note: "closed on finish" });
         // Ensure any remaining step runs are closed
         const remainingSteps = Array.from(state.stepRunTrees.entries());
@@ -350,7 +454,12 @@ function LangSmithTelemetry(config) {
             const [stepNumber, stepRt] = remainingSteps[i];
             if (stepRt.end_time == null) {
                 await stepRt.end({ note: "closed on finish" });
-                await stepRt.patchRun({ excludeInputs: true });
+                if (piUsageWorkaround != null && !_hasNonzeroUsageMetadata(stepRt)) {
+                    piUsageWorkaround.deferredStepRunTree = stepRt;
+                }
+                else {
+                    await stepRt.patchRun({ excludeInputs: true });
+                }
             }
             state.stepRunTrees.delete(stepNumber);
         }
@@ -360,9 +469,18 @@ function LangSmithTelemetry(config) {
             if ("text" in event && event.text != null) {
                 outputs.content = event.text;
             }
-            else if ("content" in event && event.content != null) {
+            else if ("content" in event &&
+                event.content != null &&
+                (!Array.isArray(event.content) || event.content.length > 0)) {
                 outputs.content = event.content;
             }
+            else if (harnessState != null &&
+                harnessState.lastStepContent.length > 0) {
+                // HarnessAgent currently reports an empty root `content` array even
+                // though each completed step contains assistant output. Only the final
+                // step belongs on the root; prior steps have their own child runs.
+                outputs.content = harnessState.lastStepContent;
+            }
             if (outputs.content != null) {
                 outputs.role = "assistant";
             }
@@ -394,18 +512,28 @@ function LangSmithTelemetry(config) {
                 }
             }
         }
-        // Set aggregated usage on root
-        if ("totalUsage" in event && event.totalUsage != null) {
-            (0, utils_js_2.setUsageMetadataOnRunTree)(
-            // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
-            { usage: event.totalUsage, providerMetadata: event.providerMetadata }, rootRunTree);
-        }
-        else if ("usage" in event && event.usage != null) {
-            // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
-            (0, utils_js_2.setUsageMetadataOnRunTree)(event, rootRunTree);
+        // Pi reports zero usage on inferred step boundaries and provides the actual
+        // aggregate only when the turn ends. Keep that usage on an LLM run rather
+        // than incorrectly elevating it to the parent chain. The latest step is the
+        // only accurate place available until Harness Pi exposes per-step usage.
+        const latestStep = piUsageWorkaround?.latestStepRunTree;
+        if (latestStep?.run_type === "llm" &&
+            !_hasNonzeroUsageMetadata(latestStep)) {
+            if ("totalUsage" in event && event.totalUsage != null) {
+                (0, utils_js_2.setUsageMetadataOnRunTree)(
+                // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
+                { usage: event.totalUsage, providerMetadata: event.providerMetadata }, latestStep);
+            }
+            else if ("usage" in event && event.usage != null) {
+                // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
+                (0, utils_js_2.setUsageMetadataOnRunTree)(event, latestStep);
+            }
         }
         await rootRunTree.end(outputs);
-        await rootRunTree.patchRun({ excludeInputs: true });
+        await Promise.all([
+            flushPiDeferredStepRun(state),
+            rootRunTree.patchRun({ excludeInputs: true }),
+        ]);
         invocationsByCallId.delete(event.callId);
     };
     const onError = async (payload) => {
@@ -436,6 +564,7 @@ function LangSmithTelemetry(config) {
             }
             state.stepRunTrees.delete(stepNumber);
         }
+        await flushPiDeferredStepRun(state);
         await rootRunTree.end(undefined, errorMsg);
         await rootRunTree.patchRun({ excludeInputs: true });
         invocationsByCallId.delete(callId);

package/dist/experimental/vercel/telemetry.js

@@ -1,5 +1,5 @@
 import { isRunTree, RunTree } from "../../run_trees.js";
-import { getCurrentRunTree, withRunTree } from "../../singletons/traceable.js";
+import { getCurrentRunTree, withRunTree } from "../../traceable.js";
 import { isEnvTracingEnabled } from "../../env.js";
 import { convertMessageToTracedFormat } from "./utils.js";
 import { setUsageMetadataOnRunTree } from "./utils.js";
@@ -55,6 +55,12 @@ function _getLsAgentType(parentRunTree) {
     }
     return "root";
 }
+function _hasNonzeroUsageMetadata(runTree) {
+    const usageMetadata = runTree.extra?.metadata?.usage_metadata;
+    if (!isRecord(usageMetadata))
+        return false;
+    return ["input_tokens", "output_tokens", "total_tokens"].some((key) => typeof usageMetadata[key] === "number" && usageMetadata[key] > 0);
+}
 /**
  * Creates a LangSmith `Telemetry` for the Vercel AI SDK.
  *
@@ -79,6 +85,31 @@ function _getLsAgentType(parentRunTree) {
  */
 export function LangSmithTelemetry(config) {
     const { name: customName, runType = "chain", metadata: customMetadata, tags: customTags, client, projectName, processInputs, processOutputs, processChildLLMRunInputs, processChildLLMRunOutputs, traceResponseMetadata, traceRawHttp, tracingEnabled, extra: customExtra, } = config ?? {};
+    function formatToolResultMessage(toolCallId, toolName, toolOutput) {
+        const rawOutput = isRecord(toolOutput)
+            ? toolOutput.type === "tool-result"
+                ? toolOutput.output
+                : "error" in toolOutput
+                    ? toolOutput.error
+                    : toolOutput
+            : toolOutput;
+        const artifact = rawOutput instanceof Error ? rawOutput.message : rawOutput;
+        const content = typeof artifact === "string"
+            ? artifact
+            : (JSON.stringify(artifact) ?? String(artifact));
+        return {
+            role: "tool",
+            content,
+            tool_call_id: toolCallId,
+            name: toolName,
+            artifact,
+        };
+    }
+    function appendToolResultMessage(harnessState, toolCallId, toolName, toolOutput) {
+        const message = formatToolResultMessage(toolCallId, toolName, toolOutput);
+        harnessState.completedStepMessages.push(message);
+        return message;
+    }
     function getOpenStepOrRoot(state) {
         let openStep;
         state.stepRunTrees.forEach((stepRt) => {
@@ -88,6 +119,16 @@ export function LangSmithTelemetry(config) {
         });
         return openStep ?? state.rootRunTree;
     }
+    async function flushPiDeferredStepRun(state) {
+        const workaround = state.experimental_harness?.piAggregateUsageWorkaround;
+        if (workaround == null)
+            return;
+        const stepRunTree = workaround.deferredStepRunTree;
+        if (stepRunTree == null)
+            return;
+        workaround.deferredStepRunTree = undefined;
+        await stepRunTree.patchRun({ excludeInputs: true });
+    }
     async function finalizeOpenToolRuns(state, opts) {
         const entries = Array.from(state.toolRunTrees.entries());
         for (let i = 0; i < entries.length; i++) {
@@ -158,7 +199,6 @@ export function LangSmithTelemetry(config) {
                     ai_sdk_method: event.operationId,
                     ls_agent_type: _getLsAgentType(parentRunTree),
                     ls_model_name: event.modelId,
-                    ls_provider: event.provider,
                     ls_integration: "vercel-ai-sdk-telemetry",
                 },
             },
@@ -178,17 +218,29 @@ export function LangSmithTelemetry(config) {
             rootRunTree,
             stepRunTrees: new Map(),
             toolRunTrees: new Map(),
+            experimental_harness: event.operationId === "ai.harness"
+                ? {
+                    lastStepContent: [],
+                    completedStepMessages: [],
+                    toolResults: new Map(),
+                    piAggregateUsageWorkaround: event.provider === "pi" ? {} : undefined,
+                }
+                : undefined,
         });
     };
     const onStepStart = async (event) => {
         const state = invocationsByCallId.get(event.callId);
         if (!state)
             return;
+        await flushPiDeferredStepRun(state);
         const stepNumber = event.stepNumber ?? 0;
         let inputs = {};
         if (event.recordInputs !== false) {
             if ("messages" in event && event.messages != null) {
-                inputs.messages = _formatMessages(event.messages);
+                inputs.messages = [
+                    ..._formatMessages(event.messages),
+                    ...(state.experimental_harness?.completedStepMessages ?? []),
+                ];
             }
             if ("runtimeContext" in event && event.runtimeContext != null) {
                 inputs.runtimeContext = event.runtimeContext;
@@ -209,9 +261,18 @@ export function LangSmithTelemetry(config) {
             name: event.provider,
             run_type: "llm",
             inputs,
-            extra: { metadata: { step_number: stepNumber } },
+            extra: {
+                metadata: {
+                    step_number: stepNumber,
+                    ls_model_name: event.modelId,
+                },
+            },
         });
         state.stepRunTrees.set(stepNumber, stepRunTree);
+        const piUsageWorkaround = state.experimental_harness?.piAggregateUsageWorkaround;
+        if (piUsageWorkaround != null) {
+            piUsageWorkaround.latestStepRunTree = stepRunTree;
+        }
         await stepRunTree.postRun();
     };
     const onLanguageModelCallStart = async (event) => {
@@ -251,22 +312,25 @@ export function LangSmithTelemetry(config) {
         const state = invocationsByCallId.get(event.callId);
         if (!state)
             return;
-        const parentRunTree = getOpenStepOrRoot(state);
+        const parentRunTree = state.rootRunTree;
         let inputs = {};
         if (event.recordInputs !== false) {
-            if (isRecord(event.toolCall.input)) {
-                inputs = { ...event.toolCall.input };
+            let toolCallInput = event.toolCall.input;
+            if (typeof toolCallInput === "string") {
+                try {
+                    toolCallInput = JSON.parse(toolCallInput);
+                }
+                catch {
+                    // The tool argument itself may be a plain string rather than JSON.
+                }
             }
-            else if (typeof event.toolCall.input !== "undefined") {
-                inputs = { input: event.toolCall.input };
+            if (isRecord(toolCallInput)) {
+                inputs = { ...toolCallInput };
             }
-            if ("toolContext" in event && event.toolContext != null) {
-                inputs.toolContext = event.toolContext;
+            else if (toolCallInput !== undefined) {
+                inputs = { input: toolCallInput };
             }
         }
-        else {
-            inputs = {};
-        }
         const toolRunTree = parentRunTree.createChild({
             name: event.toolCall.toolName,
             run_type: "tool",
@@ -288,14 +352,20 @@ export function LangSmithTelemetry(config) {
         const toolRunTree = state.toolRunTrees.get(event.toolCall.toolCallId);
         if (!toolRunTree)
             return;
-        state.toolRunTrees.delete(event.toolCall.toolCallId);
+        const harnessState = state.experimental_harness;
+        if (harnessState != null) {
+            harnessState.toolResults.set(event.toolCall.toolCallId, {
+                toolName: event.toolCall.toolName,
+                output: event.toolOutput,
+            });
+        }
         let outputs;
         let error;
         if (event.recordOutputs !== false) {
             if (event.toolOutput.type === "tool-result") {
                 outputs = { output: event.toolOutput.output };
             }
-            else if (event.toolOutput.type === "tool-error") {
+            else if (isRecord(event.toolOutput) && "error" in event.toolOutput) {
                 const err = event.toolOutput.error;
                 error = err instanceof Error ? err.message : String(err);
             }
@@ -305,6 +375,7 @@ export function LangSmithTelemetry(config) {
         }
         await toolRunTree.end(outputs, error, Math.floor(toolRunTree.start_time + event.toolExecutionMs));
         await toolRunTree.patchRun({ excludeInputs: true });
+        state.toolRunTrees.delete(event.toolCall.toolCallId);
     };
     const onStepFinish = async (event) => {
         const state = invocationsByCallId.get(event.callId);
@@ -314,8 +385,12 @@ export function LangSmithTelemetry(config) {
         const stepRunTree = state.stepRunTrees.get(stepNumber);
         if (!stepRunTree)
             return;
+        const harnessState = state.experimental_harness;
         let outputs = {};
         if (event.recordOutputs !== false) {
+            if (harnessState != null && Array.isArray(event.content)) {
+                harnessState.lastStepContent = [...event.content];
+            }
             outputs = _formatStepOutput(event, traceRawHttp);
             if (processChildLLMRunOutputs) {
                 try {
@@ -326,11 +401,38 @@ export function LangSmithTelemetry(config) {
                 }
             }
         }
+        if (harnessState != null && Array.isArray(event.content)) {
+            harnessState.completedStepMessages.push(convertMessageToTracedFormat({
+                role: "assistant",
+                content: event.content,
+            }));
+            for (const part of event.content) {
+                if (!isRecord(part) ||
+                    part.type !== "tool-call" ||
+                    typeof part.toolCallId !== "string") {
+                    continue;
+                }
+                const toolResult = harnessState.toolResults.get(part.toolCallId);
+                if (toolResult) {
+                    appendToolResultMessage(harnessState, part.toolCallId, toolResult.toolName, toolResult.output);
+                    harnessState.toolResults.delete(part.toolCallId);
+                }
+            }
+        }
         // Set usage metadata
         // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
         setUsageMetadataOnRunTree(event, stepRunTree);
         await stepRunTree.end(outputs);
-        await stepRunTree.patchRun({ excludeInputs: true });
+        const piUsageWorkaround = harnessState?.piAggregateUsageWorkaround;
+        if (piUsageWorkaround != null && !_hasNonzeroUsageMetadata(stepRunTree)) {
+            // Pi reports zero usage for inferred step boundaries and sends the real
+            // aggregate at turn end. Defer the final PATCH so that aggregate can be
+            // attached to the last LLM run without issuing a duplicate update.
+            piUsageWorkaround.deferredStepRunTree = stepRunTree;
+        }
+        else {
+            await stepRunTree.patchRun({ excludeInputs: true });
+        }
         state.stepRunTrees.delete(stepNumber);
     };
     const onEnd = async (event) => {
@@ -340,6 +442,8 @@ export function LangSmithTelemetry(config) {
         if (!state)
             return;
         const { rootRunTree } = state;
+        const harnessState = state.experimental_harness;
+        const piUsageWorkaround = harnessState?.piAggregateUsageWorkaround;
         await finalizeOpenToolRuns(state, { note: "closed on finish" });
         // Ensure any remaining step runs are closed
         const remainingSteps = Array.from(state.stepRunTrees.entries());
@@ -347,7 +451,12 @@ export function LangSmithTelemetry(config) {
             const [stepNumber, stepRt] = remainingSteps[i];
             if (stepRt.end_time == null) {
                 await stepRt.end({ note: "closed on finish" });
-                await stepRt.patchRun({ excludeInputs: true });
+                if (piUsageWorkaround != null && !_hasNonzeroUsageMetadata(stepRt)) {
+                    piUsageWorkaround.deferredStepRunTree = stepRt;
+                }
+                else {
+                    await stepRt.patchRun({ excludeInputs: true });
+                }
             }
             state.stepRunTrees.delete(stepNumber);
         }
@@ -357,9 +466,18 @@ export function LangSmithTelemetry(config) {
             if ("text" in event && event.text != null) {
                 outputs.content = event.text;
             }
-            else if ("content" in event && event.content != null) {
+            else if ("content" in event &&
+                event.content != null &&
+                (!Array.isArray(event.content) || event.content.length > 0)) {
                 outputs.content = event.content;
             }
+            else if (harnessState != null &&
+                harnessState.lastStepContent.length > 0) {
+                // HarnessAgent currently reports an empty root `content` array even
+                // though each completed step contains assistant output. Only the final
+                // step belongs on the root; prior steps have their own child runs.
+                outputs.content = harnessState.lastStepContent;
+            }
             if (outputs.content != null) {
                 outputs.role = "assistant";
             }
@@ -391,18 +509,28 @@ export function LangSmithTelemetry(config) {
                 }
             }
         }
-        // Set aggregated usage on root
-        if ("totalUsage" in event && event.totalUsage != null) {
-            setUsageMetadataOnRunTree(
-            // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
-            { usage: event.totalUsage, providerMetadata: event.providerMetadata }, rootRunTree);
-        }
-        else if ("usage" in event && event.usage != null) {
-            // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
-            setUsageMetadataOnRunTree(event, rootRunTree);
+        // Pi reports zero usage on inferred step boundaries and provides the actual
+        // aggregate only when the turn ends. Keep that usage on an LLM run rather
+        // than incorrectly elevating it to the parent chain. The latest step is the
+        // only accurate place available until Harness Pi exposes per-step usage.
+        const latestStep = piUsageWorkaround?.latestStepRunTree;
+        if (latestStep?.run_type === "llm" &&
+            !_hasNonzeroUsageMetadata(latestStep)) {
+            if ("totalUsage" in event && event.totalUsage != null) {
+                setUsageMetadataOnRunTree(
+                // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
+                { usage: event.totalUsage, providerMetadata: event.providerMetadata }, latestStep);
+            }
+            else if ("usage" in event && event.usage != null) {
+                // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
+                setUsageMetadataOnRunTree(event, latestStep);
+            }
         }
         await rootRunTree.end(outputs);
-        await rootRunTree.patchRun({ excludeInputs: true });
+        await Promise.all([
+            flushPiDeferredStepRun(state),
+            rootRunTree.patchRun({ excludeInputs: true }),
+        ]);
         invocationsByCallId.delete(event.callId);
     };
     const onError = async (payload) => {
@@ -433,6 +561,7 @@ export function LangSmithTelemetry(config) {
             }
             state.stepRunTrees.delete(stepNumber);
         }
+        await flushPiDeferredStepRun(state);
         await rootRunTree.end(undefined, errorMsg);
         await rootRunTree.patchRun({ excludeInputs: true });
         invocationsByCallId.delete(callId);

package/dist/index.cjs

@@ -35,6 +35,6 @@ Object.defineProperty(exports, "UnprocessableEntityError", { enumerable: true, g
 Object.defineProperty(exports, "RateLimitError", { enumerable: true, get: function () { return error_js_1.RateLimitError; } });
 Object.defineProperty(exports, "InternalServerError", { enumerable: true, get: function () { return error_js_1.InternalServerError; } });
 // Update using pnpm bump-version
-exports.__version__ = "0.8.5";
+exports.__version__ = "0.8.6";
 // Metadata key to hide a traced run from LangSmith's Messages View.
 exports.LS_MESSAGE_VIEW_EXCLUDE = "ls_message_view_exclude";

package/dist/index.d.ts

@@ -7,5 +7,5 @@ export { computeRunIdForSecondaryReplica, uuid7, uuid7FromTime, } from "./uuid.j
 export { isTracingEnabled } from "./utils/guard.js";
 export { Cache, PromptCache, type CacheConfig, type CacheMetrics, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
 export { LangsmithError, APIError, APIUserAbortError, APIConnectionError, APIConnectionTimeoutError, BadRequestError, AuthenticationError, PermissionDeniedError, NotFoundError, ConflictError, UnprocessableEntityError, RateLimitError, InternalServerError, } from "./_openapi_client/core/error.js";
-export declare const __version__ = "0.8.5";
+export declare const __version__ = "0.8.6";
 export declare const LS_MESSAGE_VIEW_EXCLUDE: "ls_message_view_exclude";

package/dist/index.js

@@ -7,6 +7,6 @@ export { isTracingEnabled } from "./utils/guard.js";
 export { Cache, PromptCache, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
 export { LangsmithError, APIError, APIUserAbortError, APIConnectionError, APIConnectionTimeoutError, BadRequestError, AuthenticationError, PermissionDeniedError, NotFoundError, ConflictError, UnprocessableEntityError, RateLimitError, InternalServerError, } from "./_openapi_client/core/error.js";
 // Update using pnpm bump-version
-export const __version__ = "0.8.5";
+export const __version__ = "0.8.6";
 // Metadata key to hide a traced run from LangSmith's Messages View.
 export const LS_MESSAGE_VIEW_EXCLUDE = "ls_message_view_exclude";

package/dist/sandbox/command_handle.cjs

@@ -147,7 +147,7 @@ class CommandHandle {
             return;
         const firstResult = await this._stream.next();
         if (firstResult.done) {
-            throw new errors_js_1.LangSmithSandboxOperationError("Command stream ended before 'started' message", "command");
+            throw new errors_js_1.LangSmithStreamEndedBeforeStartedError("Command stream ended before 'started' message");
         }
         const firstMsg = firstResult.value;
         if (firstMsg.type !== "started") {

package/dist/sandbox/command_handle.js

@@ -4,7 +4,7 @@
  *
  * Port of Python's AsyncCommandHandle to TypeScript.
  */
-import { LangSmithSandboxConnectionError, LangSmithSandboxOperationError, } from "./errors.js";
+import { LangSmithSandboxConnectionError, LangSmithSandboxOperationError, LangSmithStreamEndedBeforeStartedError, } from "./errors.js";
 /**
  * Async handle to a running command with streaming output and auto-reconnect.
  *
@@ -144,7 +144,7 @@ export class CommandHandle {
             return;
         const firstResult = await this._stream.next();
         if (firstResult.done) {
-            throw new LangSmithSandboxOperationError("Command stream ended before 'started' message", "command");
+            throw new LangSmithStreamEndedBeforeStartedError("Command stream ended before 'started' message");
         }
         const firstMsg = firstResult.value;
         if (firstMsg.type !== "started") {

package/dist/sandbox/errors.cjs

@@ -7,7 +7,7 @@
  * properties for specific handling when needed.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.LangSmithSandboxServerReloadError = exports.LangSmithCommandTimeoutError = exports.LangSmithSandboxOperationError = exports.LangSmithSandboxNotReadyError = exports.LangSmithDataplaneNotConfiguredError = exports.LangSmithSandboxCreationError = exports.LangSmithResourceCreationError = exports.LangSmithQuotaExceededError = exports.LangSmithValidationError = exports.LangSmithResourceNameConflictError = exports.LangSmithResourceAlreadyExistsError = exports.LangSmithResourceInUseError = exports.LangSmithResourceTimeoutError = exports.LangSmithResourceNotFoundError = exports.LangSmithSandboxConnectionError = exports.LangSmithSandboxAuthenticationError = exports.LangSmithSandboxAPIError = exports.LangSmithSandboxError = void 0;
+exports.LangSmithSandboxServerReloadError = exports.LangSmithStreamEndedBeforeStartedError = exports.LangSmithCommandTimeoutError = exports.LangSmithSandboxOperationError = exports.LangSmithSandboxNotReadyError = exports.LangSmithDataplaneNotConfiguredError = exports.LangSmithSandboxCreationError = exports.LangSmithResourceCreationError = exports.LangSmithQuotaExceededError = exports.LangSmithValidationError = exports.LangSmithResourceNameConflictError = exports.LangSmithResourceAlreadyExistsError = exports.LangSmithResourceInUseError = exports.LangSmithResourceTimeoutError = exports.LangSmithResourceNotFoundError = exports.LangSmithSandboxConnectionError = exports.LangSmithSandboxAuthenticationError = exports.LangSmithSandboxAPIError = exports.LangSmithSandboxError = void 0;
 /**
  * Base exception for sandbox client errors.
  */
@@ -357,6 +357,21 @@ class LangSmithCommandTimeoutError extends LangSmithSandboxOperationError {
     }
 }
 exports.LangSmithCommandTimeoutError = LangSmithCommandTimeoutError;
+/**
+ * Internal: a command WebSocket closed before the guest sent its 'started'
+ * frame (the proxied tunnel was torn down gracefully mid-handshake).
+ *
+ * Marks the idempotently-retryable early close, as distinct from a
+ * command-level failure. Subclasses the operation error so callers that catch
+ * the public type are unaffected.
+ */
+class LangSmithStreamEndedBeforeStartedError extends LangSmithSandboxOperationError {
+    constructor(message) {
+        super(message, "command");
+        this.name = "LangSmithStreamEndedBeforeStartedError";
+    }
+}
+exports.LangSmithStreamEndedBeforeStartedError = LangSmithStreamEndedBeforeStartedError;
 /**
  * Raised when the sandbox server is reloading (close code 1001).
  *

package/dist/sandbox/errors.d.ts

@@ -154,6 +154,17 @@ export declare class LangSmithCommandTimeoutError extends LangSmithSandboxOperat
     timeout?: number;
     constructor(message: string, timeout?: number);
 }
+/**
+ * Internal: a command WebSocket closed before the guest sent its 'started'
+ * frame (the proxied tunnel was torn down gracefully mid-handshake).
+ *
+ * Marks the idempotently-retryable early close, as distinct from a
+ * command-level failure. Subclasses the operation error so callers that catch
+ * the public type are unaffected.
+ */
+export declare class LangSmithStreamEndedBeforeStartedError extends LangSmithSandboxOperationError {
+    constructor(message: string);
+}
 /**
  * Raised when the sandbox server is reloading (close code 1001).
  *

package/dist/sandbox/errors.js

@@ -337,6 +337,20 @@ export class LangSmithCommandTimeoutError extends LangSmithSandboxOperationError
         this.timeout = timeout;
     }
 }
+/**
+ * Internal: a command WebSocket closed before the guest sent its 'started'
+ * frame (the proxied tunnel was torn down gracefully mid-handshake).
+ *
+ * Marks the idempotently-retryable early close, as distinct from a
+ * command-level failure. Subclasses the operation error so callers that catch
+ * the public type are unaffected.
+ */
+export class LangSmithStreamEndedBeforeStartedError extends LangSmithSandboxOperationError {
+    constructor(message) {
+        super(message, "command");
+        this.name = "LangSmithStreamEndedBeforeStartedError";
+    }
+}
 /**
  * Raised when the sandbox server is reloading (close code 1001).
  *

package/dist/sandbox/sandbox.cjs

@@ -4,6 +4,7 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Sandbox = void 0;
+const uuid_js_1 = require("../uuid.cjs");
 const errors_js_1 = require("./errors.cjs");
 const helpers_js_1 = require("./helpers.cjs");
 const command_handle_js_1 = require("./command_handle.cjs");
@@ -229,26 +230,49 @@ class Sandbox {
         const { timeout = 60, env, cwd, shell = "/bin/bash", onStdout, onStderr, commandId, idleTimeout, killOnDisconnect, ttlSeconds, pty, } = options;
         const dataplaneUrl = this.requireDataplaneUrl();
         const clientHeaders = this._client.getDefaultHeaders();
-        const [stream, control] = await (0, ws_execute_js_1.runWsStream)(dataplaneUrl, this._client.getApiKey(), command, {
-            timeout,
-            env,
-            cwd,
-            shell,
-            commandId,
-            idleTimeout,
-            killOnDisconnect,
-            ttlSeconds,
-            pty,
-            ...(Object.keys(clientHeaders).length > 0
-                ? { headers: clientHeaders }
-                : {}),
-        });
-        const handle = new command_handle_js_1.CommandHandle(stream, control, this, {
-            onStdout,
-            onStderr,
-        });
-        await handle._ensureStarted();
-        return handle;
+        // A client-supplied command_id makes execute idempotent: the daemon does
+        // get-or-create keyed on it, so if the tunnel closes before "started" we
+        // can re-issue the same id and reattach to the existing command instead of
+        // spawning a second one.
+        const execCommandId = commandId ?? (0, uuid_js_1.uuid7)();
+        let attempt = 0;
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            const [stream, control] = await (0, ws_execute_js_1.runWsStream)(dataplaneUrl, this._client.getApiKey(), command, {
+                timeout,
+                env,
+                cwd,
+                shell,
+                commandId: execCommandId,
+                idleTimeout,
+                killOnDisconnect,
+                ttlSeconds,
+                pty,
+                ...(Object.keys(clientHeaders).length > 0
+                    ? { headers: clientHeaders }
+                    : {}),
+            });
+            const handle = new command_handle_js_1.CommandHandle(stream, control, this, {
+                onStdout,
+                onStderr,
+            });
+            try {
+                await handle._ensureStarted();
+                return handle;
+            }
+            catch (e) {
+                // Idempotent re-issue (same command_id) after an early close.
+                if (!(e instanceof errors_js_1.LangSmithStreamEndedBeforeStartedError)) {
+                    throw e;
+                }
+                attempt++;
+                if (attempt > command_handle_js_1.CommandHandle.MAX_AUTO_RECONNECTS) {
+                    throw e;
+                }
+                const delay = Math.min(command_handle_js_1.CommandHandle.BACKOFF_BASE * 2 ** (attempt - 1), command_handle_js_1.CommandHandle.BACKOFF_MAX);
+                await new Promise((r) => setTimeout(r, delay * 1000));
+            }
+        }
     }
     /**
      * Execute a command via HTTP POST (blocking).

package/dist/sandbox/sandbox.js

@@ -1,7 +1,8 @@
 /**
  * Sandbox class for interacting with a specific sandbox instance.
  */
-import { LangSmithDataplaneNotConfiguredError } from "./errors.js";
+import { uuid7 } from "../uuid.js";
+import { LangSmithDataplaneNotConfiguredError, LangSmithStreamEndedBeforeStartedError, } from "./errors.js";
 import { handleSandboxHttpError } from "./helpers.js";
 import { CommandHandle } from "./command_handle.js";
 import { reconnectWsStream, runWsStream } from "./ws_execute.js";
@@ -226,26 +227,49 @@ export class Sandbox {
         const { timeout = 60, env, cwd, shell = "/bin/bash", onStdout, onStderr, commandId, idleTimeout, killOnDisconnect, ttlSeconds, pty, } = options;
         const dataplaneUrl = this.requireDataplaneUrl();
         const clientHeaders = this._client.getDefaultHeaders();
-        const [stream, control] = await runWsStream(dataplaneUrl, this._client.getApiKey(), command, {
-            timeout,
-            env,
-            cwd,
-            shell,
-            commandId,
-            idleTimeout,
-            killOnDisconnect,
-            ttlSeconds,
-            pty,
-            ...(Object.keys(clientHeaders).length > 0
-                ? { headers: clientHeaders }
-                : {}),
-        });
-        const handle = new CommandHandle(stream, control, this, {
-            onStdout,
-            onStderr,
-        });
-        await handle._ensureStarted();
-        return handle;
+        // A client-supplied command_id makes execute idempotent: the daemon does
+        // get-or-create keyed on it, so if the tunnel closes before "started" we
+        // can re-issue the same id and reattach to the existing command instead of
+        // spawning a second one.
+        const execCommandId = commandId ?? uuid7();
+        let attempt = 0;
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            const [stream, control] = await runWsStream(dataplaneUrl, this._client.getApiKey(), command, {
+                timeout,
+                env,
+                cwd,
+                shell,
+                commandId: execCommandId,
+                idleTimeout,
+                killOnDisconnect,
+                ttlSeconds,
+                pty,
+                ...(Object.keys(clientHeaders).length > 0
+                    ? { headers: clientHeaders }
+                    : {}),
+            });
+            const handle = new CommandHandle(stream, control, this, {
+                onStdout,
+                onStderr,
+            });
+            try {
+                await handle._ensureStarted();
+                return handle;
+            }
+            catch (e) {
+                // Idempotent re-issue (same command_id) after an early close.
+                if (!(e instanceof LangSmithStreamEndedBeforeStartedError)) {
+                    throw e;
+                }
+                attempt++;
+                if (attempt > CommandHandle.MAX_AUTO_RECONNECTS) {
+                    throw e;
+                }
+                const delay = Math.min(CommandHandle.BACKOFF_BASE * 2 ** (attempt - 1), CommandHandle.BACKOFF_MAX);
+                await new Promise((r) => setTimeout(r, delay * 1000));
+            }
+        }
     }
     /**
      * Execute a command via HTTP POST (blocking).

package/dist/utils/jestlike/globals.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.syncExamplePromises = exports.evaluatorLogFeedbackPromises = exports.testWrapperAsyncLocalStorageInstance = exports.DEFAULT_TEST_CLIENT = void 0;
+exports.evaluatorLogFeedbackPromises = exports.testWrapperAsyncLocalStorageInstance = exports.DEFAULT_TEST_CLIENT = void 0;
 exports.trackingEnabled = trackingEnabled;
 exports._logTestFeedback = _logTestFeedback;
 const node_async_hooks_1 = require("node:async_hooks");
@@ -19,7 +19,6 @@ function trackingEnabled(context) {
     return (0, env_js_2.isEnvTracingEnabled)();
 }
 exports.evaluatorLogFeedbackPromises = new Set();
-exports.syncExamplePromises = new Map();
 function _logTestFeedback(params) {
     const { exampleId, feedback, context, runTree, client, sourceRunId } = params;
     if (trackingEnabled(context)) {
@@ -33,7 +32,7 @@ function _logTestFeedback(params) {
             if (context.project == null) {
                 throw new Error("Could not log feedback to LangSmith: missing project information. Please contact us for help.");
             }
-            await exports.syncExamplePromises.get(exampleId);
+            await context.syncExamplePromises?.get(exampleId);
             await client?.logEvaluationFeedback({
                 evaluatorResponse: feedback,
                 run: runTree,

package/dist/utils/jestlike/globals.d.ts

@@ -20,11 +20,11 @@ export type TestWrapperAsyncLocalStorageData = {
     suiteName: string;
     testRootRunTree?: RunTree;
     setupPromise?: Promise<void>;
+    syncExamplePromises?: Map<string, Promise<Example>>;
 };
 export declare const testWrapperAsyncLocalStorageInstance: AsyncLocalStorage<TestWrapperAsyncLocalStorageData>;
 export declare function trackingEnabled(context: TestWrapperAsyncLocalStorageData): boolean;
 export declare const evaluatorLogFeedbackPromises: Set<unknown>;
-export declare const syncExamplePromises: Map<any, any>;
 export declare function _logTestFeedback(params: {
     exampleId?: string;
     feedback: SimpleEvaluationResult;

package/dist/utils/jestlike/globals.js

@@ -14,7 +14,6 @@ export function trackingEnabled(context) {
     return isEnvTracingEnabled();
 }
 export const evaluatorLogFeedbackPromises = new Set();
-export const syncExamplePromises = new Map();
 export function _logTestFeedback(params) {
     const { exampleId, feedback, context, runTree, client, sourceRunId } = params;
     if (trackingEnabled(context)) {
@@ -28,7 +27,7 @@ export function _logTestFeedback(params) {
             if (context.project == null) {
                 throw new Error("Could not log feedback to LangSmith: missing project information. Please contact us for help.");
             }
-            await syncExamplePromises.get(exampleId);
+            await context.syncExamplePromises?.get(exampleId);
             await client?.logEvaluationFeedback({
                 evaluatorResponse: feedback,
                 run: runTree,

package/dist/utils/jestlike/index.cjs

@@ -300,6 +300,7 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                     },
                     enableTestTracking: experimentConfig?.enableTestTracking,
                     setupPromise,
+                    syncExamplePromises: new Map(),
                 };
                 beforeAll(async () => {
                     const storageValue = await runDatasetSetup(context);
@@ -307,15 +308,14 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                     setupPromiseResolver();
                 });
                 afterAll(async () => {
+                    const examples = await Promise.all(context.syncExamplePromises.values());
                     await Promise.all([
                         client.awaitPendingTraceBatches(),
-                        ...globals_js_1.syncExamplePromises.values(),
                         ...globals_js_1.evaluatorLogFeedbackPromises.values(),
                     ]);
                     if (!(0, globals_js_1.trackingEnabled)(context)) {
                         return;
                     }
-                    const examples = [...globals_js_1.syncExamplePromises.values()];
                     if (examples.length === 0) {
                         return;
                     }
@@ -364,6 +364,7 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                         await client.updateProject(datasetInfo.project.id, {
                             metadata: {
                                 ...suiteMetadata,
+                                dataset_version: finalModifiedAt,
                                 commit,
                                 branch,
                                 dirty,
@@ -546,8 +547,11 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                             // Currently run end time has to be after example modified time
                             // for examples to render properly, so we must modify the example
                             // first before running the test.
-                            if (globals_js_1.syncExamplePromises.get(exampleId) === undefined) {
-                                globals_js_1.syncExamplePromises.set(exampleId, await syncExample({
+                            if (context.syncExamplePromises === undefined) {
+                                throw new Error("Internal error: syncExamplePromises not initialized. This is a bug in the LangSmith SDK — please open an issue at https://github.com/langchain-ai/langsmith-sdk/issues.");
+                            }
+                            if (!context.syncExamplePromises.has(exampleId)) {
+                                const syncPromise = syncExample({
                                     client,
                                     exampleId,
                                     datasetId: dataset.id,
@@ -556,8 +560,10 @@ function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                                     metadata,
                                     split,
                                     createdAt,
-                                }));
+                                });
+                                context.syncExamplePromises.set(exampleId, syncPromise);
                             }
+                            await context.syncExamplePromises.get(exampleId);
                             const traceableOptions = {
                                 reference_example_id: exampleId,
                                 project_name: project.name,

package/dist/utils/jestlike/index.js

@@ -9,7 +9,7 @@ import { execSync } from "child_process";
 import { getCurrentRunTree, traceable } from "../../traceable.js";
 import { randomName } from "../../evaluation/_random_name.js";
 import { toBeRelativeCloseTo, toBeAbsoluteCloseTo, toBeSemanticCloseTo, } from "./matchers.js";
-import { evaluatorLogFeedbackPromises, testWrapperAsyncLocalStorageInstance, _logTestFeedback, syncExamplePromises, trackingEnabled, DEFAULT_TEST_CLIENT, } from "./globals.js";
+import { evaluatorLogFeedbackPromises, testWrapperAsyncLocalStorageInstance, _logTestFeedback, trackingEnabled, DEFAULT_TEST_CLIENT, } from "./globals.js";
 import { wrapExpect } from "./vendor/chain.js";
 import { getEnvironmentVariable, isJsDom } from "../env.js";
 import { STRIP_ANSI_REGEX, TEST_ID_DELIMITER, UUID5_NAMESPACE, } from "./constants.js";
@@ -253,6 +253,7 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                     },
                     enableTestTracking: experimentConfig?.enableTestTracking,
                     setupPromise,
+                    syncExamplePromises: new Map(),
                 };
                 beforeAll(async () => {
                     const storageValue = await runDatasetSetup(context);
@@ -260,15 +261,14 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                     setupPromiseResolver();
                 });
                 afterAll(async () => {
+                    const examples = await Promise.all(context.syncExamplePromises.values());
                     await Promise.all([
                         client.awaitPendingTraceBatches(),
-                        ...syncExamplePromises.values(),
                         ...evaluatorLogFeedbackPromises.values(),
                     ]);
                     if (!trackingEnabled(context)) {
                         return;
                     }
-                    const examples = [...syncExamplePromises.values()];
                     if (examples.length === 0) {
                         return;
                     }
@@ -317,6 +317,7 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                         await client.updateProject(datasetInfo.project.id, {
                             metadata: {
                                 ...suiteMetadata,
+                                dataset_version: finalModifiedAt,
                                 commit,
                                 branch,
                                 dirty,
@@ -499,8 +500,11 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                             // Currently run end time has to be after example modified time
                             // for examples to render properly, so we must modify the example
                             // first before running the test.
-                            if (syncExamplePromises.get(exampleId) === undefined) {
-                                syncExamplePromises.set(exampleId, await syncExample({
+                            if (context.syncExamplePromises === undefined) {
+                                throw new Error("Internal error: syncExamplePromises not initialized. This is a bug in the LangSmith SDK — please open an issue at https://github.com/langchain-ai/langsmith-sdk/issues.");
+                            }
+                            if (!context.syncExamplePromises.has(exampleId)) {
+                                const syncPromise = syncExample({
                                     client,
                                     exampleId,
                                     datasetId: dataset.id,
@@ -509,8 +513,10 @@ export function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
                                     metadata,
                                     split,
                                     createdAt,
-                                }));
+                                });
+                                context.syncExamplePromises.set(exampleId, syncPromise);
                             }
+                            await context.syncExamplePromises.get(exampleId);
                             const traceableOptions = {
                                 reference_example_id: exampleId,
                                 project_name: project.name,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.8.5",
+  "version": "0.8.6",
   "description": "Client library to connect to the LangSmith Observability and Evaluation Platform.",
   "packageManager": "pnpm@10.33.0",
   "files": [
@@ -156,12 +156,18 @@
     "p-queue": "6.6.2"
   },
   "devDependencies": {
+    "@ai-sdk/anthropic": "4.0.18",
+    "@ai-sdk/harness": "1.0.41",
+    "@ai-sdk/harness-claude-code": "1.0.42",
+    "@ai-sdk/harness-pi": "1.0.41",
     "@ai-sdk/openai": "4.0.4",
     "@ai-sdk/provider": "4.0.1",
-    "@ai-sdk/anthropic": "4.0.0-beta.67",
+    "@ai-sdk/sandbox-just-bash": "1.0.40",
+    "@ai-sdk/sandbox-vercel": "1.0.41",
     "@anthropic-ai/claude-agent-sdk": "^0.3.150",
     "@anthropic-ai/sdk": "^0.98.0",
     "@babel/preset-env": "^8.0.2",
+    "@earendil-works/pi-coding-agent": "0.80.10",
     "@faker-js/faker": "^8.4.1",
     "@google/genai": "^2.0.1",
     "@jest/globals": "^30.4.1",
@@ -184,7 +190,7 @@
     "@types/ws": "^8.18.1",
     "@typescript-eslint/eslint-plugin": "^8.62.1",
     "@typescript-eslint/parser": "^8.62.1",
-    "ai": "7.0.0-beta.178",
+    "ai": "7.0.36",
     "babel-jest": "^30.2.0",
     "cross-env": "^10.1.0",
     "dotenv": "^17.3.1",