langsmith 0.8.4 → 0.8.6

package/dist/experimental/vercel/telemetry.cjs

@@ -2,7 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LangSmithTelemetry = LangSmithTelemetry;
 const run_trees_js_1 = require("../../run_trees.cjs");
-const traceable_js_1 = require("../../singletons/traceable.cjs");
+const traceable_js_1 = require("../../traceable.cjs");
 const env_js_1 = require("../../env.cjs");
 const utils_js_1 = require("./utils.cjs");
 const utils_js_2 = require("./utils.cjs");
@@ -58,6 +58,12 @@ function _getLsAgentType(parentRunTree) {
     }
     return "root";
 }
+function _hasNonzeroUsageMetadata(runTree) {
+    const usageMetadata = runTree.extra?.metadata?.usage_metadata;
+    if (!(0, types_js_1.isRecord)(usageMetadata))
+        return false;
+    return ["input_tokens", "output_tokens", "total_tokens"].some((key) => typeof usageMetadata[key] === "number" && usageMetadata[key] > 0);
+}
 /**
  * Creates a LangSmith `Telemetry` for the Vercel AI SDK.
  *
@@ -82,6 +88,31 @@ function _getLsAgentType(parentRunTree) {
  */
 function LangSmithTelemetry(config) {
     const { name: customName, runType = "chain", metadata: customMetadata, tags: customTags, client, projectName, processInputs, processOutputs, processChildLLMRunInputs, processChildLLMRunOutputs, traceResponseMetadata, traceRawHttp, tracingEnabled, extra: customExtra, } = config ?? {};
+    function formatToolResultMessage(toolCallId, toolName, toolOutput) {
+        const rawOutput = (0, types_js_1.isRecord)(toolOutput)
+            ? toolOutput.type === "tool-result"
+                ? toolOutput.output
+                : "error" in toolOutput
+                    ? toolOutput.error
+                    : toolOutput
+            : toolOutput;
+        const artifact = rawOutput instanceof Error ? rawOutput.message : rawOutput;
+        const content = typeof artifact === "string"
+            ? artifact
+            : (JSON.stringify(artifact) ?? String(artifact));
+        return {
+            role: "tool",
+            content,
+            tool_call_id: toolCallId,
+            name: toolName,
+            artifact,
+        };
+    }
+    function appendToolResultMessage(harnessState, toolCallId, toolName, toolOutput) {
+        const message = formatToolResultMessage(toolCallId, toolName, toolOutput);
+        harnessState.completedStepMessages.push(message);
+        return message;
+    }
     function getOpenStepOrRoot(state) {
         let openStep;
         state.stepRunTrees.forEach((stepRt) => {
@@ -91,6 +122,16 @@ function LangSmithTelemetry(config) {
         });
         return openStep ?? state.rootRunTree;
     }
+    async function flushPiDeferredStepRun(state) {
+        const workaround = state.experimental_harness?.piAggregateUsageWorkaround;
+        if (workaround == null)
+            return;
+        const stepRunTree = workaround.deferredStepRunTree;
+        if (stepRunTree == null)
+            return;
+        workaround.deferredStepRunTree = undefined;
+        await stepRunTree.patchRun({ excludeInputs: true });
+    }
     async function finalizeOpenToolRuns(state, opts) {
         const entries = Array.from(state.toolRunTrees.entries());
         for (let i = 0; i < entries.length; i++) {
@@ -161,7 +202,6 @@ function LangSmithTelemetry(config) {
                     ai_sdk_method: event.operationId,
                     ls_agent_type: _getLsAgentType(parentRunTree),
                     ls_model_name: event.modelId,
-                    ls_provider: event.provider,
                     ls_integration: "vercel-ai-sdk-telemetry",
                 },
             },
@@ -181,17 +221,29 @@ function LangSmithTelemetry(config) {
             rootRunTree,
             stepRunTrees: new Map(),
             toolRunTrees: new Map(),
+            experimental_harness: event.operationId === "ai.harness"
+                ? {
+                    lastStepContent: [],
+                    completedStepMessages: [],
+                    toolResults: new Map(),
+                    piAggregateUsageWorkaround: event.provider === "pi" ? {} : undefined,
+                }
+                : undefined,
         });
     };
     const onStepStart = async (event) => {
         const state = invocationsByCallId.get(event.callId);
         if (!state)
             return;
+        await flushPiDeferredStepRun(state);
         const stepNumber = event.stepNumber ?? 0;
         let inputs = {};
         if (event.recordInputs !== false) {
             if ("messages" in event && event.messages != null) {
-                inputs.messages = _formatMessages(event.messages);
+                inputs.messages = [
+                    ..._formatMessages(event.messages),
+                    ...(state.experimental_harness?.completedStepMessages ?? []),
+                ];
             }
             if ("runtimeContext" in event && event.runtimeContext != null) {
                 inputs.runtimeContext = event.runtimeContext;
@@ -212,9 +264,18 @@ function LangSmithTelemetry(config) {
             name: event.provider,
             run_type: "llm",
             inputs,
-            extra: { metadata: { step_number: stepNumber } },
+            extra: {
+                metadata: {
+                    step_number: stepNumber,
+                    ls_model_name: event.modelId,
+                },
+            },
         });
         state.stepRunTrees.set(stepNumber, stepRunTree);
+        const piUsageWorkaround = state.experimental_harness?.piAggregateUsageWorkaround;
+        if (piUsageWorkaround != null) {
+            piUsageWorkaround.latestStepRunTree = stepRunTree;
+        }
         await stepRunTree.postRun();
     };
     const onLanguageModelCallStart = async (event) => {
@@ -254,22 +315,25 @@ function LangSmithTelemetry(config) {
         const state = invocationsByCallId.get(event.callId);
         if (!state)
             return;
-        const parentRunTree = getOpenStepOrRoot(state);
+        const parentRunTree = state.rootRunTree;
         let inputs = {};
         if (event.recordInputs !== false) {
-            if ((0, types_js_1.isRecord)(event.toolCall.input)) {
-                inputs = { ...event.toolCall.input };
+            let toolCallInput = event.toolCall.input;
+            if (typeof toolCallInput === "string") {
+                try {
+                    toolCallInput = JSON.parse(toolCallInput);
+                }
+                catch {
+                    // The tool argument itself may be a plain string rather than JSON.
+                }
             }
-            else if (typeof event.toolCall.input !== "undefined") {
-                inputs = { input: event.toolCall.input };
+            if ((0, types_js_1.isRecord)(toolCallInput)) {
+                inputs = { ...toolCallInput };
             }
-            if ("toolContext" in event && event.toolContext != null) {
-                inputs.toolContext = event.toolContext;
+            else if (toolCallInput !== undefined) {
+                inputs = { input: toolCallInput };
             }
         }
-        else {
-            inputs = {};
-        }
         const toolRunTree = parentRunTree.createChild({
             name: event.toolCall.toolName,
             run_type: "tool",
@@ -291,14 +355,20 @@ function LangSmithTelemetry(config) {
         const toolRunTree = state.toolRunTrees.get(event.toolCall.toolCallId);
         if (!toolRunTree)
             return;
-        state.toolRunTrees.delete(event.toolCall.toolCallId);
+        const harnessState = state.experimental_harness;
+        if (harnessState != null) {
+            harnessState.toolResults.set(event.toolCall.toolCallId, {
+                toolName: event.toolCall.toolName,
+                output: event.toolOutput,
+            });
+        }
         let outputs;
         let error;
         if (event.recordOutputs !== false) {
             if (event.toolOutput.type === "tool-result") {
                 outputs = { output: event.toolOutput.output };
             }
-            else if (event.toolOutput.type === "tool-error") {
+            else if ((0, types_js_1.isRecord)(event.toolOutput) && "error" in event.toolOutput) {
                 const err = event.toolOutput.error;
                 error = err instanceof Error ? err.message : String(err);
             }
@@ -308,6 +378,7 @@ function LangSmithTelemetry(config) {
         }
         await toolRunTree.end(outputs, error, Math.floor(toolRunTree.start_time + event.toolExecutionMs));
         await toolRunTree.patchRun({ excludeInputs: true });
+        state.toolRunTrees.delete(event.toolCall.toolCallId);
     };
     const onStepFinish = async (event) => {
         const state = invocationsByCallId.get(event.callId);
@@ -317,8 +388,12 @@ function LangSmithTelemetry(config) {
         const stepRunTree = state.stepRunTrees.get(stepNumber);
         if (!stepRunTree)
             return;
+        const harnessState = state.experimental_harness;
         let outputs = {};
         if (event.recordOutputs !== false) {
+            if (harnessState != null && Array.isArray(event.content)) {
+                harnessState.lastStepContent = [...event.content];
+            }
             outputs = _formatStepOutput(event, traceRawHttp);
             if (processChildLLMRunOutputs) {
                 try {
@@ -329,11 +404,38 @@ function LangSmithTelemetry(config) {
                 }
             }
         }
+        if (harnessState != null && Array.isArray(event.content)) {
+            harnessState.completedStepMessages.push((0, utils_js_1.convertMessageToTracedFormat)({
+                role: "assistant",
+                content: event.content,
+            }));
+            for (const part of event.content) {
+                if (!(0, types_js_1.isRecord)(part) ||
+                    part.type !== "tool-call" ||
+                    typeof part.toolCallId !== "string") {
+                    continue;
+                }
+                const toolResult = harnessState.toolResults.get(part.toolCallId);
+                if (toolResult) {
+                    appendToolResultMessage(harnessState, part.toolCallId, toolResult.toolName, toolResult.output);
+                    harnessState.toolResults.delete(part.toolCallId);
+                }
+            }
+        }
         // Set usage metadata
         // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
         (0, utils_js_2.setUsageMetadataOnRunTree)(event, stepRunTree);
         await stepRunTree.end(outputs);
-        await stepRunTree.patchRun({ excludeInputs: true });
+        const piUsageWorkaround = harnessState?.piAggregateUsageWorkaround;
+        if (piUsageWorkaround != null && !_hasNonzeroUsageMetadata(stepRunTree)) {
+            // Pi reports zero usage for inferred step boundaries and sends the real
+            // aggregate at turn end. Defer the final PATCH so that aggregate can be
+            // attached to the last LLM run without issuing a duplicate update.
+            piUsageWorkaround.deferredStepRunTree = stepRunTree;
+        }
+        else {
+            await stepRunTree.patchRun({ excludeInputs: true });
+        }
         state.stepRunTrees.delete(stepNumber);
     };
     const onEnd = async (event) => {
@@ -343,6 +445,8 @@ function LangSmithTelemetry(config) {
         if (!state)
             return;
         const { rootRunTree } = state;
+        const harnessState = state.experimental_harness;
+        const piUsageWorkaround = harnessState?.piAggregateUsageWorkaround;
         await finalizeOpenToolRuns(state, { note: "closed on finish" });
         // Ensure any remaining step runs are closed
         const remainingSteps = Array.from(state.stepRunTrees.entries());
@@ -350,7 +454,12 @@ function LangSmithTelemetry(config) {
             const [stepNumber, stepRt] = remainingSteps[i];
             if (stepRt.end_time == null) {
                 await stepRt.end({ note: "closed on finish" });
-                await stepRt.patchRun({ excludeInputs: true });
+                if (piUsageWorkaround != null && !_hasNonzeroUsageMetadata(stepRt)) {
+                    piUsageWorkaround.deferredStepRunTree = stepRt;
+                }
+                else {
+                    await stepRt.patchRun({ excludeInputs: true });
+                }
             }
             state.stepRunTrees.delete(stepNumber);
         }
@@ -360,9 +469,18 @@ function LangSmithTelemetry(config) {
             if ("text" in event && event.text != null) {
                 outputs.content = event.text;
             }
-            else if ("content" in event && event.content != null) {
+            else if ("content" in event &&
+                event.content != null &&
+                (!Array.isArray(event.content) || event.content.length > 0)) {
                 outputs.content = event.content;
             }
+            else if (harnessState != null &&
+                harnessState.lastStepContent.length > 0) {
+                // HarnessAgent currently reports an empty root `content` array even
+                // though each completed step contains assistant output. Only the final
+                // step belongs on the root; prior steps have their own child runs.
+                outputs.content = harnessState.lastStepContent;
+            }
             if (outputs.content != null) {
                 outputs.role = "assistant";
             }
@@ -394,18 +512,28 @@ function LangSmithTelemetry(config) {
                 }
             }
         }
-        // Set aggregated usage on root
-        if ("totalUsage" in event && event.totalUsage != null) {
-            (0, utils_js_2.setUsageMetadataOnRunTree)(
-            // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
-            { usage: event.totalUsage, providerMetadata: event.providerMetadata }, rootRunTree);
-        }
-        else if ("usage" in event && event.usage != null) {
-            // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
-            (0, utils_js_2.setUsageMetadataOnRunTree)(event, rootRunTree);
+        // Pi reports zero usage on inferred step boundaries and provides the actual
+        // aggregate only when the turn ends. Keep that usage on an LLM run rather
+        // than incorrectly elevating it to the parent chain. The latest step is the
+        // only accurate place available until Harness Pi exposes per-step usage.
+        const latestStep = piUsageWorkaround?.latestStepRunTree;
+        if (latestStep?.run_type === "llm" &&
+            !_hasNonzeroUsageMetadata(latestStep)) {
+            if ("totalUsage" in event && event.totalUsage != null) {
+                (0, utils_js_2.setUsageMetadataOnRunTree)(
+                // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
+                { usage: event.totalUsage, providerMetadata: event.providerMetadata }, latestStep);
+            }
+            else if ("usage" in event && event.usage != null) {
+                // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
+                (0, utils_js_2.setUsageMetadataOnRunTree)(event, latestStep);
+            }
         }
         await rootRunTree.end(outputs);
-        await rootRunTree.patchRun({ excludeInputs: true });
+        await Promise.all([
+            flushPiDeferredStepRun(state),
+            rootRunTree.patchRun({ excludeInputs: true }),
+        ]);
         invocationsByCallId.delete(event.callId);
     };
     const onError = async (payload) => {
@@ -436,6 +564,7 @@ function LangSmithTelemetry(config) {
             }
             state.stepRunTrees.delete(stepNumber);
         }
+        await flushPiDeferredStepRun(state);
         await rootRunTree.end(undefined, errorMsg);
         await rootRunTree.patchRun({ excludeInputs: true });
         invocationsByCallId.delete(callId);

package/dist/experimental/vercel/telemetry.js

@@ -1,5 +1,5 @@
 import { isRunTree, RunTree } from "../../run_trees.js";
-import { getCurrentRunTree, withRunTree } from "../../singletons/traceable.js";
+import { getCurrentRunTree, withRunTree } from "../../traceable.js";
 import { isEnvTracingEnabled } from "../../env.js";
 import { convertMessageToTracedFormat } from "./utils.js";
 import { setUsageMetadataOnRunTree } from "./utils.js";
@@ -55,6 +55,12 @@ function _getLsAgentType(parentRunTree) {
     }
     return "root";
 }
+function _hasNonzeroUsageMetadata(runTree) {
+    const usageMetadata = runTree.extra?.metadata?.usage_metadata;
+    if (!isRecord(usageMetadata))
+        return false;
+    return ["input_tokens", "output_tokens", "total_tokens"].some((key) => typeof usageMetadata[key] === "number" && usageMetadata[key] > 0);
+}
 /**
  * Creates a LangSmith `Telemetry` for the Vercel AI SDK.
  *
@@ -79,6 +85,31 @@ function _getLsAgentType(parentRunTree) {
  */
 export function LangSmithTelemetry(config) {
     const { name: customName, runType = "chain", metadata: customMetadata, tags: customTags, client, projectName, processInputs, processOutputs, processChildLLMRunInputs, processChildLLMRunOutputs, traceResponseMetadata, traceRawHttp, tracingEnabled, extra: customExtra, } = config ?? {};
+    function formatToolResultMessage(toolCallId, toolName, toolOutput) {
+        const rawOutput = isRecord(toolOutput)
+            ? toolOutput.type === "tool-result"
+                ? toolOutput.output
+                : "error" in toolOutput
+                    ? toolOutput.error
+                    : toolOutput
+            : toolOutput;
+        const artifact = rawOutput instanceof Error ? rawOutput.message : rawOutput;
+        const content = typeof artifact === "string"
+            ? artifact
+            : (JSON.stringify(artifact) ?? String(artifact));
+        return {
+            role: "tool",
+            content,
+            tool_call_id: toolCallId,
+            name: toolName,
+            artifact,
+        };
+    }
+    function appendToolResultMessage(harnessState, toolCallId, toolName, toolOutput) {
+        const message = formatToolResultMessage(toolCallId, toolName, toolOutput);
+        harnessState.completedStepMessages.push(message);
+        return message;
+    }
     function getOpenStepOrRoot(state) {
         let openStep;
         state.stepRunTrees.forEach((stepRt) => {
@@ -88,6 +119,16 @@ export function LangSmithTelemetry(config) {
         });
         return openStep ?? state.rootRunTree;
     }
+    async function flushPiDeferredStepRun(state) {
+        const workaround = state.experimental_harness?.piAggregateUsageWorkaround;
+        if (workaround == null)
+            return;
+        const stepRunTree = workaround.deferredStepRunTree;
+        if (stepRunTree == null)
+            return;
+        workaround.deferredStepRunTree = undefined;
+        await stepRunTree.patchRun({ excludeInputs: true });
+    }
     async function finalizeOpenToolRuns(state, opts) {
         const entries = Array.from(state.toolRunTrees.entries());
         for (let i = 0; i < entries.length; i++) {
@@ -158,7 +199,6 @@ export function LangSmithTelemetry(config) {
                     ai_sdk_method: event.operationId,
                     ls_agent_type: _getLsAgentType(parentRunTree),
                     ls_model_name: event.modelId,
-                    ls_provider: event.provider,
                     ls_integration: "vercel-ai-sdk-telemetry",
                 },
             },
@@ -178,17 +218,29 @@ export function LangSmithTelemetry(config) {
             rootRunTree,
             stepRunTrees: new Map(),
             toolRunTrees: new Map(),
+            experimental_harness: event.operationId === "ai.harness"
+                ? {
+                    lastStepContent: [],
+                    completedStepMessages: [],
+                    toolResults: new Map(),
+                    piAggregateUsageWorkaround: event.provider === "pi" ? {} : undefined,
+                }
+                : undefined,
         });
     };
     const onStepStart = async (event) => {
         const state = invocationsByCallId.get(event.callId);
         if (!state)
             return;
+        await flushPiDeferredStepRun(state);
         const stepNumber = event.stepNumber ?? 0;
         let inputs = {};
         if (event.recordInputs !== false) {
             if ("messages" in event && event.messages != null) {
-                inputs.messages = _formatMessages(event.messages);
+                inputs.messages = [
+                    ..._formatMessages(event.messages),
+                    ...(state.experimental_harness?.completedStepMessages ?? []),
+                ];
             }
             if ("runtimeContext" in event && event.runtimeContext != null) {
                 inputs.runtimeContext = event.runtimeContext;
@@ -209,9 +261,18 @@ export function LangSmithTelemetry(config) {
             name: event.provider,
             run_type: "llm",
             inputs,
-            extra: { metadata: { step_number: stepNumber } },
+            extra: {
+                metadata: {
+                    step_number: stepNumber,
+                    ls_model_name: event.modelId,
+                },
+            },
         });
         state.stepRunTrees.set(stepNumber, stepRunTree);
+        const piUsageWorkaround = state.experimental_harness?.piAggregateUsageWorkaround;
+        if (piUsageWorkaround != null) {
+            piUsageWorkaround.latestStepRunTree = stepRunTree;
+        }
         await stepRunTree.postRun();
     };
     const onLanguageModelCallStart = async (event) => {
@@ -251,22 +312,25 @@ export function LangSmithTelemetry(config) {
         const state = invocationsByCallId.get(event.callId);
         if (!state)
             return;
-        const parentRunTree = getOpenStepOrRoot(state);
+        const parentRunTree = state.rootRunTree;
         let inputs = {};
         if (event.recordInputs !== false) {
-            if (isRecord(event.toolCall.input)) {
-                inputs = { ...event.toolCall.input };
+            let toolCallInput = event.toolCall.input;
+            if (typeof toolCallInput === "string") {
+                try {
+                    toolCallInput = JSON.parse(toolCallInput);
+                }
+                catch {
+                    // The tool argument itself may be a plain string rather than JSON.
+                }
             }
-            else if (typeof event.toolCall.input !== "undefined") {
-                inputs = { input: event.toolCall.input };
+            if (isRecord(toolCallInput)) {
+                inputs = { ...toolCallInput };
             }
-            if ("toolContext" in event && event.toolContext != null) {
-                inputs.toolContext = event.toolContext;
+            else if (toolCallInput !== undefined) {
+                inputs = { input: toolCallInput };
             }
         }
-        else {
-            inputs = {};
-        }
         const toolRunTree = parentRunTree.createChild({
             name: event.toolCall.toolName,
             run_type: "tool",
@@ -288,14 +352,20 @@ export function LangSmithTelemetry(config) {
         const toolRunTree = state.toolRunTrees.get(event.toolCall.toolCallId);
         if (!toolRunTree)
             return;
-        state.toolRunTrees.delete(event.toolCall.toolCallId);
+        const harnessState = state.experimental_harness;
+        if (harnessState != null) {
+            harnessState.toolResults.set(event.toolCall.toolCallId, {
+                toolName: event.toolCall.toolName,
+                output: event.toolOutput,
+            });
+        }
         let outputs;
         let error;
         if (event.recordOutputs !== false) {
             if (event.toolOutput.type === "tool-result") {
                 outputs = { output: event.toolOutput.output };
             }
-            else if (event.toolOutput.type === "tool-error") {
+            else if (isRecord(event.toolOutput) && "error" in event.toolOutput) {
                 const err = event.toolOutput.error;
                 error = err instanceof Error ? err.message : String(err);
             }
@@ -305,6 +375,7 @@ export function LangSmithTelemetry(config) {
         }
         await toolRunTree.end(outputs, error, Math.floor(toolRunTree.start_time + event.toolExecutionMs));
         await toolRunTree.patchRun({ excludeInputs: true });
+        state.toolRunTrees.delete(event.toolCall.toolCallId);
     };
     const onStepFinish = async (event) => {
         const state = invocationsByCallId.get(event.callId);
@@ -314,8 +385,12 @@ export function LangSmithTelemetry(config) {
         const stepRunTree = state.stepRunTrees.get(stepNumber);
         if (!stepRunTree)
             return;
+        const harnessState = state.experimental_harness;
         let outputs = {};
         if (event.recordOutputs !== false) {
+            if (harnessState != null && Array.isArray(event.content)) {
+                harnessState.lastStepContent = [...event.content];
+            }
             outputs = _formatStepOutput(event, traceRawHttp);
             if (processChildLLMRunOutputs) {
                 try {
@@ -326,11 +401,38 @@ export function LangSmithTelemetry(config) {
                 }
             }
         }
+        if (harnessState != null && Array.isArray(event.content)) {
+            harnessState.completedStepMessages.push(convertMessageToTracedFormat({
+                role: "assistant",
+                content: event.content,
+            }));
+            for (const part of event.content) {
+                if (!isRecord(part) ||
+                    part.type !== "tool-call" ||
+                    typeof part.toolCallId !== "string") {
+                    continue;
+                }
+                const toolResult = harnessState.toolResults.get(part.toolCallId);
+                if (toolResult) {
+                    appendToolResultMessage(harnessState, part.toolCallId, toolResult.toolName, toolResult.output);
+                    harnessState.toolResults.delete(part.toolCallId);
+                }
+            }
+        }
         // Set usage metadata
         // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
         setUsageMetadataOnRunTree(event, stepRunTree);
         await stepRunTree.end(outputs);
-        await stepRunTree.patchRun({ excludeInputs: true });
+        const piUsageWorkaround = harnessState?.piAggregateUsageWorkaround;
+        if (piUsageWorkaround != null && !_hasNonzeroUsageMetadata(stepRunTree)) {
+            // Pi reports zero usage for inferred step boundaries and sends the real
+            // aggregate at turn end. Defer the final PATCH so that aggregate can be
+            // attached to the last LLM run without issuing a duplicate update.
+            piUsageWorkaround.deferredStepRunTree = stepRunTree;
+        }
+        else {
+            await stepRunTree.patchRun({ excludeInputs: true });
+        }
         state.stepRunTrees.delete(stepNumber);
     };
     const onEnd = async (event) => {
@@ -340,6 +442,8 @@ export function LangSmithTelemetry(config) {
         if (!state)
             return;
         const { rootRunTree } = state;
+        const harnessState = state.experimental_harness;
+        const piUsageWorkaround = harnessState?.piAggregateUsageWorkaround;
         await finalizeOpenToolRuns(state, { note: "closed on finish" });
         // Ensure any remaining step runs are closed
         const remainingSteps = Array.from(state.stepRunTrees.entries());
@@ -347,7 +451,12 @@ export function LangSmithTelemetry(config) {
             const [stepNumber, stepRt] = remainingSteps[i];
             if (stepRt.end_time == null) {
                 await stepRt.end({ note: "closed on finish" });
-                await stepRt.patchRun({ excludeInputs: true });
+                if (piUsageWorkaround != null && !_hasNonzeroUsageMetadata(stepRt)) {
+                    piUsageWorkaround.deferredStepRunTree = stepRt;
+                }
+                else {
+                    await stepRt.patchRun({ excludeInputs: true });
+                }
             }
             state.stepRunTrees.delete(stepNumber);
         }
@@ -357,9 +466,18 @@ export function LangSmithTelemetry(config) {
             if ("text" in event && event.text != null) {
                 outputs.content = event.text;
             }
-            else if ("content" in event && event.content != null) {
+            else if ("content" in event &&
+                event.content != null &&
+                (!Array.isArray(event.content) || event.content.length > 0)) {
                 outputs.content = event.content;
             }
+            else if (harnessState != null &&
+                harnessState.lastStepContent.length > 0) {
+                // HarnessAgent currently reports an empty root `content` array even
+                // though each completed step contains assistant output. Only the final
+                // step belongs on the root; prior steps have their own child runs.
+                outputs.content = harnessState.lastStepContent;
+            }
             if (outputs.content != null) {
                 outputs.role = "assistant";
             }
@@ -391,18 +509,28 @@ export function LangSmithTelemetry(config) {
                 }
             }
         }
-        // Set aggregated usage on root
-        if ("totalUsage" in event && event.totalUsage != null) {
-            setUsageMetadataOnRunTree(
-            // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
-            { usage: event.totalUsage, providerMetadata: event.providerMetadata }, rootRunTree);
-        }
-        else if ("usage" in event && event.usage != null) {
-            // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
-            setUsageMetadataOnRunTree(event, rootRunTree);
+        // Pi reports zero usage on inferred step boundaries and provides the actual
+        // aggregate only when the turn ends. Keep that usage on an LLM run rather
+        // than incorrectly elevating it to the parent chain. The latest step is the
+        // only accurate place available until Harness Pi exposes per-step usage.
+        const latestStep = piUsageWorkaround?.latestStepRunTree;
+        if (latestStep?.run_type === "llm" &&
+            !_hasNonzeroUsageMetadata(latestStep)) {
+            if ("totalUsage" in event && event.totalUsage != null) {
+                setUsageMetadataOnRunTree(
+                // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
+                { usage: event.totalUsage, providerMetadata: event.providerMetadata }, latestStep);
+            }
+            else if ("usage" in event && event.usage != null) {
+                // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
+                setUsageMetadataOnRunTree(event, latestStep);
+            }
         }
         await rootRunTree.end(outputs);
-        await rootRunTree.patchRun({ excludeInputs: true });
+        await Promise.all([
+            flushPiDeferredStepRun(state),
+            rootRunTree.patchRun({ excludeInputs: true }),
+        ]);
         invocationsByCallId.delete(event.callId);
     };
     const onError = async (payload) => {
@@ -433,6 +561,7 @@ export function LangSmithTelemetry(config) {
             }
             state.stepRunTrees.delete(stepNumber);
         }
+        await flushPiDeferredStepRun(state);
         await rootRunTree.end(undefined, errorMsg);
         await rootRunTree.patchRun({ excludeInputs: true });
         invocationsByCallId.delete(callId);