@agent-native/core 0.109.0 → 0.109.2

package/corpus/core/src/observability/traces.ts

@@ -26,6 +26,16 @@ function costUsdFromCenticents(value: number): number {
   return Math.round((value / 10_000) * 1_000_000) / 1_000_000;
 }
 
+const MAX_TRACKED_GENERATION_TOOL_CALLS = 50;
+
+type GenerationToolCall = {
+  name: string;
+  started_offset_ms: number;
+  duration_ms: number;
+  status: "success" | "error";
+  error_class: "tool_error" | "legacy_inferred_error" | "interrupted" | null;
+};
+
 function emitLlmGenerationTrackingEvent(args: {
   runId: string;
   threadId: string | null;
@@ -45,6 +55,15 @@ function emitLlmGenerationTrackingEvent(args: {
   toolCalls: number;
   successfulTools: number;
   failedTools: number;
+  tools: GenerationToolCall[];
+  toolsTruncated: boolean;
+  delegation?: {
+    protocol: "a2a" | "mcp";
+    callerApp?: string;
+    taskId?: string;
+    parentRunId?: string;
+    parentTurnId?: string;
+  };
   createdAt: number;
   experimentAssignments?: Array<{
     experimentId: string;
@@ -77,6 +96,14 @@ function emitLlmGenerationTrackingEvent(args: {
     tool_calls: args.toolCalls,
     successful_tools: args.successfulTools,
     failed_tools: args.failedTools,
+    tools: args.tools,
+    tools_truncated: args.toolsTruncated,
+    delegated: args.delegation ? true : undefined,
+    delegation_protocol: args.delegation?.protocol,
+    caller_app: args.delegation?.callerApp,
+    a2a_task_id: args.delegation?.taskId,
+    parent_run_id: args.delegation?.parentRunId,
+    parent_turn_id: args.delegation?.parentTurnId,
     model_selection_source: args.modelSelectionSource,
     created_at: new Date(args.createdAt).toISOString(),
     created_at_ms: args.createdAt,
@@ -191,6 +218,7 @@ export async function instrumentAgentLoop(opts: {
     send: (event: AgentChatEvent) => void;
     signal: AbortSignal;
     providerOptions?: any;
+    runId?: string;
   }) => Promise<AgentLoopUsage>;
   loopOpts: {
     engine: any;
@@ -202,6 +230,7 @@ export async function instrumentAgentLoop(opts: {
     send: (event: AgentChatEvent) => void;
     signal: AbortSignal;
     providerOptions?: any;
+    runId?: string;
   };
   runId: string;
   threadId: string | null;
@@ -217,6 +246,13 @@ export async function instrumentAgentLoop(opts: {
     variantId: string;
   }>;
   modelSelectionSource?: string;
+  delegation?: {
+    protocol: "a2a" | "mcp";
+    callerApp?: string;
+    taskId?: string;
+    parentRunId?: string;
+    parentTurnId?: string;
+  };
   /** Raw user-authored message before prompt/context enrichment. */
   sentimentInput?: string;
   classifyError?: (error: unknown) =>
@@ -271,6 +307,7 @@ export async function instrumentAgentLoop(opts: {
     number,
     {
       spanId: string;
+      callId?: string;
       startMs: number;
       toolName: string;
       input: AgentToolInput;
@@ -278,12 +315,11 @@ export async function instrumentAgentLoop(opts: {
       endResult?: { status: "success" | "error"; errorMessage: string | null };
     }
   >();
-  // Secondary index: tool name → FIFO queue of pending invocation counters.
-  // tool_start/tool_done events carry only the tool name (no call id), so to
-  // pair starts and dones correctly when the agent runs concurrent calls to the
-  // same tool name (read-only / parallelSafe batches via Promise.all), we keep a
-  // queue per name and match each done to the OLDEST still-pending start.
+  // Secondary index for legacy emitters without call ids. Current tool events
+  // are paired by id first; same-name FIFO remains as a compatibility fallback.
   const toolNameToCounters = new Map<string, number[]>();
+  const toolCallIdToCounter = new Map<string, number>();
+  const generationToolCalls = new Map<number, GenerationToolCall>();
 
   let toolCallCount = 0;
   let successfulTools = 0;
@@ -304,6 +340,7 @@ export async function instrumentAgentLoop(opts: {
         // microtask gap by recording the span on the pending entry when ready.
         const entry: {
           spanId: string;
+          callId?: string;
           startMs: number;
           toolName: string;
           input: AgentToolInput;
@@ -316,12 +353,14 @@ export async function instrumentAgentLoop(opts: {
           };
         } = {
           spanId: sid,
+          ...(event.id ? { callId: event.id } : {}),
           startMs: Date.now(),
           toolName: event.tool,
           input: event.input,
           otelSpan: null,
         };
         pendingTools.set(counter, entry);
+        if (event.id) toolCallIdToCounter.set(event.id, counter);
         void startAgentSpan("tool.call", {
           "tool.name": event.tool,
         }).then((span) => {
@@ -343,16 +382,39 @@ export async function instrumentAgentLoop(opts: {
         else toolNameToCounters.set(event.tool, [counter]);
       } else if (event.type === "tool_done") {
         const queue = toolNameToCounters.get(event.tool);
-        const counter = queue?.shift();
+        const counterFromId = event.id
+          ? toolCallIdToCounter.get(event.id)
+          : undefined;
+        const legacyQueueIndex =
+          event.id && counterFromId === undefined && queue
+            ? queue.findIndex(
+                (candidate) => !pendingTools.get(candidate)?.callId,
+              )
+            : -1;
+        const counter =
+          counterFromId ??
+          (event.id
+            ? legacyQueueIndex >= 0
+              ? queue?.[legacyQueueIndex]
+              : undefined
+            : queue?.shift());
         const pending =
           counter !== undefined ? pendingTools.get(counter) : undefined;
         if (counter !== undefined) {
           pendingTools.delete(counter);
+          if (pending?.callId) toolCallIdToCounter.delete(pending.callId);
+          if ((counterFromId !== undefined || legacyQueueIndex >= 0) && queue) {
+            const queueIndex = queue.indexOf(counter);
+            if (queueIndex >= 0) queue.splice(queueIndex, 1);
+          }
           if (queue && queue.length === 0)
             toolNameToCounters.delete(event.tool);
         }
         toolCallCount++;
 
+        const finishedAt = Date.now();
+
+        const explicitError = event.isError === true;
         const isError =
           typeof event.isError === "boolean"
             ? event.isError
@@ -362,6 +424,24 @@ export async function instrumentAgentLoop(opts: {
         if (isError) failedTools++;
         else successfulTools++;
 
+        if (
+          counter !== undefined &&
+          counter < MAX_TRACKED_GENERATION_TOOL_CALLS &&
+          pending
+        ) {
+          generationToolCalls.set(counter, {
+            name: pending.toolName,
+            started_offset_ms: Math.max(0, pending.startMs - runStart),
+            duration_ms: Math.max(0, finishedAt - pending.startMs),
+            status: isError ? "error" : "success",
+            error_class: !isError
+              ? null
+              : explicitError
+                ? "tool_error"
+                : "legacy_inferred_error",
+          });
+        }
+
         // Finalize the OTel tool span. If the span promise hasn't resolved yet
         // we record the result on the entry so its `.then` handler ends it.
         const otelEndResult = {
@@ -392,7 +472,7 @@ export async function instrumentAgentLoop(opts: {
           cacheReadTokens: 0,
           cacheWriteTokens: 0,
           costCentsX100: 0,
-          durationMs: pending ? Date.now() - pending.startMs : 0,
+          durationMs: pending ? Math.max(0, finishedAt - pending.startMs) : 0,
           status: isError ? "error" : "success",
           errorMessage: isError ? event.result : null,
           metadata:
@@ -422,7 +502,11 @@ export async function instrumentAgentLoop(opts: {
   let errorMessage: string | null = null;
   let runMetadata: Record<string, unknown> | null = opts.metadata ?? null;
   try {
-    usage = await runAgentLoop({ ...loopOpts, send: instrumentedSend });
+    usage = await runAgentLoop({
+      ...loopOpts,
+      runId,
+      send: instrumentedSend,
+    });
   } catch (err: any) {
     const classification = opts.classifyError?.(err) ?? null;
     runStatus = classification?.status ?? "error";
@@ -440,6 +524,62 @@ export async function instrumentAgentLoop(opts: {
     const runEnd = Date.now();
     const totalDurationMs = runEnd - runStart;
 
+    if (pendingTools.size > 0) {
+      if (runStatus === "success") {
+        runStatus = "error";
+        errorMessage ??= "Agent run ended with interrupted tool calls";
+      }
+      for (const [counter, pending] of pendingTools) {
+        toolCallCount += 1;
+        failedTools += 1;
+        if (counter < MAX_TRACKED_GENERATION_TOOL_CALLS) {
+          generationToolCalls.set(counter, {
+            name: pending.toolName,
+            started_offset_ms: Math.max(0, pending.startMs - runStart),
+            duration_ms: Math.max(0, runEnd - pending.startMs),
+            status: "error",
+            error_class: "interrupted",
+          });
+        }
+        const interruptedMessage = "Tool call interrupted before completion";
+        if (pending.otelSpan) {
+          openOtelToolSpans.delete(pending.otelSpan);
+          endAgentSpan(pending.otelSpan, {
+            status: "error",
+            errorMessage: interruptedMessage,
+            attributes: { "tool.name": pending.toolName },
+          });
+        } else {
+          pending.endResult = {
+            status: "error",
+            errorMessage: interruptedMessage,
+          };
+        }
+        spans.push({
+          id: pending.spanId,
+          runId,
+          threadId,
+          userId,
+          parentSpanId,
+          spanType: "tool_call",
+          name: pending.toolName,
+          inputTokens: 0,
+          outputTokens: 0,
+          cacheReadTokens: 0,
+          cacheWriteTokens: 0,
+          costCentsX100: 0,
+          durationMs: Math.max(0, runEnd - pending.startMs),
+          status: "error",
+          errorMessage: interruptedMessage,
+          metadata: null,
+          createdAt: runEnd,
+        });
+      }
+      pendingTools.clear();
+      toolNameToCounters.clear();
+      toolCallIdToCounter.clear();
+    }
+
     let costCentsX100 = 0;
     try {
       const { calculateCost } = await import("../usage/store.js");
@@ -455,8 +595,15 @@ export async function instrumentAgentLoop(opts: {
     } catch {}
 
     let llmCallCount = 0;
-    if (usage) {
+    if (usage || runStatus === "error") {
       llmCallCount = 1;
+      const generationUsage = usage ?? {
+        inputTokens: 0,
+        outputTokens: 0,
+        cacheReadTokens: 0,
+        cacheWriteTokens: 0,
+        model: loopOpts.model,
+      };
       const llmSpanId = spanId();
       const llmSpan: TraceSpan = {
         id: llmSpanId,
@@ -465,11 +612,11 @@ export async function instrumentAgentLoop(opts: {
         userId,
         parentSpanId,
         spanType: "llm_call",
-        name: usage.model,
-        inputTokens: usage.inputTokens,
-        outputTokens: usage.outputTokens,
-        cacheReadTokens: usage.cacheReadTokens,
-        cacheWriteTokens: usage.cacheWriteTokens,
+        name: generationUsage.model,
+        inputTokens: generationUsage.inputTokens,
+        outputTokens: generationUsage.outputTokens,
+        cacheReadTokens: generationUsage.cacheReadTokens,
+        cacheWriteTokens: generationUsage.cacheWriteTokens,
         costCentsX100,
         durationMs: totalDurationMs,
         status: runStatus,
@@ -488,11 +635,11 @@ export async function instrumentAgentLoop(opts: {
           typeof loopOpts.engine?.name === "string"
             ? loopOpts.engine.name
             : undefined,
-        model: usage.model,
-        inputTokens: usage.inputTokens,
-        outputTokens: usage.outputTokens,
-        cacheReadTokens: usage.cacheReadTokens,
-        cacheWriteTokens: usage.cacheWriteTokens,
+        model: generationUsage.model,
+        inputTokens: generationUsage.inputTokens,
+        outputTokens: generationUsage.outputTokens,
+        cacheReadTokens: generationUsage.cacheReadTokens,
+        cacheWriteTokens: generationUsage.cacheWriteTokens,
         costCentsX100,
         durationMs: totalDurationMs,
         status: runStatus,
@@ -500,6 +647,12 @@ export async function instrumentAgentLoop(opts: {
         toolCalls: toolCallCount,
         successfulTools,
         failedTools,
+        tools: [...generationToolCalls.entries()]
+          .sort(([a], [b]) => a - b)
+          .map(([, detail]) => detail),
+        toolsTruncated:
+          toolInvocationCounter > MAX_TRACKED_GENERATION_TOOL_CALLS,
+        delegation: opts.delegation,
         createdAt: runStart,
         experimentAssignments: opts.experimentAssignments,
         modelSelectionSource: opts.modelSelectionSource,

package/corpus/core/src/scripts/call-agent.ts

@@ -1,4 +1,4 @@
-import { createHash } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 
 import {
   A2ATaskTimeoutError,
@@ -7,7 +7,11 @@ import {
   signA2AToken,
 } from "../a2a/client.js";
 import { invokeAgentAction } from "../a2a/invoke.js";
-import type { A2AApprovedAction, Task } from "../a2a/types.js";
+import type {
+  A2AApprovedAction,
+  A2ACorrelationMetadata,
+  Task,
+} from "../a2a/types.js";
 import {
   formatLlmCredentialErrorMessage,
   isLlmCredentialError,
@@ -28,6 +32,38 @@ const DEFAULT_SERVERLESS_INTEGRATION_A2A_TIMEOUT_MS = 18_000;
 const NETLIFY_INTEGRATION_A2A_TIMEOUT_MS = 2_000;
 const INTEGRATION_A2A_TOKEN_TTL = "30m";
 
+function buildDelegationCorrelation(
+  context: ActionRunContext | undefined,
+  selfAppId: string | undefined,
+  invocationId?: string,
+): A2ACorrelationMetadata {
+  return {
+    ...(selfAppId?.trim() ? { callerApp: selfAppId.trim() } : {}),
+    ...(context?.threadId ? { callerThreadId: context.threadId } : {}),
+    ...(context?.runId ? { parentRunId: context.runId } : {}),
+    ...(context?.turnId ? { parentTurnId: context.turnId } : {}),
+    ...(invocationId ? { invocationId } : {}),
+  };
+}
+
+function buildMessageIdempotencyKey(
+  originatingTurnId: string | undefined,
+  target: string,
+  exactMessage: string,
+): string | undefined {
+  if (!originatingTurnId) return undefined;
+  const digest = createHash("sha256")
+    .update(
+      JSON.stringify({
+        originatingTurnId,
+        target,
+        message: exactMessage,
+      }),
+    )
+    .digest("hex");
+  return `v1:${digest}`;
+}
+
 function parseTimeoutMs(value: string | undefined): number | undefined {
   if (!value) return undefined;
   const parsed = Number(value);
@@ -187,6 +223,12 @@ export async function run(
     return `Error: Agent "${agentIdOrName}" not found. Available agents: ${available || "(none)"}`;
   }
 
+  const correlation = buildDelegationCorrelation(context, selfAppId);
+  const idempotencyKey =
+    message && !taskId
+      ? buildMessageIdempotencyKey(context?.turnId, agent.url, message)
+      : undefined;
+
   if (action) {
     if (context?.send) {
       context.send({ type: "agent_call", agent: agent.name, status: "start" });
@@ -196,6 +238,7 @@ export async function run(
         agent,
         action,
         input as Record<string, unknown>,
+        buildDelegationCorrelation(context, selfAppId, randomUUID()),
       );
       return output;
     } finally {
@@ -389,6 +432,9 @@ export async function run(
           orgDomain: callerOrgDomain,
           orgSecret: callerOrgSecret,
           approvedActions,
+          contextId: context.threadId,
+          correlation,
+          idempotencyKey,
           ...(taskId ? { taskId } : {}),
           onUpdate: onRemotePollUpdate,
           returnRecoverableArtifactsOnTimeout: false,
@@ -480,6 +526,9 @@ export async function run(
       orgDomain: domain,
       orgSecret,
       approvedActions,
+      contextId: context?.threadId,
+      correlation,
+      idempotencyKey,
       ...(taskId ? { taskId } : {}),
       returnRecoverableArtifactsOnTimeout: false,
     });
@@ -514,6 +563,7 @@ async function invokeReadOnlyAppAction(
   agent: { name: string; url: string },
   action: string,
   input: Record<string, unknown>,
+  correlation: A2ACorrelationMetadata,
 ): Promise<string> {
   const callerEmail = getRequestUserEmail();
   if (!callerEmail) {
@@ -544,6 +594,7 @@ async function invokeReadOnlyAppAction(
       userEmail: callerEmail,
       orgDomain: callerOrgDomain,
       orgSecret: callerOrgSecret,
+      correlation,
     });
     return invocation.result.status === "completed"
       ? invocation.result.output

package/corpus/core/src/server/agent-chat/action-filters-a2a.ts

@@ -191,34 +191,81 @@ function formatA2ATerminalError(
 
 type A2AAgentLoopRunner = typeof runAgentLoopDirectWithSoftTimeout;
 
-function runDelegatedAgentLoop(
+export interface DelegatedAgentLoopTelemetry {
+  runId: string;
+  threadId: string | null;
+  userId: string | null;
+  delegation: {
+    protocol: "a2a" | "mcp";
+    callerApp?: string;
+    taskId?: string;
+    parentRunId?: string;
+    parentTurnId?: string;
+  };
+}
+
+interface DelegatedAgentLoopOptions {
+  runner?: A2AAgentLoopRunner;
+  telemetry?: DelegatedAgentLoopTelemetry;
+}
+
+async function runDelegatedAgentLoop(
   runOptions: Parameters<A2AAgentLoopRunner>[0],
   pluginOptions: Pick<
     AgentChatPluginOptions,
     "finalResponseGuard" | "runSoftTimeoutMs"
   >,
   timeoutOptions: Parameters<A2AAgentLoopRunner>[2],
-  runner: A2AAgentLoopRunner,
+  options: DelegatedAgentLoopOptions,
 ) {
-  return runner(
-    {
-      ...runOptions,
-      // Delegated runs resolve their own model and do not pass through the
-      // interactive request handler's output-token setup. Use the same
-      // model-aware headroom here so reasoning models (notably GPT-5.x) do
-      // not spend the small internal default entirely on reasoning before
-      // emitting a tool call or answer. Preserve explicit test/caller values.
-      maxOutputTokens:
-        runOptions.maxOutputTokens ??
-        resolveMainChatMaxOutputTokens(runOptions.model),
-      reasoningEffort:
-        runOptions.reasoningEffort ??
-        resolveAgentRequestReasoningEffort({ model: runOptions.model }),
-      finalResponseGuard: pluginOptions.finalResponseGuard,
-    },
-    pluginOptions.runSoftTimeoutMs,
-    timeoutOptions,
-  );
+  const runner = options.runner ?? runAgentLoopDirectWithSoftTimeout;
+  const resolvedRunOptions = {
+    ...runOptions,
+    // Delegated runs resolve their own model and do not pass through the
+    // interactive request handler's output-token setup. Use the same
+    // model-aware headroom here so reasoning models (notably GPT-5.x) do
+    // not spend the small internal default entirely on reasoning before
+    // emitting a tool call or answer. Preserve explicit test/caller values.
+    maxOutputTokens:
+      runOptions.maxOutputTokens ??
+      resolveMainChatMaxOutputTokens(runOptions.model),
+    reasoningEffort:
+      runOptions.reasoningEffort ??
+      resolveAgentRequestReasoningEffort({ model: runOptions.model }),
+    finalResponseGuard: pluginOptions.finalResponseGuard,
+  };
+  const execute = (loopOptions = resolvedRunOptions) =>
+    runner(loopOptions, pluginOptions.runSoftTimeoutMs, timeoutOptions);
+
+  if (!options.telemetry) return execute();
+
+  let instrumented = false;
+  try {
+    const { getObservabilityConfig, instrumentAgentLoop } =
+      await import("../../observability/traces.js");
+    const config = await getObservabilityConfig();
+    if (config.enabled) {
+      instrumented = true;
+      return await instrumentAgentLoop({
+        runAgentLoop: (loopOptions) =>
+          execute(
+            loopOptions as Parameters<A2AAgentLoopRunner>[0] &
+              typeof resolvedRunOptions,
+          ),
+        loopOpts: resolvedRunOptions,
+        runId: options.telemetry.runId,
+        threadId: options.telemetry.threadId,
+        userId: options.telemetry.userId,
+        config,
+        delegation: options.telemetry.delegation,
+      });
+    }
+  } catch (error) {
+    // Match interactive chat: setup failures fall through, but a failure from
+    // inside an instrumented agent loop is the real run failure and rethrows.
+    if (instrumented) throw error;
+  }
+  return execute();
 }
 
 /**
@@ -236,13 +283,13 @@ export function runA2AAgentLoop(
     "finalResponseGuard" | "runSoftTimeoutMs"
   >,
   timeoutOptions: Parameters<A2AAgentLoopRunner>[2],
-  runner: A2AAgentLoopRunner = runAgentLoopDirectWithSoftTimeout,
+  options: DelegatedAgentLoopOptions = {},
 ) {
   return runDelegatedAgentLoop(
     runOptions,
     pluginOptions,
     timeoutOptions,
-    runner,
+    options,
   );
 }
 
@@ -258,13 +305,13 @@ export function runMCPAgentLoop(
     "finalResponseGuard" | "runSoftTimeoutMs"
   >,
   timeoutOptions: Parameters<A2AAgentLoopRunner>[2],
-  runner: A2AAgentLoopRunner = runAgentLoopDirectWithSoftTimeout,
+  options: DelegatedAgentLoopOptions = {},
 ) {
   return runDelegatedAgentLoop(
     runOptions,
     pluginOptions,
     timeoutOptions,
-    runner,
+    options,
   );
 }