@vellumai/assistant 0.12.2 → 0.12.3-staging.2

package/src/agent/loop.ts

@@ -1,4 +1,7 @@
 import type { AnsweredQuestion } from "../api/events/question-answered.js";
+import type { ToolActivityMetadata } from "../api/events/tool-result.js";
+import { resolveCallSiteConfig } from "../config/llm-resolver.js";
+import { getConfig } from "../config/loader.js";
 import type { LLMCallSite } from "../config/schemas/llm.js";
 import { SEND_USER_MESSAGE_TOOL_NAME } from "../config/send-user-message-constants.js";
 import { recordEstimate } from "../context/estimator-calibration.js";
@@ -11,7 +14,6 @@ import {
   getCalibrationProviderKey,
 } from "../context/token-estimator.js";
 import { spoolAndStubOversizedToolResults } from "../context/tool-result-spool.js";
-import type { ToolActivityMetadata } from "../daemon/message-types/web-activity.js";
 import { parseActualTokensFromError } from "../daemon/parse-actual-tokens-from-error.js";
 import type { TrustContext } from "../daemon/trust-context-types.js";
 import type {
@@ -42,6 +44,7 @@ import type {
   Provider,
   ProviderResponse,
   SendMessageOptions,
+  TextContent,
   ToolDefinition,
   ToolResultContent,
 } from "../providers/types.js";
@@ -74,6 +77,7 @@ import {
   isRepairableOrderingError,
   isUserTerminalHistoryError,
 } from "./history-repair/history-repair.js";
+import { buildToolResultFollowUp } from "./tool-result-follow-up.js";
 
 const log = getLogger("agent-loop");
 
@@ -441,25 +445,26 @@ export type AgentEvent =
       /**
        * Emitted when the provider call throws — i.e. the provider
        * rejected the request before returning a usable response. Carries
-       * the loop-level raw request we attempted to send (messages, tools,
-       * system prompt, provider-agnostic config) plus the thrown error.
+       * the wire request we attempted to send plus the thrown error.
        * Consumers (`handleProviderError` in the daemon handlers, the
        * `onEvent` in `agent-wake`) persist these as `llm_request_logs`
        * rows so failed calls are queryable in the LLM inspector instead
        * of only surfacing in pino logs.
        *
-       * `rawRequest` is the loop-level abstract shape rather than the
-       * provider-specific payload (which the provider builds internally
-       * and never returns when it throws). `actualProvider` echoes the
-       * `ProviderError.provider` tag when available so the persisted row
-       * has the same `provider` column value as a successful `usage` row.
+       * `rawRequest` is `ProviderError.rawRequest` when the throw carried
+       * an inspectable SDK/wire payload (including extra body fields such
+       * as `directions`). The loop does not invent a substitute snapshot
+       * of messages/tools/systemPrompt; a missing payload stays missing.
+       * `actualProvider` echoes `ProviderError.provider` so a routed
+       * invocation (e.g. Vellum via a Fireworks default wrapper) is
+       * attributed to the transport that actually ran.
        *
        * Re-thrown by the inner LLM-call try/catch after emission so the
        * outer agent-loop catch still handles abort, the existing `error`
        * event, and the loop break.
        */
       type: "provider_error";
-      rawRequest: unknown;
+      rawRequest?: unknown;
       error: Error;
       actualProvider?: string;
     }
@@ -661,6 +666,18 @@ type AgentLoopContextWindowResolver = () => {
   overflowRecovery: { enabled: boolean; safetyMarginRatio: number };
 };
 
+/** Final request surface after the pre-model hook has settled. */
+export interface PreparedModelCall {
+  callSite?: LLMCallSite;
+  overrideProfile?: string;
+  forceOverrideProfile: boolean;
+  /** Present when the finalized route opts out of prompt caching. */
+  disableCache?: true;
+  signal?: AbortSignal;
+  systemPrompt: string | null;
+  tools: ToolDefinition[];
+}
+
 interface AgentLoopRunOptionsBase {
   /** Input history the run starts from; the loop appends its output onto a copy. */
   messages: Message[];
@@ -688,7 +705,10 @@ interface AgentLoopRunOptionsBase {
   onCheckpoint?: (
     checkpoint: CheckpointInfo,
   ) => CheckpointDecision | Promise<CheckpointDecision>;
+  /** Semantic call site exposed to hooks, events, and loop behavior. */
   callSite?: LLMCallSite;
+  /** Provider-resolution call site when it differs from turn semantics. */
+  inferenceCallSite?: LLMCallSite;
   /**
    * Route this run's user-facing text through the `send_user_message` tool
    * instead of streamed assistant text. The daemon sets it for main-agent runs
@@ -738,6 +758,8 @@ interface AgentLoopRunOptionsBase {
    */
   forceOverrideProfile?: boolean;
   resolveOverrideProfile?: () => string | undefined;
+  /** Observe a finalized model request without delaying provider dispatch. */
+  onModelCallPrepared?: (prepared: PreparedModelCall) => void;
   /**
    * When `true`, the loop owns turn-start and mid-loop compaction. The pre-call
    * budget gate runs before the very first provider call — subsuming the
@@ -1047,6 +1069,17 @@ export interface AgentLoopConstructorOptions {
   tools?: ToolDefinition[];
   toolExecutor?: LoopToolExecutor;
   resolveTools?: (history: Message[]) => ToolDefinition[];
+  /**
+   * Observer for the final tool array of each provider call, invoked
+   * immediately before the request leaves with exactly what goes on the wire
+   * (after any provider-native tool is appended, past the inter-call throttle
+   * and the pre-model hooks, and not at all once the run is aborted). This is
+   * the only point that sees the sent array: the dynamic `resolveTools`
+   * callback is also consulted out of band (token counting, compaction
+   * estimates), so a consumer that needs "what the last request sent"
+   * subscribes here rather than wrapping the resolver. Must not throw.
+   */
+  onToolsSent?: (tools: ToolDefinition[]) => void;
   /**
    * Conversation this loop drives. Scopes the loop-held compaction circuit
    * breaker and is the source of truth the loop's pipeline contexts and
@@ -1087,6 +1120,7 @@ export class AgentLoop {
   private config: AgentLoopConfig;
   private tools: ToolDefinition[];
   private resolveTools: ((history: Message[]) => ToolDefinition[]) | null;
+  private onToolsSent: ((tools: ToolDefinition[]) => void) | null;
   private toolExecutor: LoopToolExecutor | null;
 
   /**
@@ -1122,6 +1156,7 @@ export class AgentLoop {
       tools,
       toolExecutor,
       resolveTools,
+      onToolsSent,
       conversationId,
       resolveConversationDir,
       transformCompactedHistory,
@@ -1131,6 +1166,7 @@ export class AgentLoop {
     this.config = { ...DEFAULT_CONFIG, ...config };
     this.tools = tools ?? [];
     this.resolveTools = resolveTools ?? null;
+    this.onToolsSent = onToolsSent ?? null;
     this.toolExecutor = toolExecutor ?? null;
     this.conversationId = conversationId;
     this.resolveConversationDir = resolveConversationDir ?? null;
@@ -1413,18 +1449,21 @@ export class AgentLoop {
       requestId,
       onCheckpoint,
       callSite,
+      inferenceCallSite,
       suppressAssistantText = false,
       supportsDynamicUi = true,
       trust,
       overrideProfile,
       forceOverrideProfile = false,
       resolveOverrideProfile,
+      onModelCallPrepared,
       compactInPlace = false,
       isNonInteractive = false,
       model: runModel,
       latencyTracker,
       injectionLedgerResets,
     } = options;
+    const providerCallSite = inferenceCallSite ?? callSite;
     // Snapshot the system prompt once per run. The instance field is mutable
     // (the conversation may update it between turns), but a single run must
     // use one consistent prompt — an aborted run left detached after the
@@ -1544,7 +1583,7 @@ export class AgentLoop {
       turn: number,
     ): Promise<{
       resultBlocks: ContentBlock[];
-      additionalContextBlocks: ContentBlock[];
+      additionalContextBlocks: TextContent[];
     }> => {
       if (conversationDir) {
         const toolCallByUseId = new Map(
@@ -1569,7 +1608,7 @@ export class AgentLoop {
         180_000;
 
       const resultBlocks: ContentBlock[] = [];
-      const additionalContextBlocks: ContentBlock[] = [];
+      const additionalContextBlocks: TextContent[] = [];
       for (const block of rawBlocks) {
         if (block.type !== "tool_result") {
           resultBlocks.push(block);
@@ -1870,7 +1909,7 @@ export class AgentLoop {
         // unexecutable client tool. The advisor consult's `advisorProfile` can
         // route `subagentSpawn` to a provider/model whose native-search support
         // differs from the construction-time default, so the gate resolves the
-        // routed target (callSite + overrideProfile) via
+        // routed target (providerCallSite + overrideProfile) via
         // `supportsNativeWebSearchFor` rather than the static
         // `this.provider.supportsNativeWebSearch` snapshot; providers without
         // the routing-aware probe fall back to the static flag. This is a SERVER
@@ -1882,7 +1921,7 @@ export class AgentLoop {
           .supportsNativeWebSearchFor
           ? this.provider.supportsNativeWebSearchFor(
               buildNativeWebSearchProbeOptions(
-                callSite,
+                providerCallSite,
                 resolveEffectiveOverrideProfile(),
                 forceOverrideProfile,
                 this.conversationId,
@@ -1901,11 +1940,11 @@ export class AgentLoop {
         //   1. Per-run explicit (`runModel`)
         //   2. Call-site resolved values (filled by
         //      `RetryProvider.normalizeSendMessageOptions` from
-        //      `resolveCallSiteConfig(callSite, llm)`)
+        //      `resolveCallSiteConfig(providerCallSite, llm)`)
         //   3. Conversation defaults (`this.config.*`, from the resolved
         //      default call-site config)
         //
-        // When `callSite` is present we deliberately leave
+        // When `providerCallSite` is present we deliberately leave
         // `max_tokens`/`thinking`/`effort`/`speed` *unset* in `providerConfig`
         // so the normalizer can fill them from the call-site resolution. The
         // normalizer only writes these fields when they're undefined; if we
@@ -1913,10 +1952,10 @@ export class AgentLoop {
         // for these knobs is silently ignored.
         //
         // `toolChoice` and `cacheTtl` are not part of the call-site schema, so
-        // they always come from `this.config` regardless of `callSite`.
+        // they always come from `this.config` regardless of `providerCallSite`.
         const providerConfig: Record<string, unknown> = {};
 
-        if (!callSite) {
+        if (!providerCallSite) {
           providerConfig.max_tokens = this.config.maxTokens;
         }
 
@@ -1924,7 +1963,7 @@ export class AgentLoop {
           providerConfig.model = runModel;
         }
 
-        if (!callSite) {
+        if (!providerCallSite) {
           const thinking = normalizeThinkingConfigForWire(this.config.thinking);
           if (thinking !== undefined) {
             providerConfig.thinking = thinking;
@@ -1963,9 +2002,9 @@ export class AgentLoop {
         // defaults when absent).
         // User-initiated conversation turns default to `mainAgent` in the
         // agent loop's caller; other invocation contexts (heartbeat, filing,
-        // analyze, etc.) pass their own `callSite`.
-        if (callSite) {
-          providerConfig.callSite = callSite;
+        // analyze, etc.) pass their own provider-resolution site.
+        if (providerCallSite) {
+          providerConfig.callSite = providerCallSite;
           providerConfig.usageTracking = "manual";
           // Per-conversation seed for deterministic `mix`-profile expansion.
           // Sourced from the loop's own conversation id so every LLM call in a
@@ -1993,7 +2032,7 @@ export class AgentLoop {
         // `activeProfile` and any call-site named profile. Threading it on
         // every send (rather than once at construction) keeps subagents that
         // share an `AgentLoop` instance but ought to inherit a different
-        // profile correct — and matches how `callSite` is plumbed.
+        // profile correct, matching how the provider call site is plumbed.
         const effectiveOverrideProfile = resolveEffectiveOverrideProfile();
         if (effectiveOverrideProfile) {
           providerConfig.overrideProfile = effectiveOverrideProfile;
@@ -2217,6 +2256,49 @@ export class AgentLoop {
           );
         }
 
+        if (onModelCallPrepared && !signal?.aborted) {
+          const preparedOverrideProfile =
+            typeof providerConfig.overrideProfile === "string" &&
+            providerConfig.overrideProfile.length > 0
+              ? providerConfig.overrideProfile
+              : undefined;
+          try {
+            const preparedForceOverrideProfile =
+              providerConfig.forceOverrideProfile === true;
+            const disableCache =
+              providerCallSite !== undefined &&
+              resolveCallSiteConfig(providerCallSite, getConfig().llm, {
+                ...(preparedOverrideProfile !== undefined
+                  ? { overrideProfile: preparedOverrideProfile }
+                  : {}),
+                ...(preparedForceOverrideProfile
+                  ? { forceOverrideProfile: true }
+                  : {}),
+                ...(this.conversationId !== undefined
+                  ? { selectionSeed: this.conversationId }
+                  : {}),
+              }).disableCache === true;
+            onModelCallPrepared({
+              ...(providerCallSite !== undefined
+                ? { callSite: providerCallSite }
+                : {}),
+              ...(preparedOverrideProfile !== undefined
+                ? { overrideProfile: preparedOverrideProfile }
+                : {}),
+              forceOverrideProfile: preparedForceOverrideProfile,
+              ...(disableCache ? { disableCache: true as const } : {}),
+              ...(signal !== undefined ? { signal } : {}),
+              systemPrompt: providerOptions.systemPrompt ?? null,
+              tools: currentTools,
+            });
+          } catch (preparedError) {
+            rlog.warn(
+              { err: preparedError },
+              "Prepared model-call observer failed; continuing with provider dispatch",
+            );
+          }
+        }
+
         // Announce the LLM-call boundary so downstream handlers (the
         // daemon's persistence pipeline) can reserve an empty assistant row
         // and stamp the resulting `messageId` onto every streaming event the
@@ -2234,13 +2316,18 @@ export class AgentLoop {
         // turn body (tool execution, plugin pipelines, checkpoints), so
         // recording there would risk mis-attributing tool/plugin throws as
         // provider rejections. On provider failure we emit `provider_error`
-        // with the loop-level raw request so consumers can persist it as an
-        // `llm_request_logs` row, then re-throw so the existing outer catch
-        // continues to handle abort sync, the `error` event, and the loop
-        // break unchanged.
+        // with the inspectable wire request from the throw (when present)
+        // so consumers can persist it as an `llm_request_logs` row, then
+        // re-throw so the existing outer catch continues to handle abort
+        // sync, the `error` event, and the loop break unchanged.
         // Latency: the request is about to leave for the provider. The span
         // from here to the first streamed token is time-to-first-token.
         latencyTracker?.mark("request_sent");
+        // Every await that could cancel the call is behind us; a run aborted
+        // during the throttle or a hook never reports an array it did not send.
+        if (!signal?.aborted) {
+          this.onToolsSent?.(currentTools);
+        }
         let response: ProviderResponse;
         try {
           response = await traceAsyncSection("agent-loop:provider-send", () =>
@@ -2255,26 +2342,18 @@ export class AgentLoop {
               llmCallError instanceof Error
                 ? llmCallError
                 : new Error(String(llmCallError));
-            // Strip non-serializable / runtime-only fields from `options`
-            // before snapshotting. `onEvent` is a closure with side effects
-            // and `signal` is an AbortSignal — neither is meaningful in a
-            // persisted log row, and `JSON.stringify` would silently drop or
-            // misrepresent both.
-            const rawRequest = {
-              provider: this.provider.name,
-              messages: sanitizedHistory,
-              tools: providerOptions.tools,
-              systemPrompt: providerOptions.systemPrompt,
-              config: providerOptions.config,
-            };
+            const invocationProvider =
+              errInstance instanceof ProviderError
+                ? errInstance.provider
+                : this.provider.name;
             onEvent({
               type: "provider_error",
-              rawRequest,
+              ...(errInstance instanceof ProviderError &&
+              errInstance.rawRequest !== undefined
+                ? { rawRequest: errInstance.rawRequest }
+                : {}),
               error: errInstance,
-              actualProvider:
-                errInstance instanceof ProviderError
-                  ? errInstance.provider
-                  : this.provider.name,
+              actualProvider: invocationProvider,
             });
           }
           providerCallError = llmCallError;
@@ -2945,16 +3024,20 @@ export class AgentLoop {
 
         toolUseTurns++;
 
-        // Append any guidance a post-tool-use hook surfaced via
-        // `additionalContext` (e.g. tool-error retry coaching) as separate
-        // blocks. They join the provider-bound history below but were not part
-        // of the tool_result events emitted above, so the model sees the
-        // guidance while the client-facing and persisted tool output stay the
-        // tool's actual result.
-        resultBlocks.push(...additionalContextBlocks);
-
-        // Add tool results as a user message and continue the loop.
-        history.push({ role: "user", content: resultBlocks });
+        // Add the tool results, plus any guidance a post-tool-use hook
+        // surfaced via `additionalContext` (e.g. tool-error retry coaching),
+        // as a user message and continue the loop. The guidance joins the
+        // provider-bound history only: it was not part of the tool_result
+        // events emitted above, so the client-facing and persisted tool
+        // output stay the tool's actual result.
+        history.push({
+          role: "user",
+          content: buildToolResultFollowUp(
+            history,
+            resultBlocks,
+            additionalContextBlocks,
+          ),
+        });
 
         // Invoke checkpoint callback after tool results are in history.
         // Handoff takes precedence over the budget gate: a handoff decision

package/src/agent/tool-result-follow-up.ts

@@ -0,0 +1,70 @@
+// Content of the user message that answers a tool-calling assistant turn:
+// the tool_result blocks plus any guidance a post-tool-use hook surfaced via
+// `additionalContext`.
+
+import { analyzeServerToolPairing } from "../providers/server-tool-pairing.js";
+import type {
+  ContentBlock,
+  Message,
+  TextContent,
+  ToolResultContent,
+} from "../providers/types.js";
+
+/**
+ * Hook guidance rides as separate blocks after the results, so the model sees
+ * it while each tool_result stays the tool's actual output. The exception is
+ * an assistant turn that left a server tool (native web search) deferred: the
+ * provider runs a deferred server tool only when the answering message holds
+ * tool_result blocks alone, and any block after the results closes the
+ * assistant turn, after which the provider rejects the unanswered server tool
+ * as unpaired. The guidance is then folded into the content of the last
+ * errored tool_result (the last tool_result when none errored), where the
+ * model still sees it and the search still runs.
+ */
+export function buildToolResultFollowUp(
+  history: ReadonlyArray<Message>,
+  resultBlocks: ContentBlock[],
+  additionalContextBlocks: ReadonlyArray<TextContent>,
+): ContentBlock[] {
+  if (additionalContextBlocks.length === 0) {
+    return resultBlocks;
+  }
+  const targetIndex = foldTargetIndex(resultBlocks);
+  const pairing = analyzeServerToolPairing([
+    ...history,
+    { role: "user", content: resultBlocks },
+  ]);
+  if (pairing.deferredUseIds.size === 0 || targetIndex < 0) {
+    return [...resultBlocks, ...additionalContextBlocks];
+  }
+  const target = resultBlocks[targetIndex] as ToolResultContent;
+  const folded: ToolResultContent = {
+    ...target,
+    content: [target.content, ...additionalContextBlocks.map((b) => b.text)]
+      .filter((text) => text.length > 0)
+      .join("\n\n"),
+  };
+  return resultBlocks.map((block, index) =>
+    index === targetIndex ? folded : block,
+  );
+}
+
+/** Index of the last errored tool_result, else the last tool_result, else -1. */
+function foldTargetIndex(blocks: ReadonlyArray<ContentBlock>): number {
+  let lastResult = -1;
+  for (let index = blocks.length - 1; index >= 0; index--) {
+    const block = blocks[index];
+    // guard:allow-tool-result-only: the answering user message carries client
+    // tool_result blocks only; server-side results live in assistant messages.
+    if (block.type !== "tool_result") {
+      continue;
+    }
+    if (block.is_error) {
+      return index;
+    }
+    if (lastResult < 0) {
+      lastResult = index;
+    }
+  }
+  return lastResult;
+}

package/src/api/attachment-provenance.test.ts

@@ -0,0 +1,68 @@
+import { describe, expect, test } from "bun:test";
+
+import { AssistantOutboundAttachmentSchema } from "./events/assistant-outbound-attachment.js";
+import { ConversationMessageAttachmentSchema } from "./responses/conversation-message.js";
+
+const outboundFixture = {
+  id: "attachment-1",
+  filename: "screenshot.png",
+  mimeType: "image/png",
+  data: "c2NyZWVuc2hvdA==",
+  sourceType: "tool_block" as const,
+};
+
+const historyFixture = {
+  id: "attachment-1",
+  filename: "screenshot.png",
+  mimeType: "image/png",
+  sizeBytes: 10,
+  kind: "image",
+};
+
+describe("computer-use screenshot attachment provenance", () => {
+  test("new schemas accept the marker and its absence", () => {
+    expect(
+      AssistantOutboundAttachmentSchema.parse({
+        ...outboundFixture,
+        computerUseScreenshot: true,
+      }).computerUseScreenshot,
+    ).toBe(true);
+    expect(
+      AssistantOutboundAttachmentSchema.parse(outboundFixture)
+        .computerUseScreenshot,
+    ).toBeUndefined();
+
+    expect(
+      ConversationMessageAttachmentSchema.parse({
+        ...historyFixture,
+        computerUseScreenshot: true,
+      }).computerUseScreenshot,
+    ).toBe(true);
+    expect(
+      ConversationMessageAttachmentSchema.parse(historyFixture)
+        .computerUseScreenshot,
+    ).toBeUndefined();
+  });
+
+  test("legacy object parsers tolerate and strip the additive marker", () => {
+    const legacyOutbound = AssistantOutboundAttachmentSchema.omit({
+      computerUseScreenshot: true,
+    });
+    const legacyHistory = ConversationMessageAttachmentSchema.omit({
+      computerUseScreenshot: true,
+    });
+
+    expect(
+      legacyOutbound.parse({
+        ...outboundFixture,
+        computerUseScreenshot: true,
+      }),
+    ).toEqual(outboundFixture);
+    expect(
+      legacyHistory.parse({
+        ...historyFixture,
+        computerUseScreenshot: true,
+      }),
+    ).toEqual(historyFixture);
+  });
+});

package/src/api/computer-use-tool.test.ts

@@ -0,0 +1,55 @@
+import { describe, expect, test } from "bun:test";
+
+import {
+  isComputerUseToolCall,
+  resolveComputerUseToolName,
+} from "./computer-use-tool.js";
+
+describe("isComputerUseToolCall", () => {
+  test("recognizes direct and skill-wrapped computer-use calls", () => {
+    expect(isComputerUseToolCall("computer_use_click", {})).toBe(true);
+    expect(
+      isComputerUseToolCall("skill_execute", { tool: "computer_use_type" }),
+    ).toBe(true);
+  });
+
+  test("resolves the direct or wrapped computer-use tool name", () => {
+    expect(resolveComputerUseToolName("computer_use_click", {})).toBe(
+      "computer_use_click",
+    );
+    expect(
+      resolveComputerUseToolName("skill_execute", {
+        tool: "computer_use_scroll",
+      }),
+    ).toBe("computer_use_scroll");
+  });
+
+  test("rejects unrelated media tools and malformed skill input", () => {
+    expect(isComputerUseToolCall("browser_screenshot", {})).toBe(false);
+    expect(isComputerUseToolCall("file_read", {})).toBe(false);
+    expect(isComputerUseToolCall("image_generate", {})).toBe(false);
+    expect(isComputerUseToolCall("skill_execute", null)).toBe(false);
+    expect(isComputerUseToolCall("skill_execute", "computer_use_click")).toBe(
+      false,
+    );
+    expect(isComputerUseToolCall("skill_execute", { tool: 42 })).toBe(false);
+    expect(
+      isComputerUseToolCall("skill_execute", {
+        tool: "browser_screenshot",
+      }),
+    ).toBe(false);
+  });
+
+  test("does not classify raw-only recovered skill input", () => {
+    expect(
+      isComputerUseToolCall("skill_execute", {
+        _raw: '{"tool":"computer_use_click"}',
+      }),
+    ).toBe(false);
+    expect(
+      resolveComputerUseToolName("skill_execute", {
+        _raw: '{"tool":"computer_use_click"}',
+      }),
+    ).toBeUndefined();
+  });
+});

package/src/api/computer-use-tool.ts

@@ -0,0 +1,27 @@
+/**
+ * Resolve the host computer-use tool behind a direct or skill-wrapped call.
+ *
+ * Skills expose their selected tool in structured input. Calls recovered only
+ * from `_raw` retain their legacy behavior because this helper deliberately
+ * does not parse a second input representation.
+ */
+export function resolveComputerUseToolName(
+  name: string,
+  input: unknown,
+): string | undefined {
+  if (name.startsWith("computer_use_")) {
+    return name;
+  }
+  if (name !== "skill_execute" || input === null || typeof input !== "object") {
+    return undefined;
+  }
+  const tool = (input as Record<string, unknown>).tool;
+  return typeof tool === "string" && tool.startsWith("computer_use_")
+    ? tool
+    : undefined;
+}
+
+/** True when a tool call is backed by the host computer-use capability. */
+export function isComputerUseToolCall(name: string, input: unknown): boolean {
+  return resolveComputerUseToolName(name, input) !== undefined;
+}

package/src/api/events/assistant-outbound-attachment.ts

@@ -28,6 +28,9 @@ export const AssistantOutboundAttachmentSchema = z.object({
    *  true and the client should hydrate via the /content endpoint. */
   data: z.string(),
   sourceType: z.enum(["sandbox_file", "host_file", "tool_block"]).optional(),
+  /** True when this existing computer-use screenshot was linked to the reply
+   *  automatically. Missing or false identifies legacy or explicit placement. */
+  computerUseScreenshot: z.boolean().optional(),
   /** Original file size in bytes. Present when `data` was omitted to
    *  keep payloads small. */
   sizeBytes: z.number().optional(),

package/src/api/events/assistant-turn-start.ts

@@ -20,10 +20,13 @@
 
 import { z } from "zod";
 
+import { ModeSessionSchema } from "../mode-session.js";
+
 export const AssistantTurnStartEventSchema = z.object({
   type: z.literal("assistant_turn_start"),
   messageId: z.string(),
   conversationId: z.string().optional(),
+  modeSession: ModeSessionSchema.optional(),
 });
 
 export type AssistantTurnStartEvent = z.infer<

package/src/api/events/desktop-activity-changed.ts

@@ -0,0 +1,10 @@
+import { z } from "zod";
+
+/** Transient browser activity changed; refetch desktop status. */
+export const DesktopActivityChangedEventSchema = z.object({
+  type: z.literal("desktop_activity_changed"),
+});
+
+export type DesktopActivityChangedEvent = z.infer<
+  typeof DesktopActivityChangedEventSchema
+>;

package/src/api/events/generation-handoff.ts

@@ -18,6 +18,7 @@
 
 import { z } from "zod";
 
+import { ModeSessionSchema } from "../mode-session.js";
 import { AssistantOutboundAttachmentSchema } from "./assistant-outbound-attachment.js";
 
 export const GenerationHandoffEventSchema = z.object({
@@ -30,6 +31,7 @@ export const GenerationHandoffEventSchema = z.object({
   queuedCount: z.number(),
   /** Database row id of the just-finished assistant turn. */
   messageId: z.string().optional(),
+  modeSession: ModeSessionSchema.optional(),
   attachments: z.array(AssistantOutboundAttachmentSchema).optional(),
   attachmentWarnings: z.array(z.string()).optional(),
 });