@vellumai/assistant 0.12.2 → 0.12.3-staging.2

package/src/daemon/conversation-attachments.ts

@@ -2,10 +2,14 @@ import type { PermissionPrompter } from "../permissions/prompter.js";
 import {
   attachInlineAttachmentToMessage,
   AttachmentUploadError,
+  getAttachmentsByIds,
   getFilePathForAttachment,
+  linkAttachmentToMessage,
   setAttachmentThumbnail,
 } from "../persistence/attachments-store.js";
-import type { ContentBlock } from "../providers/types.js";
+import { updateMessageMetadata } from "../persistence/conversation-crud.js";
+import { COMPUTER_USE_SCREENSHOT_ATTACHMENT_IDS_KEY } from "../persistence/conversation-types.js";
+import type { ContentBlock, ImageContent } from "../providers/types.js";
 import { getLogger } from "../util/logger.js";
 import {
   type ApproveHostRead,
@@ -14,7 +18,9 @@ import {
   contentBlocksToDrafts,
   deduplicateDrafts,
   type DirectiveRequest,
+  estimateBase64Bytes,
   resolveDirectives,
+  toolImageFilename,
   validateDrafts,
 } from "./assistant-attachments.js";
 import type { UserMessageAttachment } from "./message-protocol.js";
@@ -88,6 +94,19 @@ export interface AttachmentResolutionResult {
   emittedAttachments: UserMessageAttachment[];
   directiveWarnings: string[];
   persistedFiles: PersistedAttachmentFile[];
+  /** Attachment ids successfully linked to the target assistant row. */
+  linkedAttachmentIds: string[];
+  computerUseScreenshotAttachmentIds: string[];
+}
+
+export interface ComputerUseScreenshotCandidate {
+  toolName: string;
+  block: ImageContent;
+}
+
+interface ResolvedAttachmentDraft extends AssistantAttachmentDraft {
+  existingAttachmentId?: string;
+  computerUseScreenshot?: boolean;
 }
 
 /**
@@ -102,10 +121,13 @@ export async function resolveAssistantAttachments(
   approveHostRead: ApproveHostRead,
   lastAssistantMessageId: string | undefined,
   toolContentBlockToolNames?: ReadonlyMap<number, string>,
+  computerUseScreenshotCandidate?: ComputerUseScreenshotCandidate,
 ): Promise<AttachmentResolutionResult> {
-  let assistantAttachments: AssistantAttachmentDraft[] = [];
+  let assistantAttachments: ResolvedAttachmentDraft[] = [];
   const emittedAttachments: UserMessageAttachment[] = [];
   const persistedFiles: PersistedAttachmentFile[] = [];
+  const linkedAttachmentIds: string[] = [];
+  const computerUseScreenshotAttachmentIds: string[] = [];
 
   const recordPersistedFile = (draft: AssistantAttachmentDraft): void => {
     if (draft.sourcePath) {
@@ -128,7 +150,8 @@ export async function resolveAssistantAttachments(
 
   if (
     accumulatedDirectives.length > 0 ||
-    accumulatedToolContentBlocks.length > 0
+    accumulatedToolContentBlocks.length > 0 ||
+    computerUseScreenshotCandidate !== undefined
   ) {
     const directiveDrafts =
       accumulatedDirectives.length > 0
@@ -159,10 +182,42 @@ export async function resolveAssistantAttachments(
       "Directive resolution complete",
     );
 
-    const toolDrafts = contentBlocksToDrafts(
+    const toolDrafts: ResolvedAttachmentDraft[] = contentBlocksToDrafts(
       accumulatedToolContentBlocks,
       toolContentBlockToolNames,
     );
+    if (computerUseScreenshotCandidate) {
+      const { block, toolName } = computerUseScreenshotCandidate;
+      const filename =
+        block.source.filename ??
+        toolImageFilename(block.source.media_type, toolName);
+      if (block.source.type === "workspace_ref") {
+        const stored = getAttachmentsByIds([block.source.attachmentId], {
+          hydrateFileData: true,
+        })[0];
+        if (stored?.dataBase64) {
+          toolDrafts.push({
+            sourceType: "tool_block",
+            filename,
+            mimeType: stored.mimeType,
+            dataBase64: stored.dataBase64,
+            sizeBytes: stored.sizeBytes,
+            kind: "image",
+            existingAttachmentId: stored.id,
+            computerUseScreenshot: true,
+          });
+        }
+      } else {
+        toolDrafts.push({
+          sourceType: "tool_block",
+          filename,
+          mimeType: block.source.media_type,
+          dataBase64: block.source.data,
+          sizeBytes: estimateBase64Bytes(block.source.data),
+          kind: "image",
+        });
+      }
+    }
     // Most recent tool outputs first so deduplication keeps the latest version.
     toolDrafts.reverse();
     const merged = deduplicateDrafts([
@@ -195,14 +250,27 @@ export async function resolveAssistantAttachments(
       const draft = assistantAttachments[i];
       let stored;
       try {
-        stored = await attachInlineAttachmentToMessage(
-          lastAssistantMessageId,
-          i,
-          draft.filename,
-          draft.mimeType,
-          draft.dataBase64,
-          { skipSizeLimit: true },
-        );
+        stored = draft.existingAttachmentId
+          ? getAttachmentsByIds([
+              linkAttachmentToMessage(
+                lastAssistantMessageId,
+                draft.existingAttachmentId,
+                i,
+              ),
+            ])[0]
+          : await attachInlineAttachmentToMessage(
+              lastAssistantMessageId,
+              i,
+              draft.filename,
+              draft.mimeType,
+              draft.dataBase64,
+              { skipSizeLimit: true },
+            );
+        if (!stored) {
+          throw new Error(
+            `Attachment not found: ${draft.existingAttachmentId}`,
+          );
+        }
       } catch (err) {
         if (err instanceof AttachmentUploadError) {
           log.warn(
@@ -241,6 +309,7 @@ export async function resolveAssistantAttachments(
       }
 
       recordPersistedFile(draft);
+      linkedAttachmentIds.push(stored.id);
       emittedAttachments.push({
         id: stored.id,
         filename: draft.filename,
@@ -250,6 +319,16 @@ export async function resolveAssistantAttachments(
         ...(omitData ? { sizeBytes: draft.sizeBytes } : {}),
         fileBacked: true,
         ...(thumbnailData ? { thumbnailData } : {}),
+        ...(draft.computerUseScreenshot ? { computerUseScreenshot: true } : {}),
+      });
+      if (draft.computerUseScreenshot) {
+        computerUseScreenshotAttachmentIds.push(stored.id);
+      }
+    }
+    if (computerUseScreenshotAttachmentIds.length > 0) {
+      updateMessageMetadata(lastAssistantMessageId, {
+        [COMPUTER_USE_SCREENSHOT_ATTACHMENT_IDS_KEY]:
+          computerUseScreenshotAttachmentIds,
       });
     }
   } else if (assistantAttachments.length > 0) {
@@ -271,5 +350,7 @@ export async function resolveAssistantAttachments(
     emittedAttachments,
     directiveWarnings,
     persistedFiles,
+    linkedAttachmentIds,
+    computerUseScreenshotAttachmentIds,
   };
 }

package/src/daemon/conversation-evictor.ts

@@ -6,8 +6,7 @@ const log = getLogger("conversation-evictor");
 
 /** Minimal interface a conversation must satisfy to be evictable. */
 export interface EvictableConversation {
-  isProcessing(): boolean;
-  hasQueuedMessages(): boolean;
+  hasInFlightWork(): boolean;
   dispose(): void;
 }
 
@@ -29,7 +28,7 @@ export interface EvictionResult {
   lruEvicted: number;
   /** Conversations evicted due to memory pressure. */
   memoryEvicted: number;
-  /** Conversations skipped because they were actively processing. */
+  /** Conversations skipped because they retained in-flight work. */
   skipped: number;
 }
 
@@ -74,11 +73,6 @@ export class ConversationEvictor {
     getSubagentManager().abortAllForParent(conversationId);
   }
 
-  /** Protect conversations with in-flight subagents from eviction. */
-  shouldProtect(conversationId: string): boolean {
-    return getSubagentManager().hasActiveChildren(conversationId);
-  }
-
   /** Record an access for the given conversation (resets its idle clock). */
   touch(conversationId: string): void {
     this.lastAccess.set(conversationId, Date.now());
@@ -141,11 +135,7 @@ export class ConversationEvictor {
       if (now - lastAccessTime < this.ttlMs) {
         continue;
       }
-      if (
-        conversation.isProcessing() ||
-        conversation.hasQueuedMessages() ||
-        this.shouldProtect(id)
-      ) {
+      if (conversation.hasInFlightWork()) {
         result.skipped++;
         continue;
       }
@@ -216,20 +206,11 @@ export class ConversationEvictor {
     log.debug({ conversationId: id }, "Evicted idle conversation");
   }
 
-  /**
-   * Return idle (non-processing) conversations sorted by last access time
-   * ascending (least recently used first).
-   */
+  /** Return evictable conversations in least-recently-used order. */
   private idleConversationsByLru(): Array<[string, EvictableConversation]> {
     const idle: Array<[string, EvictableConversation, number]> = [];
     for (const [id, conversation] of this.conversations) {
-      if (conversation.isProcessing()) {
-        continue;
-      }
-      if (conversation.hasQueuedMessages()) {
-        continue;
-      }
-      if (this.shouldProtect(id)) {
+      if (conversation.hasInFlightWork()) {
         continue;
       }
       idle.push([id, conversation, this.lastAccess.get(id) ?? 0]);

package/src/daemon/conversation-history.ts

@@ -16,6 +16,8 @@ import { relinkLlmRequestLogs } from "../persistence/llm-request-log-store.js";
 import { getSummaryFromContextMessage } from "../plugins/defaults/compaction/window-manager.js";
 import type { ContentBlock, Message } from "../providers/types.js";
 import { getLogger } from "../util/logger.js";
+import type { ConversationModeSessionCoordinator } from "./conversation-mode-session.js";
+import { bestEffortModeSessionTracking } from "./mode-session-tracking.js";
 import { startsNewTurn } from "./summarize-boundary.js";
 
 const log = getLogger("conversation-history");
@@ -360,6 +362,10 @@ export function consolidateAssistantMessages(
 export interface HistoryConversationContext {
   readonly conversationId: string;
   messages: Message[];
+  modeSessions: Pick<
+    ConversationModeSessionCoordinator,
+    "invalidateAllStructuralWaits"
+  >;
   isProcessing(): boolean;
 }
 
@@ -377,6 +383,10 @@ export function undo(conversation: HistoryConversationContext): number {
     return 0;
   }
 
+  bestEffortModeSessionTracking("conversation wait invalidation", () =>
+    conversation.modeSessions.invalidateAllStructuralWaits(),
+  );
+
   const removed = conversation.messages.length - lastUserIdx;
   conversation.messages = conversation.messages.slice(0, lastUserIdx);
 

package/src/daemon/conversation-interrupt-repair.ts

@@ -15,37 +15,6 @@ import type { Conversation } from "./conversation.js";
 
 const log = getLogger("conversation-interrupt-repair");
 
-/**
- * What the repair leaves behind, for the caller that has to decide whether the
- * model was told its turn was cut off.
- */
-export interface InterruptRepairResult {
-  /**
-   * True when the tail of the history carries a preempted `tool_result`,
-   * whether the agent loop's own abort handler wrote it or this repair did.
-   *
-   * That result is the model's notice that a message preempted it. False means
-   * the abort landed with no tool call in flight, so the notice has to ride on
-   * the interrupting user message instead.
-   */
-  preemptedToolResultOnTail: boolean;
-}
-
-/** Whether the last message answers a tool call with the preemption text. */
-function tailCarriesPreemptedToolResult(messages: Message[]): boolean {
-  const tail = messages.at(-1);
-  if (!tail || tail.role !== "user") {
-    return false;
-  }
-  return tail.content.some(
-    (block) =>
-      // Only a synthetic tool_result carries the preemption text; a
-      // server-side web_search_tool_result never does.
-      block.type === "tool_result" &&
-      block.content === PREEMPTED_TOOL_RESULT_TEXT,
-  );
-}
-
 /**
  * Give every abandoned `tool_use` block at the tail of the history a
  * `tool_result`, so the next provider request is well formed.
@@ -78,22 +47,18 @@ function tailCarriesPreemptedToolResult(messages: Message[]): boolean {
 export async function repairInterruptedToolUseBlocks(
   conversation: Conversation,
   options: { force?: boolean; requireDurable?: boolean } = {},
-): Promise<InterruptRepairResult> {
+): Promise<void> {
   const wasSteerArmed = conversation.pendingSteerRepair;
   const wasArmed = conversation.pendingInterruptRepair;
   if (!wasSteerArmed && !wasArmed && options.force !== true) {
-    return {
-      preemptedToolResultOnTail: tailCarriesPreemptedToolResult(
-        conversation.messages,
-      ),
-    };
+    return;
   }
   conversation.pendingSteerRepair = false;
   conversation.pendingInterruptRepair = false;
 
   const messages = conversation.messages;
   if (messages.length === 0) {
-    return { preemptedToolResultOnTail: false };
+    return;
   }
 
   // Walk backwards from the tail to find the last assistant message with
@@ -127,10 +92,7 @@ export async function repairInterruptedToolUseBlocks(
   if (pendingToolUseIds.length === 0) {
     // Nothing dangling either because the turn made no tool call before the
     // abort, or because the loop's abort handler already answered the batch.
-    // Only the second case leaves the model a notice.
-    return {
-      preemptedToolResultOnTail: tailCarriesPreemptedToolResult(messages),
-    };
+    return;
   }
 
   log.info(
@@ -181,5 +143,4 @@ export async function repairInterruptedToolUseBlocks(
       "Failed to persist the synthetic tool_result repair row; the in-memory history carries the repair and the next turn runs on it",
     );
   }
-  return { preemptedToolResultOnTail: true };
 }

package/src/daemon/conversation-interrupt.ts

@@ -316,13 +316,11 @@ export async function interruptRunningTurn(
   }
 
   let repaired = false;
-  let preemptedToolResultOnTail = false;
   try {
-    const repair = await repairInterruptedToolUseBlocks(conversation, {
+    await repairInterruptedToolUseBlocks(conversation, {
       force: true,
       requireDurable: true,
     });
-    preemptedToolResultOnTail = repair.preemptedToolResultOnTail;
     repaired = true;
   } catch (err) {
     // The repair row is not durable, so the caller must not write the
@@ -352,14 +350,12 @@ export async function interruptRunningTurn(
   // busy conversation with nothing running and no path back to idle.
   conversation.pendingInterruptActivityBridge = true;
 
-  // An abort that landed during the provider call answered no tool call, so
-  // nothing in the history tells the model its turn was cut off. Arm the note
-  // the interrupting user message carries instead. A tail that already holds a
-  // preempted `tool_result` needs none: that result says the same thing, and
-  // saying it twice in one prompt is noise.
-  if (!preemptedToolResultOnTail) {
-    conversation.pendingInterruptNote = true;
-  }
+  // Arm the note the interrupting user message carries. It is the one place
+  // the behavior after an interrupt is spelled out, so it rides on every
+  // handover, tool call stopped or not: a synthetic `tool_result` states only
+  // what happened to the one call it answers, and it lands ahead of the user's
+  // words rather than on them.
+  conversation.pendingInterruptNote = true;
 
   return "released";
 }

package/src/daemon/conversation-messaging.ts

@@ -63,7 +63,10 @@ import {
   syncMessageToDisk,
   updateMetaFile,
 } from "../persistence/conversation-disk-view.js";
-import { SIGHT_FRAME_ATTACHMENT_IDS_KEY } from "../persistence/conversation-types.js";
+import {
+  isEchoSuppressedUserMessage,
+  SIGHT_FRAME_ATTACHMENT_IDS_KEY,
+} from "../persistence/conversation-types.js";
 import {
   attachmentIdFragment,
   type ContentBlock,
@@ -72,10 +75,13 @@ import {
 import type { AuthContext } from "../runtime/auth/types.js";
 import { INTERRUPTED_TURN_NOTE_TEXT } from "../util/abort-reasons.js";
 import { getLogger } from "../util/logger.js";
+import type { ConversationModeSessionCoordinator } from "./conversation-mode-session.js";
 import type { MessageQueue } from "./conversation-queue-manager.js";
 import type { SlackInboundMessageMetadata } from "./handlers/shared.js";
 import type { UserMessageAttachment } from "./message-protocol.js";
+import { actorAuthorProvenance } from "./message-provenance.js";
 import type { ConversationTransportMetadata } from "./message-types/conversations.js";
+import { bestEffortModeSessionTracking } from "./mode-session-tracking.js";
 import {
   assembleUserContentBlocks,
   offloadLinkPlan,
@@ -230,6 +236,11 @@ export interface MessagingConversationContext {
   releaseProcessing(owner: number): boolean;
   abortController: AbortController | null;
   currentRequestId?: string;
+  currentActiveSurfaceId?: string;
+  readonly modeSessions?: Pick<
+    ConversationModeSessionCoordinator,
+    "acceptTurn" | "trackPersistedRow"
+  >;
   /** See {@link Conversation.currentTurnClientMessageId}. */
   currentTurnClientMessageId?: string;
   readonly queue: MessageQueue;
@@ -957,6 +968,13 @@ export interface PersistMessageOptions {
    * persisting a message the current actor just sent.
    */
   trustContext?: TrustContext;
+  /**
+   * The person whose own inbound message this row records, passed only by a
+   * caller relaying one (channel ingress and its retry replay). It names the
+   * row's author (`actorAuthorProvenance`). Machine-authored callers omit it,
+   * so their rows name no author whatever the conversation's trust is.
+   */
+  author?: TrustContext;
   /**
    * Persist the row without indexing it (no memory segments, embeddings, or
    * lexical-index entry). For machine-authored prompts that must not enter
@@ -1005,6 +1023,10 @@ export interface PersistMessageOptions {
    * what a consumer that must not misattribute a turn to a surface needs.
    */
   requestClientOs?: string;
+  /** Existing structural surface whose accepted action created this turn. */
+  activeSurfaceId?: string;
+  /** Whether mode-session stamping publishes its own history invalidation. */
+  publishModeSessionChanges?: boolean;
   /**
    * Which of `attachments`, by the id the caller holds, arrived as ambient
    * camera frames rather than files the user picked. Stamps
@@ -1229,12 +1251,14 @@ export async function persistQueuedMessageBody(
       channelInbound: rawChannelInbound,
       scripted: rawScriptedFromMetadata,
       clientOsFromRequest: _rawClientOsFromRequest,
+      modeSession: _rawModeSession,
       ...metadataWithoutSlackInbound
     } = (metadata ?? {}) as Record<string, unknown> & {
       slackInbound?: SlackInboundMessageMetadata;
       channelInbound?: ProviderMessageMetadata;
       scripted?: unknown;
       clientOsFromRequest?: unknown;
+      modeSession?: unknown;
     };
     const slackMeta = buildSlackMetaForPersistence({
       slackInbound: rawSlackInbound,
@@ -1307,6 +1331,13 @@ export async function persistQueuedMessageBody(
     const mergedMetadata = {
       ...metadataWithoutSlackInbound,
       ...provenance,
+      // A scripted row, or one the repo classes as machine-injected (hidden,
+      // ACP or subagent notification, background event), is not a person's
+      // own words, so even a relayed author is not named on one.
+      ...(resolvedScripted ||
+      isEchoSuppressedUserMessage(metadataWithoutSlackInbound)
+        ? {}
+        : actorAuthorProvenance(options.author)),
       ...(turnCtx
         ? {
             userMessageChannel: turnCtx.userMessageChannel,
@@ -1440,6 +1471,26 @@ export async function persistQueuedMessageBody(
       return { id: persistedUserMessage.id, deduplicated: true };
     }
 
+    const activeSurfaceId =
+      options.activeSurfaceId ?? ctx.currentActiveSurfaceId;
+    bestEffortModeSessionTracking("persist_user_message", () => {
+      ctx.modeSessions?.acceptTurn(
+        requestId,
+        activeSurfaceId
+          ? { kind: "surface", responseId: activeSurfaceId }
+          : undefined,
+      );
+      ctx.modeSessions?.trackPersistedRow(
+        requestId,
+        persistedUserMessage.id,
+        persistedUserMessage.createdAt,
+        {
+          startsDisplayBoundary: false,
+          publishMessagesChanged: options.publishModeSessionChanges ?? true,
+        },
+      );
+    });
+
     if (turnCtx) {
       setConversationOriginChannelIfUnset(
         ctx.conversationId,
@@ -1560,10 +1611,12 @@ export async function persistQueuedMessageBody(
       updateMessageMetadata(persistedUserMessage.id, { attachmentStoredPaths });
     }
 
-    // An interrupt whose abort landed before the turn made a tool call left
-    // the model no `tool_result` saying it was cut off, so this message
-    // carries the notice instead. Consumed here, once: a second message must
-    // not repeat a note about a turn it did not interrupt. Stamped after the
+    // This is the first user row after a handover, so it carries the note that
+    // tells the model what to do about the work the handover stopped. The note
+    // goes on this row because that work sits directly above it in the history,
+    // which is also why it is usually the interrupting message and never has to
+    // be the one that armed the flag. Consumed here, once: a later message sits
+    // under a completed turn, where the note would be stale. Stamped after the
     // insert, like the stored paths above, so a persist that never lands
     // leaves the flag armed for the send that replaces it.
     const carriesInterruptNote = ctx.pendingInterruptNote === true;