@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2

package/src/daemon/conversation-process.ts

@@ -1089,6 +1089,7 @@ async function drainSingleMessage(
       content: resolvedContent,
       attachments: next.attachments,
       requestId: next.requestId,
+      activeSurfaceId: next.activeSurfaceId,
       metadata: { ...next.metadata, sentAt: next.sentAt },
       displayContent: next.displayContent,
       clientMessageId: next.clientMessageId,
@@ -1155,6 +1156,7 @@ async function drainSingleMessage(
       messageId: userMessageId,
       requestId: next.requestId,
       clientMessageId: next.clientMessageId,
+      modeSession: conversation.modeSessions.getTurnOwner(next.requestId),
     });
     // The row this echo announces is already durably persisted, so advance
     // the snapshot↔stream anchor to the echo's seq (stamped inline by the
@@ -1464,6 +1466,7 @@ async function drainBatch(
         content: qmContent,
         attachments: qm.attachments,
         requestId: qm.requestId,
+        activeSurfaceId: qm.activeSurfaceId,
         metadata: { ...qm.metadata, sentAt: qm.sentAt },
         displayContent: qm.displayContent,
         clientMessageId: qm.clientMessageId,
@@ -1574,6 +1577,7 @@ async function drainBatch(
         messageId: lastUserMessageId,
         requestId: qm.requestId,
         clientMessageId: qm.clientMessageId,
+        modeSession: conversation.modeSessions.getTurnOwner(qm.requestId),
       });
       // Advance the snapshot↔stream anchor to this echo's seq — the batched
       // row persisted just above and the agent loop for the batch has not
@@ -2298,6 +2302,7 @@ export async function processMessage(
       content: resolvedContent,
       attachments,
       requestId,
+      activeSurfaceId,
       displayContent,
       scripted,
       ...(callerMetadata ? { metadata: callerMetadata } : {}),

package/src/daemon/conversation-runtime-assembly.ts

@@ -8,6 +8,8 @@
 import { statSync } from "node:fs";
 import { join } from "node:path";
 
+import { TrustClassSchema } from "@vellumai/gateway-client";
+
 import {
   getApp,
   getAppDirPath,
@@ -81,7 +83,6 @@ import type {
 import type { ContentBlock, Message } from "../providers/types.js";
 import type { TrustClass } from "../runtime/actor-trust-resolver.js";
 import { resolveCapabilities } from "../runtime/capabilities.js";
-import { trustClassSchema } from "../runtime/trust-class.js";
 import type { SubagentState } from "../subagent/types.js";
 import { TERMINAL_STATUSES } from "../subagent/types.js";
 import { canonicalizeInboundIdentity } from "../util/canonicalize-identity.js";
@@ -903,6 +904,11 @@ export function buildChannelCapabilityBlock(
         "- Do NOT use markdown tables — use bullet lists instead. No markdown headers — use **bold** or CAPS for emphasis.",
       );
     }
+    if (caps.channel === "email") {
+      lines.push(
+        "- Conversation text is not emailed. To reply, run `assistant email send` (see `assistant email send --help`). Use `--reply-to` to keep the thread. Skip a reply only when none is needed.",
+      );
+    }
   }
 
   // Inject group chat etiquette only when the chat type indicates a multi-party
@@ -1183,7 +1189,7 @@ function placeholderForBlockType(type: ContentBlock["type"]): string | null {
 function rowToRenderable(row: SlackTranscriptInputRow): RenderableSlackMessage {
   const slackMeta = readSlackMetadataFromMessageMetadata(row.metadata);
   const provenanceTrustClass = row.metadata
-    ? trustClassSchema.safeParse(
+    ? TrustClassSchema.safeParse(
         safeParseRecord(row.metadata).provenanceTrustClass,
       ).data
     : undefined;
@@ -2035,20 +2041,6 @@ export async function composeInjectorChain(ctx: TurnContext): Promise<string> {
  */
 const DEFAULT_PLACEMENT: InjectionPlacement = "append-user-tail";
 
-/**
- * Count leading memory-prefix blocks on a user message's `content`.
- *
- * Delegates to {@link countMemoryPrefixBlocks} from
- * `memory/graph/conversation-graph-memory.js` — the canonical state-machine
- * for locating the memory-prefix boundary. Reusing it here keeps the
- * PKB-context / PKB-reminder / NOW splice rules aligned on a single source
- * of truth so their ordering relative to any memory prefix is stable and
- * testable.
- */
-function countMemoryPrefixBlocksOnContent(content: ContentBlock[]): number {
-  return countMemoryPrefixBlocks(content);
-}
-
 /**
  * Apply one injector block to a `runMessages` array according to its
  * declared {@link InjectionPlacement}:
@@ -2101,9 +2093,7 @@ function applyInjectionBlock(
         { ...userTail, content: [...userTail.content, textBlock] },
       ];
     case "after-memory-prefix": {
-      const memoryPrefixCount = countMemoryPrefixBlocksOnContent(
-        userTail.content,
-      );
+      const memoryPrefixCount = countMemoryPrefixBlocks(userTail.content);
       return [
         ...runMessages.slice(0, -1),
         {
@@ -2161,7 +2151,7 @@ function stripTailV2DynamicMemoryPrefix(
   if (!last || last.role !== "user") {
     return messages;
   }
-  const prefixCount = countMemoryPrefixBlocksOnContent(last.content);
+  const prefixCount = countMemoryPrefixBlocks(last.content);
   if (prefixCount === 0) {
     return messages;
   }

package/src/daemon/conversation-skill-tools.ts

@@ -463,6 +463,7 @@ export function projectSkillTools(
       skill.directoryPath,
       currentHash,
       skill.bundled,
+      skill.owner?.kind === "plugin" ? skill.owner.id : undefined,
     );
 
     if (tools.length > 0) {

package/src/daemon/conversation-store.ts

@@ -550,10 +550,10 @@ export function clearAllActiveConversations(): number {
 /**
  * Evict in-memory conversations after a config/prompt reload so the next
  * turn rebuilds them against the new config. Conversations with in-flight work
- * (a live turn, a queued successor, or an active subagent) are marked stale
- * and rebuilt by `getOrCreateConversation` once that work finishes. Idle
- * conversations are disposed and dropped. Also used when provider credentials
- * change.
+ * are marked stale and rebuilt by `getOrCreateConversation` once that work
+ * finishes. This includes live mode-session sources and process-local
+ * structural continuations. Idle conversations are disposed and dropped. Also
+ * used when provider credentials change.
  */
 export function evictConversationsForReload(): void {
   const subagentManager = getSubagentManager();

package/src/daemon/conversation-surface-snapshots.ts

@@ -0,0 +1,135 @@
+/**
+ * Canonical conversation surface snapshots for host readers.
+ *
+ * Combines an unbounded persisted `ui_surface` scan with the loaded
+ * conversation's live `surfaceState` and current-turn snapshots. Hits from
+ * the persisted scan are never written into `surfaceState`.
+ */
+
+import { coerceSurfaceDataRecord } from "../api/surfaces.js";
+import { getMessages } from "../persistence/conversation-crud.js";
+import { resolveCapabilities } from "../runtime/capabilities.js";
+import { isPlainObject } from "../util/object.js";
+import { findConversationOrSubagent } from "./conversation-registry.js";
+import { isRowVisibleToUntrustedActor } from "./message-provenance.js";
+
+export interface ConversationSurfaceSnapshot {
+  surfaceId: string;
+  surfaceType: string;
+  data: Record<string, unknown>;
+  completed: boolean;
+  completionSummary?: string;
+}
+
+function cloneSurfaceData(value: unknown): Record<string, unknown> {
+  const record = coerceSurfaceDataRecord(value);
+  try {
+    return structuredClone(record);
+  } catch {
+    return JSON.parse(JSON.stringify(record)) as Record<string, unknown>;
+  }
+}
+
+function surfaceTypeFromBlock(value: unknown): string {
+  return typeof value === "string" && value.trim().length > 0
+    ? value
+    : "dynamic_page";
+}
+
+function rememberOrder(order: string[], surfaceId: string): void {
+  if (!order.includes(surfaceId)) {
+    order.push(surfaceId);
+  }
+}
+
+/**
+ * Current surface snapshots for a loaded conversation.
+ *
+ * Fails closed to an empty list when the conversation is not already
+ * loaded or the turn-or-resting trust snapshot is missing. Untrusted
+ * views reuse {@link isRowVisibleToUntrustedActor} and do not add
+ * live-only cards that never appeared in a visible persisted row.
+ */
+export function listConversationSurfaceSnapshots(
+  conversationId: string,
+): ConversationSurfaceSnapshot[] {
+  const conversation = findConversationOrSubagent(conversationId);
+  if (!conversation) {
+    return [];
+  }
+  const trust = conversation.getTurnOrRestingTrust();
+  if (!trust) {
+    return [];
+  }
+  const canAccessMemory = resolveCapabilities(trust.trustClass).canAccessMemory;
+  const filterByProvenance = !canAccessMemory;
+
+  const order: string[] = [];
+  const records = new Map<string, ConversationSurfaceSnapshot>();
+
+  const rows = getMessages(conversationId);
+  for (const row of rows) {
+    if (filterByProvenance && !isRowVisibleToUntrustedActor(row.metadata)) {
+      continue;
+    }
+    if (!Array.isArray(row.content)) {
+      continue;
+    }
+    for (const block of row.content) {
+      if (!isPlainObject(block) || block.type !== "ui_surface") {
+        continue;
+      }
+      const surfaceId = block.surfaceId;
+      if (typeof surfaceId !== "string" || surfaceId.length === 0) {
+        continue;
+      }
+      const completionSummary =
+        typeof block.completionSummary === "string"
+          ? block.completionSummary
+          : undefined;
+      rememberOrder(order, surfaceId);
+      records.set(surfaceId, {
+        surfaceId,
+        surfaceType: surfaceTypeFromBlock(block.surfaceType),
+        data: cloneSurfaceData(block.data),
+        completed: block.completed === true,
+        ...(completionSummary !== undefined ? { completionSummary } : {}),
+      });
+    }
+  }
+
+  for (const [surfaceId, entry] of conversation.surfaceState.entries()) {
+    const existing = records.get(surfaceId);
+    if (!existing && filterByProvenance) {
+      continue;
+    }
+    rememberOrder(order, surfaceId);
+    records.set(surfaceId, {
+      surfaceId,
+      surfaceType: entry.surfaceType,
+      data: cloneSurfaceData(entry.data),
+      completed: existing?.completed === true,
+      ...(existing?.completionSummary !== undefined
+        ? { completionSummary: existing.completionSummary }
+        : {}),
+    });
+  }
+
+  for (const surface of conversation.currentTurnSurfaces) {
+    const existing = records.get(surface.surfaceId);
+    const completionSummary =
+      typeof surface.completionSummary === "string"
+        ? surface.completionSummary
+        : existing?.completionSummary;
+    rememberOrder(order, surface.surfaceId);
+    records.set(surface.surfaceId, {
+      surfaceId: surface.surfaceId,
+      surfaceType: surface.surfaceType,
+      data: cloneSurfaceData(surface.data),
+      completed: surface.completed === true || existing?.completed === true,
+      ...(completionSummary !== undefined ? { completionSummary } : {}),
+    });
+  }
+
+  return order.map((surfaceId) => records.get(surfaceId)!);
+}

package/src/daemon/conversation-surfaces.ts

@@ -28,6 +28,8 @@ import {
   resolveEffectiveAppHtml,
   updateApp,
 } from "../apps/app-store.js";
+import { executeDesktopComputerUse } from "../desktop/desktop-computer-use.js";
+import { canUseVirtualDesktop } from "../desktop/virtual-desktop-feature.js";
 import { recordActivationEvent } from "../onboarding/onboarding-events-store.js";
 import {
   getMessages,
@@ -57,6 +59,7 @@ import {
 import { resolveAppId } from "../tools/apps/resolve-app-id.js";
 import { formatDesktopAppRequired } from "../tools/capability-offer.js";
 import { POINT_AT_PROXY_TOOL } from "../tools/computer-use/skill-proxy-bridge.js";
+import { computerUseTarget } from "../tools/computer-use/target.js";
 import type { ToolExecutionResult } from "../tools/types.js";
 import { getLogger } from "../util/logger.js";
 import { isPlainObject } from "../util/object.js";
@@ -70,7 +73,7 @@ import {
   type SurfaceShowPair,
   type SurfaceStateEntry,
 } from "./conversation-surface-state.js";
-import type { HostCuProxy } from "./host-cu-proxy.js";
+import { HostCuProxy } from "./host-cu-proxy.js";
 import { resolveHostCuTarget } from "./host-cu-target.js";
 import type {
   AnySurfaceData,
@@ -89,9 +92,11 @@ import type {
 } from "./message-protocol.js";
 import { INTERACTIVE_SURFACE_TYPES } from "./message-protocol.js";
 import { isRowVisibleToUntrustedActor } from "./message-provenance.js";
+import { bestEffortModeSessionTracking } from "./mode-session-tracking.js";
 import type { TrustContext } from "./trust-context-types.js";
 import { restingTrust } from "./trust-context-types.js";
 import { turnActorPrincipalId } from "./turn-actor.js";
+import { virtualDesktopContext } from "./virtual-desktop-context.js";
 export {
   buildSurfaceShowPair,
   type CurrentTurnSurface,
@@ -178,12 +183,61 @@ const NON_BLOCKING_PENDING_SURFACE_TYPES = new Set<SurfaceType>([
 export function hasBlockingPendingSurface(ctx: {
   pendingSurfaceActions: Map<string, { surfaceType: SurfaceType }>;
 }): boolean {
-  for (const entry of ctx.pendingSurfaceActions.values()) {
+  return blockingPendingSurfaceIds(ctx).length > 0;
+}
+
+export function blockingPendingSurfaceIds(ctx: {
+  pendingSurfaceActions: Map<string, { surfaceType: SurfaceType }>;
+}): string[] {
+  const ids: string[] = [];
+  for (const [surfaceId, entry] of ctx.pendingSurfaceActions) {
     if (!NON_BLOCKING_PENDING_SURFACE_TYPES.has(entry.surfaceType)) {
-      return true;
+      ids.push(surfaceId);
     }
   }
-  return false;
+  return ids;
+}
+
+function acceptModeSessionSurfaceResponse(
+  ctx: Conversation,
+  requestId: string,
+  surfaceId: string,
+): void {
+  bestEffortModeSessionTracking("surface response admission", () =>
+    ctx.modeSessions?.acceptTurn(requestId, {
+      kind: "surface",
+      responseId: surfaceId,
+    }),
+  );
+}
+
+function invalidateModeSessionSurfaceWait(
+  ctx: Partial<Pick<Conversation, "modeSessions">>,
+  surfaceId: string,
+): void {
+  bestEffortModeSessionTracking("surface wait invalidation", () =>
+    ctx.modeSessions?.invalidateStructuralWait({
+      kind: "surface",
+      responseId: surfaceId,
+    }),
+  );
+}
+
+function settleModeSessionSurfaceWait(
+  ctx: Pick<Conversation, "conversationId" | "modeSessions">,
+  surfaceId: string,
+): void {
+  try {
+    ctx.modeSessions?.settleStructuralWait(
+      { kind: "surface", responseId: surfaceId },
+      { status: "completed", endReason: "surface_launch_settled" },
+    );
+  } catch (err) {
+    log.warn(
+      { err, conversationId: ctx.conversationId, surfaceId },
+      "Mode-session launcher settlement failed",
+    );
+  }
 }
 
 /**
@@ -1292,7 +1346,8 @@ export function cleanupStandaloneSurface(
     | "lastSurfaceAction"
     | "accumulatedSurfaceState"
     | "surfaceUndoStacks"
-  >,
+  > &
+    Partial<Pick<Conversation, "modeSessions">>,
   surfaceId: string,
 ): void {
   const entry = ctx.pendingStandaloneSurfaces?.get(surfaceId);
@@ -1305,6 +1360,7 @@ export function cleanupStandaloneSurface(
   ctx.lastSurfaceAction.delete(surfaceId);
   ctx.accumulatedSurfaceState.delete(surfaceId);
   ctx.surfaceUndoStacks.delete(surfaceId);
+  invalidateModeSessionSurfaceWait(ctx, surfaceId);
 
   // Record a tombstone so late client actions are silently dropped.
   if (ctx.recentlyCompletedStandaloneSurfaces) {
@@ -2110,6 +2166,7 @@ export async function handleSurfaceAction(
       ...(anchorMessageId ? { anchorMessageId } : {}),
       ...(originTrustContext ? { originTrustContext } : {}),
     });
+    settleModeSessionSurfaceWait(ctx, surfaceId);
     log.info(
       { originConversationId: ctx.conversationId, conversationId, surfaceId },
       "launch_conversation dispatched inline from surface action",
@@ -2284,6 +2341,8 @@ export async function handleSurfaceAction(
       return QUEUE_FULL_RESULT;
     }
 
+    acceptModeSessionSurfaceResponse(ctx, requestId, surfaceId);
+
     // Terminal user commit accepted — record the activation milestone if this
     // surface was tagged (best-effort, no-op otherwise). Deferred until after
     // the rejection check so a queue-full click doesn't over-report a moment
@@ -2310,6 +2369,7 @@ export async function handleSurfaceAction(
         type: "user_message_echo",
         text: prompt,
         conversationId: ctx.conversationId,
+        modeSession: ctx.modeSessions.getTurnOwner(requestId),
       });
     }
 
@@ -2537,6 +2597,8 @@ export async function handleSurfaceAction(
     return QUEUE_FULL_RESULT;
   }
 
+  acceptModeSessionSurfaceResponse(ctx, requestId, surfaceId);
+
   // Terminal user commit accepted — record the activation milestone if this
   // surface was tagged (best-effort, no-op otherwise). Deferred until after the
   // rejection check so a queue-full click doesn't over-report a moment (and the
@@ -2563,6 +2625,7 @@ export async function handleSurfaceAction(
       type: "user_message_echo",
       text: prompt,
       conversationId: ctx.conversationId,
+      modeSession: ctx.modeSessions.getTurnOwner(requestId),
     });
   }
   if (result.queued) {
@@ -3161,6 +3224,28 @@ export async function surfaceProxyResolver(
 ): Promise<ToolExecutionResult> {
   // Route CU proxy tools (all computer_use_* action tools)
   if (toolName.startsWith("computer_use_")) {
+    const desktopContext = virtualDesktopContext(ctx, signal);
+    if (
+      toolName !== POINT_AT_PROXY_TOOL &&
+      computerUseTarget(input, desktopContext) === "assistant-desktop"
+    ) {
+      if (!canUseVirtualDesktop(desktopContext)) {
+        return {
+          content:
+            "The assistant desktop requires an identified guardian and an enabled platform-hosted assistant.",
+          isError: true,
+        };
+      }
+      if (!ctx.hostCuProxy) {
+        ctx.setHostCuProxy(new HostCuProxy());
+      }
+      return executeDesktopComputerUse(
+        toolName,
+        input,
+        desktopContext,
+        ctx.hostCuProxy!,
+      );
+    }
     const hostCuProxy = ensureHostCuProxy(ctx);
     if (!hostCuProxy || !hostCuProxy.isAvailable()) {
       return {
@@ -3180,6 +3265,15 @@ export async function surfaceProxyResolver(
           : typeof input.answer === "string"
             ? input.answer
             : "Task complete";
+      bestEffortModeSessionTracking("computer completion", () =>
+        ctx.computerUseModeSessions.endTask({
+          turnId: ctx.currentRequestId,
+          source: {
+            sourceId: hostCuProxy.sourceId,
+            generation: hostCuProxy.resetGeneration,
+          },
+        }),
+      );
       hostCuProxy.endTask(ctx.conversationId);
       return { content: summary, isError: false };
     }
@@ -3215,9 +3309,11 @@ export async function surfaceProxyResolver(
     // `maxStepsPerSession` would let a long walkthrough exhaust a budget
     // meant for actions and be told to call `computer_use_done`, which has
     // nothing to do with what it was doing.
+    const activityAt = Date.now();
     if (toolName !== POINT_AT_PROXY_TOOL) {
       hostCuProxy.recordAction(toolName, input, reasoning);
     }
+    const turnId = ctx.currentRequestId;
     return hostCuProxy.request(
       toolName,
       input,
@@ -3227,6 +3323,18 @@ export async function surfaceProxyResolver(
       signal,
       targetClientId,
       sourceActorPrincipalId,
+      toolName !== POINT_AT_PROXY_TOOL && turnId
+        ? () => {
+            ctx.computerUseModeSessions.recordAction({
+              turnId,
+              source: {
+                sourceId: hostCuProxy.sourceId,
+                generation: hostCuProxy.resetGeneration,
+              },
+              at: activityAt,
+            });
+          }
+        : undefined,
     );
   }
 

package/src/daemon/conversation-tool-setup.ts

@@ -19,6 +19,7 @@ import {
   resolveSendUserMessageActive,
   SEND_USER_MESSAGE_TOOL_NAME,
 } from "../config/send-user-message-gate.js";
+import { canUseVirtualDesktop } from "../desktop/virtual-desktop-feature.js";
 import { supportsChannelReaction } from "../messaging/providers/index.js";
 import type { PermissionPrompter } from "../permissions/prompter.js";
 import type { SecretPrompter } from "../permissions/secret-prompter.js";
@@ -87,6 +88,7 @@ import {
 } from "./doordash-steps.js";
 import { runPostExecutionSideEffects } from "./tool-side-effects.js";
 import { FALLBACK_TURN_TRUST, resolveTrustClass } from "./trust-context.js";
+import { virtualDesktopContext } from "./virtual-desktop-context.js";
 
 const log = getLogger("conversation-tool-setup");
 
@@ -462,8 +464,7 @@ export function createToolExecutor(
       sendUserMessageActive: resolveSendUserMessageActive(ctx),
       toolUseId,
       isPlatformHosted: getIsPlatform(),
-      transportInterface: ctx.transportInterface,
-      clientOs: resolveTurnClientOs(ctx).clientOs,
+      ...resolveTurnClientOs(ctx),
       overrideProfile: ctx.currentTurnOverrideProfile,
       cronRunId: ctx.currentTurnCronRunId,
       invokingCallSite: ctx.currentCallSite ?? "mainAgent",
@@ -743,6 +744,7 @@ function isToolSupportedOnClientOs(name: string, ctx: Conversation): boolean {
     clientOs,
     transportInterface,
     sourceActorPrincipalId: ctx.getTurnActorPrincipalId?.(),
+    trustClass: virtualDesktopContext(ctx).trustClass,
   });
 }
 
@@ -1191,7 +1193,7 @@ export function createResolveToolsCallback(
         definition.name === "bash"
           ? {
               ...definition,
-              description: `${definition.description} For browser tasks, use assistant browser navigate --url <url> directly. It installs the virtual desktop if needed, starts Chrome, and completes the action in one call. Use timeout_seconds: ${getConfig().timeouts.shellMaxTimeoutSec} for first use; setup progress is visible in the Virtual desktop panel. Use assistant browser --help for other browser actions. Use this managed path even if saved notes describe manual setup. Do not install packages or launch Chrome, X servers, or screenshot scripts yourself.`,
+              description: `${definition.description} For browser tasks, use assistant browser navigate --url <url> directly. Chrome and desktop components are included in the assistant image. The command starts Chrome and completes the action without installing dependencies. If components are missing, report the image problem. Use assistant browser --help for other browser actions. Use this managed path even if saved notes describe manual setup. Do not install packages or launch Chrome, X servers, or screenshot scripts yourself.`,
             }
           : definition,
       );
@@ -1207,6 +1209,9 @@ export function createResolveToolsCallback(
     const effectivePreactivated = [
       ...DEFAULT_PREACTIVATED_SKILL_IDS,
       ...(ctx.preactivatedSkillIds ?? []),
+      ...(canUseVirtualDesktop(virtualDesktopContext(ctx))
+        ? ["computer-use"]
+        : []),
     ];
     const projection = projectSkillTools(history, {
       preactivatedSkillIds: effectivePreactivated,