@vellumai/assistant 0.10.8 → 0.10.9-dev.202607152225.ec6fdbf

package/src/daemon/conversation-surfaces.ts

@@ -11,12 +11,12 @@ import {
   getApp,
   getAppDirPath,
   getAppPreview,
-  isMultifileApp,
   listAppsByConversation,
   resolveAppDir,
   resolveEffectiveAppHtml,
   updateApp,
 } from "../apps/app-store.js";
+import { getVoiceResumeHandler } from "../live-voice/live-voice-resume-registry.js";
 import { recordActivationEvent } from "../onboarding/onboarding-events-store.js";
 import {
   getMessages,
@@ -132,15 +132,7 @@ function persistSurfaceData(
   try {
     const rows = getMessages(conversationId);
     for (let r = rows.length - 1; r >= 0; r--) {
-      let parsed: unknown[];
-      try {
-        const result = JSON.parse(rows[r].content);
-        if (!Array.isArray(result)) continue;
-        parsed = result;
-      } catch {
-        // Plain-text content rows — skip and keep scanning.
-        continue;
-      }
+      const parsed: unknown[] = rows[r].content;
       let found = false;
       for (const pb of parsed) {
         const rb = pb as Record<string, unknown>;
@@ -191,7 +183,9 @@ export function scheduleSurfaceDataPersist(
  */
 export function flushSurfaceDataPersist(surfaceId: string): void {
   const pending = pendingSurfacePersists.get(surfaceId);
-  if (!pending) return;
+  if (!pending) {
+    return;
+  }
   clearTimeout(pending.timer);
   pendingSurfacePersists.delete(surfaceId);
   persistSurfaceData(pending.conversationId, surfaceId, pending.data);
@@ -204,7 +198,9 @@ export function flushSurfaceDataPersist(surfaceId: string): void {
  */
 export function cancelSurfaceDataPersist(surfaceId: string): void {
   const pending = pendingSurfacePersists.get(surfaceId);
-  if (!pending) return;
+  if (!pending) {
+    return;
+  }
   clearTimeout(pending.timer);
   pendingSurfacePersists.delete(surfaceId);
 }
@@ -221,7 +217,9 @@ export function cancelPendingSurfaceDataPersists(
   conversationId?: string,
 ): void {
   for (const [surfaceId, pending] of pendingSurfacePersists) {
-    if (conversationId && pending.conversationId !== conversationId) continue;
+    if (conversationId && pending.conversationId !== conversationId) {
+      continue;
+    }
     clearTimeout(pending.timer);
     pendingSurfacePersists.delete(surfaceId);
   }
@@ -236,7 +234,9 @@ export function cancelPendingSurfaceDataPersists(
  */
 export function flushPendingSurfaceDataPersists(conversationId?: string): void {
   for (const [surfaceId, pending] of pendingSurfacePersists) {
-    if (conversationId && pending.conversationId !== conversationId) continue;
+    if (conversationId && pending.conversationId !== conversationId) {
+      continue;
+    }
     clearTimeout(pending.timer);
     pendingSurfacePersists.delete(surfaceId);
     persistSurfaceData(pending.conversationId, surfaceId, pending.data);
@@ -262,7 +262,9 @@ export function markSurfaceCompleted(
   if (ctx.messages) {
     for (let i = ctx.messages.length - 1; i >= 0; i--) {
       const msg = ctx.messages[i];
-      if (!Array.isArray(msg.content)) continue;
+      if (!Array.isArray(msg.content)) {
+        continue;
+      }
       for (const block of msg.content) {
         const b = block as Record<string, unknown>;
         if (b.type === "ui_surface" && b.surfaceId === surfaceId) {
@@ -278,16 +280,7 @@ export function markSurfaceCompleted(
   try {
     const rows = getMessages(ctx.conversationId);
     for (let r = rows.length - 1; r >= 0; r--) {
-      let parsed: unknown[];
-      try {
-        const result = JSON.parse(rows[r].content);
-        if (!Array.isArray(result)) continue;
-        parsed = result;
-      } catch {
-        // Some rows store plain text content (e.g. notification seeding) —
-        // skip them and keep scanning.
-        continue;
-      }
+      const parsed: unknown[] = rows[r].content;
       let found = false;
       for (const pb of parsed) {
         const rb = pb as Record<string, unknown>;
@@ -350,7 +343,9 @@ export function removeSurfaceBlock(
   if (ctx.messages) {
     for (let i = ctx.messages.length - 1; i >= 0; i--) {
       const msg = ctx.messages[i];
-      if (!Array.isArray(msg.content)) continue;
+      if (!Array.isArray(msg.content)) {
+        continue;
+      }
       const idx = msg.content.findIndex((block) => {
         const b = block as Record<string, unknown>;
         return b.type === "ui_surface" && b.surfaceId === surfaceId;
@@ -365,14 +360,7 @@ export function removeSurfaceBlock(
   try {
     const rows = getMessages(ctx.conversationId);
     for (let r = rows.length - 1; r >= 0; r--) {
-      let parsed: unknown[];
-      try {
-        const result = JSON.parse(rows[r].content);
-        if (!Array.isArray(result)) continue;
-        parsed = result;
-      } catch {
-        continue;
-      }
+      const parsed: unknown[] = rows[r].content;
       const idx = parsed.findIndex((pb) => {
         const rb = pb as Record<string, unknown>;
         return rb.type === "ui_surface" && rb.surfaceId === surfaceId;
@@ -410,7 +398,9 @@ const TASK_PROGRESS_STEP_STATUSES = new Set([
 function normalizeTaskProgressSteps(
   value: unknown,
 ): Array<Record<string, unknown>> {
-  if (!Array.isArray(value)) return [];
+  if (!Array.isArray(value)) {
+    return [];
+  }
   return value
     .filter((step): step is Record<string, unknown> => isPlainObject(step))
     .map((step) => {
@@ -568,8 +558,9 @@ function normalizeCardShowData(
     const candidates = allNonEmptyStrings(
       bodyAliasKeys.map((k) => {
         const dataVal = normalized[k];
-        if (typeof dataVal === "string" && dataVal.trim().length > 0)
+        if (typeof dataVal === "string" && dataVal.trim().length > 0) {
           return dataVal;
+        }
         return input[k];
       }),
     );
@@ -740,7 +731,9 @@ function normalizeChoiceShowData(
               : typeof option.label === "string"
                 ? option.label.trim()
                 : "";
-          if (!id || !title) return null;
+          if (!id || !title) {
+            return null;
+          }
           return {
             id,
             title,
@@ -852,20 +845,30 @@ function isSlackTaskProgressUiException(
   toolName: string,
   input: Record<string, unknown>,
 ): boolean {
-  if (ctx.channelCapabilities?.channel !== "slack") return false;
+  if (ctx.channelCapabilities?.channel !== "slack") {
+    return false;
+  }
   if (toolName === "ui_show") {
     const surfaceType = input.surface_type as SurfaceType;
-    if (surfaceType !== "card") return false;
+    if (surfaceType !== "card") {
+      return false;
+    }
     const rawData = isPlainObject(input.data) ? input.data : {};
     const data = normalizeCardShowData(input, rawData);
     return isTaskProgressCardData(data);
   }
   if (toolName === "ui_update") {
     const surfaceId = input.surface_id;
-    if (typeof surfaceId !== "string") return false;
+    if (typeof surfaceId !== "string") {
+      return false;
+    }
     const stored = ctx.surfaceState.get(surfaceId);
-    if (!stored || stored.surfaceType !== "card") return false;
-    if (!isTaskProgressCardData(stored.data)) return false;
+    if (!stored || stored.surfaceType !== "card") {
+      return false;
+    }
+    if (!isTaskProgressCardData(stored.data)) {
+      return false;
+    }
     const rawPatch = isPlainObject(input.data) ? input.data : {};
     const patch = normalizeTaskProgressCardPatch(
       stored.data as CardSurfaceData,
@@ -1062,7 +1065,9 @@ const STANDALONE_TOMBSTONE_TTL_MS = 30_000; // 30 seconds
 export function canShowInteractiveUi(
   ctx: Pick<SurfaceConversationContext, "hasNoClient" | "channelCapabilities">,
 ): boolean {
-  if (ctx.hasNoClient) return false;
+  if (ctx.hasNoClient) {
+    return false;
+  }
   if (ctx.channelCapabilities && !ctx.channelCapabilities.supportsDynamicUi) {
     return false;
   }
@@ -1285,7 +1290,9 @@ export function cleanupStandaloneSurface(
     // Clear any existing tombstone timer for this surfaceId (idempotency).
     const existingTimer =
       ctx.recentlyCompletedStandaloneSurfaces.get(surfaceId);
-    if (existingTimer) clearTimeout(existingTimer);
+    if (existingTimer) {
+      clearTimeout(existingTimer);
+    }
 
     const tombstoneTimer = setTimeout(() => {
       ctx.recentlyCompletedStandaloneSurfaces?.delete(surfaceId);
@@ -1568,9 +1575,13 @@ export function handleSurfaceUndo(
 
     // Update ALL surfaces that share this appId (not just the requesting one)
     for (const [sid, s] of ctx.surfaceState.entries()) {
-      if (s.surfaceType !== "dynamic_page") continue;
+      if (s.surfaceType !== "dynamic_page") {
+        continue;
+      }
       const sData = s.data as DynamicPageSurfaceData;
-      if (sData.appId !== data.appId) continue;
+      if (sData.appId !== data.appId) {
+        continue;
+      }
       const revertedData: DynamicPageSurfaceData = {
         ...sData,
         html: previousHtml,
@@ -1587,10 +1598,16 @@ export function handleSurfaceUndo(
     // Sync sibling undo stacks: pop the top entry if it matches the HTML we
     // just reverted to, preventing phantom no-op undo steps on siblings.
     for (const [sid, s] of ctx.surfaceState.entries()) {
-      if (sid === surfaceId) continue;
-      if (s.surfaceType !== "dynamic_page") continue;
+      if (sid === surfaceId) {
+        continue;
+      }
+      if (s.surfaceType !== "dynamic_page") {
+        continue;
+      }
       const sData = s.data as DynamicPageSurfaceData;
-      if (sData.appId !== data.appId) continue;
+      if (sData.appId !== data.appId) {
+        continue;
+      }
 
       const siblingStack = ctx.surfaceUndoStacks.get(sid);
       if (siblingStack && siblingStack.length > 0) {
@@ -1634,11 +1651,17 @@ export function describeTableRow(
   row: TableRow,
   columns: TableColumn[],
 ): string {
-  if (columns.length === 0) return row.id;
+  if (columns.length === 0) {
+    return row.id;
+  }
   const firstColId = columns[0].id;
   const cell = row.cells[firstColId];
-  if (cell == null) return row.id;
-  if (typeof cell === "string") return cell;
+  if (cell == null) {
+    return row.id;
+  }
+  if (typeof cell === "string") {
+    return cell;
+  }
   return cell.text;
 }
 
@@ -1646,7 +1669,9 @@ const MAX_DESELECTION_ITEMS = 20;
 
 /** Format a list of deselected item labels as a bullet list, capped at MAX_DESELECTION_ITEMS. */
 export function formatDeselectionList(labels: string[]): string {
-  if (labels.length === 0) return "";
+  if (labels.length === 0) {
+    return "";
+  }
   const shown = labels.slice(0, MAX_DESELECTION_ITEMS);
   const lines = shown.map((l) => `- ${l}`);
   if (labels.length > MAX_DESELECTION_ITEMS) {
@@ -1664,19 +1689,25 @@ export function buildDeselectionDescription(
   surfaceState: { surfaceType: SurfaceType; data: SurfaceData } | undefined,
   selectedIds: string[],
 ): string {
-  if (!surfaceState) return "";
+  if (!surfaceState) {
+    return "";
+  }
   const selectedSet = new Set(selectedIds);
 
   if (surfaceType === "table" && surfaceState.surfaceType === "table") {
     const tableData = surfaceState.data as TableSurfaceData;
     const deselectedLabels: string[] = [];
     for (const row of tableData.rows) {
-      if (row.selectable === false) continue;
+      if (row.selectable === false) {
+        continue;
+      }
       if (!selectedSet.has(row.id)) {
         deselectedLabels.push(describeTableRow(row, tableData.columns));
       }
     }
-    if (deselectedLabels.length === 0) return "";
+    if (deselectedLabels.length === 0) {
+      return "";
+    }
     return `\n\nDeselected items (user chose NOT to include):\n${formatDeselectionList(
       deselectedLabels,
     )}`;
@@ -1690,7 +1721,9 @@ export function buildDeselectionDescription(
         deselectedLabels.push(item.title);
       }
     }
-    if (deselectedLabels.length === 0) return "";
+    if (deselectedLabels.length === 0) {
+      return "";
+    }
     return `\n\nDeselected items (user chose NOT to include):\n${formatDeselectionList(
       deselectedLabels,
     )}`;
@@ -1710,7 +1743,9 @@ const SURFACE_COMPLETION_SUMMARY_FIELD = "_completionSummary";
 function getRequestedSurfaceCompletionSummary(
   data?: Record<string, unknown>,
 ): string | null {
-  if (data?.[SURFACE_COMPLETE_FLAG] !== true) return null;
+  if (data?.[SURFACE_COMPLETE_FLAG] !== true) {
+    return null;
+  }
   const summary =
     typeof data[SURFACE_COMPLETION_SUMMARY_FIELD] === "string"
       ? data[SURFACE_COMPLETION_SUMMARY_FIELD].trim()
@@ -1731,7 +1766,9 @@ function recordActivationMoment(
   moment: ActivationMomentParam,
 ): void {
   try {
-    if (!isActivationSession(ctx.conversationId)) return;
+    if (!isActivationSession(ctx.conversationId)) {
+      return;
+    }
     recordActivationEvent({
       stepName: activationStepNameForMomentParam(moment),
       sessionId: ctx.conversationId,
@@ -1765,7 +1802,9 @@ function maybeEmitActivationMoment(
 ): void {
   const stored = ctx.surfaceState.get(surfaceId);
   const moment = stored?.activationMoment;
-  if (!moment) return;
+  if (!moment) {
+    return;
+  }
   // Clear the tag first so this can fire at most once per surface even if the
   // commit path is re-entered.
   stored.activationMoment = undefined;
@@ -1786,6 +1825,18 @@ function completeSurfaceFromAction(
   markSurfaceCompleted(ctx, surfaceId, summary);
 }
 
+// One-shot interactive surfaces auto-complete once their action message is
+// accepted (they never accept further actions). Shared by the text-dispatch
+// path and the voice-resume branch in handleSurfaceAction.
+const ONE_SHOT_SURFACE_TYPES = [
+  "choice",
+  "oauth_connect",
+  "form",
+  "confirmation",
+  "file_upload",
+  "task_preferences",
+];
+
 export async function handleSurfaceAction(
   ctx: SurfaceConversationContext,
   surfaceId: string,
@@ -2330,6 +2381,55 @@ export async function handleSurfaceAction(
     "Surface action follow-up: preparing to send message to model",
   );
 
+  // A live-voice session owns this conversation: resume the follow-up as a
+  // SPOKEN turn on that session instead of an unspoken text run that never
+  // reaches TTS (JARVIS-1287). OAuth connect carries no attachments; if any are
+  // present, fall through to the text path (voice resume has no attachment
+  // pipeline). This branch mirrors the one-shot completion, accumulated-state
+  // cleanup, prompt echo, and pending-action delete of the text path below.
+  const voiceResumeHandler = getVoiceResumeHandler(ctx.conversationId);
+  if (voiceResumeHandler && pendingAttachments.length === 0) {
+    maybeEmitActivationMoment(ctx, surfaceId);
+    const requestedCompletionSummary =
+      getRequestedSurfaceCompletionSummary(mergedData);
+    if (
+      requestedCompletionSummary ||
+      ONE_SHOT_SURFACE_TYPES.includes(pending.surfaceType)
+    ) {
+      const completionSummary = requestedCompletionSummary ?? summary;
+      broadcastMessage({
+        type: "ui_surface_complete",
+        conversationId: ctx.conversationId,
+        surfaceId,
+        summary: completionSummary,
+        submittedData: mergedDataForText,
+      });
+      markSurfaceCompleted(ctx, surfaceId, completionSummary);
+    }
+    if (accumulatedState && Object.keys(accumulatedState).length > 0) {
+      ctx.accumulatedSurfaceState.delete(surfaceId);
+    }
+    // The resumed turn's own turn-start (startVoiceTurn) broadcasts the
+    // user_message_echo for the resume content, so this path does not pre-echo
+    // — otherwise a relayed-prompt click would render two user bubbles.
+    if (!retainPending) {
+      ctx.pendingSurfaceActions.delete(surfaceId);
+    }
+    // The voice-resume turn creates its own turn/events, so the tracking id
+    // minted for the (unused) text dispatch would otherwise linger.
+    ctx.surfaceActionRequestIds.delete(requestId);
+    log.info(
+      { surfaceId, actionId, conversationId: ctx.conversationId },
+      "Surface action follow-up: resuming as spoken live-voice turn",
+    );
+    voiceResumeHandler.resumeWithText(content, {
+      activeSurfaceId: surfaceId,
+      ...(displayContent ? { displayContent } : {}),
+      ...(sourceActorPrincipalId ? { sourceActorPrincipalId } : {}),
+    });
+    return;
+  }
+
   const result = ctx.enqueueMessage({
     content,
     attachments: pendingAttachments,
@@ -2356,14 +2456,6 @@ export async function handleSurfaceAction(
   // One-shot interactive surfaces — auto-complete now that the message has
   // been accepted. Deferred until after rejection check so the surface stays
   // active and retryable if the queue was full.
-  const ONE_SHOT_SURFACE_TYPES = [
-    "choice",
-    "oauth_connect",
-    "form",
-    "confirmation",
-    "file_upload",
-    "task_preferences",
-  ];
   if (
     requestedCompletionSummary ||
     ONE_SHOT_SURFACE_TYPES.includes(pending.surfaceType)
@@ -2457,13 +2549,19 @@ export function refreshSurfacesForApp(
   opts?: { fileChange?: boolean; status?: string },
 ): boolean {
   const app = getApp(appId);
-  if (!app) return false;
+  if (!app) {
+    return false;
+  }
 
   let refreshed = false;
   for (const [surfaceId, stored] of ctx.surfaceState.entries()) {
-    if (stored.surfaceType !== "dynamic_page") continue;
+    if (stored.surfaceType !== "dynamic_page") {
+      continue;
+    }
     const data = stored.data as DynamicPageSurfaceData;
-    if (data.appId !== appId) continue;
+    if (data.appId !== appId) {
+      continue;
+    }
 
     // Push current HTML onto the undo stack before overwriting
     pushUndoState(ctx.surfaceUndoStacks, surfaceId, data.html);
@@ -2562,9 +2660,12 @@ export function buildCompletionSummary(
   if (surfaceType === "choice" && data) {
     const choiceTitle =
       typeof data.choiceTitle === "string" ? data.choiceTitle : undefined;
-    if (choiceTitle) return `User chose: "${choiceTitle}"`;
-    if (selectedTitles.length === 1)
+    if (choiceTitle) {
+      return `User chose: "${choiceTitle}"`;
+    }
+    if (selectedTitles.length === 1) {
       return `User chose: "${selectedTitles[0]}"`;
+    }
     if (selectedTitles.length > 1) {
       return `User chose ${selectedTitles.length} options: ${selectedTitles
         .map((title) => `"${title}"`)
@@ -2603,17 +2704,22 @@ export function buildCompletionSummary(
   if (surfaceType === "list" && data) {
     const selectedIds = data.selectedIds as string[] | undefined;
     const actionSuffix = actionId ? ` (action: ${actionId})` : "";
-    if (selectedIds?.length === 1)
+    if (selectedIds?.length === 1) {
       return `Selected: ${selectedIds[0]}${actionSuffix}`;
-    if (selectedIds?.length)
+    }
+    if (selectedIds?.length) {
       return `Selected ${selectedIds.length} items${actionSuffix}`;
+    }
   }
   if (surfaceType === "table" && data) {
     const selectedIds = data.selectedIds as string[] | undefined;
     const actionSuffix = actionId ? ` (action: ${actionId})` : "";
-    if (selectedIds?.length === 1) return `Selected 1 row${actionSuffix}`;
-    if (selectedIds?.length)
+    if (selectedIds?.length === 1) {
+      return `Selected 1 row${actionSuffix}`;
+    }
+    if (selectedIds?.length) {
       return `Selected ${selectedIds.length} rows${actionSuffix}`;
+    }
   }
   return actionId.charAt(0).toUpperCase() + actionId.slice(1);
 }
@@ -2660,14 +2766,21 @@ function buildUserFacingLabel(
     }
     return `Selected: ${actionId}`;
   }
-  if (surfaceType === "form") return "Submitted";
+  if (surfaceType === "form") {
+    return "Submitted";
+  }
   if (surfaceType === "choice") {
     const choiceTitle =
       typeof data?.choiceTitle === "string" ? data.choiceTitle : undefined;
-    if (choiceTitle) return choiceTitle;
-    if (selectedTitles.length === 1) return selectedTitles[0];
-    if (selectedTitles.length > 1)
+    if (choiceTitle) {
+      return choiceTitle;
+    }
+    if (selectedTitles.length === 1) {
+      return selectedTitles[0];
+    }
+    if (selectedTitles.length > 1) {
       return `Selected ${selectedTitles.length} options`;
+    }
     return "Selected";
   }
   if (surfaceType === "oauth_connect") {
@@ -2785,7 +2898,9 @@ export async function surfaceProxyResolver(
         targetClientId,
         op: "host_cu",
       });
-      if (rejection) return rejection;
+      if (rejection) {
+        return rejection;
+      }
     }
 
     // Untargeted CU must resolve to exactly one same-user capable client
@@ -2894,7 +3009,9 @@ export async function surfaceProxyResolver(
         targetClientId,
         op: "host_app_control",
       });
-      if (rejection) return rejection;
+      if (rejection) {
+        return rejection;
+      }
     }
 
     if (targetClientId == null) {
@@ -3344,7 +3461,9 @@ export async function surfaceProxyResolver(
       const turnIdx = ctx.currentTurnSurfaces.findIndex(
         (s) => s.surfaceId === surfaceId,
       );
-      if (turnIdx !== -1) ctx.currentTurnSurfaces.splice(turnIdx, 1);
+      if (turnIdx !== -1) {
+        ctx.currentTurnSurfaces.splice(turnIdx, 1);
+      }
       removeSurfaceBlock(ctx, surfaceId);
     }
     ctx.pendingSurfaceActions.delete(surfaceId);
@@ -3393,22 +3512,20 @@ export async function surfaceProxyResolver(
     const storedPreview = getAppPreview(app.id);
     const { dirName } = resolveAppDir(app.id);
 
-    // For multifile TSX apps, auto-compile if dist is missing, then
-    // resolve HTML from compiled dist/index.html with inlined assets.
-    if (isMultifileApp(app)) {
-      const { existsSync } = await import("node:fs");
-      const { join } = await import("node:path");
-      const appDir = getAppDirPath(app.id);
-      const distIndex = join(appDir, "dist", "index.html");
-      if (!existsSync(distIndex)) {
-        const { compileApp } = await import("../bundler/app-compiler.js");
-        const result = await compileApp(appDir);
-        if (!result.ok) {
-          log.warn(
-            { appId, errors: result.errors },
-            "Auto-compile failed on app_open",
-          );
-        }
+    // Auto-compile if dist is missing, then resolve HTML from compiled
+    // dist/index.html with inlined assets.
+    const { existsSync } = await import("node:fs");
+    const { join } = await import("node:path");
+    const appDir = getAppDirPath(app.id);
+    const distIndex = join(appDir, "dist", "index.html");
+    if (!existsSync(distIndex)) {
+      const { compileApp } = await import("../bundler/app-compiler.js");
+      const result = await compileApp(appDir);
+      if (!result.ok) {
+        log.warn(
+          { appId, errors: result.errors },
+          "Auto-compile failed on app_open",
+        );
       }
     }
     const html = resolveEffectiveAppHtml(app);