@vellumai/assistant 0.8.2 → 0.8.4

package/src/runtime/routes/conversation-routes.ts

@@ -1,14 +1,7 @@
 /**
  * Route handlers for conversation messages and suggestions.
  */
-import {
-  existsSync,
-  readdirSync,
-  readFileSync,
-  statSync,
-  writeFileSync,
-} from "node:fs";
-import { join, relative } from "node:path";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
 
 import { z } from "zod";
 
@@ -31,6 +24,7 @@ import { createApprovalConversationGenerator } from "../../daemon/approval-gener
 import type { Conversation } from "../../daemon/conversation.js";
 import {
   buildModelInfoEvent,
+  formatCleanResult,
   formatCompactResult,
   isModelSlashCommand,
 } from "../../daemon/conversation-process.js";
@@ -41,6 +35,7 @@ import {
 import { getOrCreateConversation as getOrCreateConversationInstance } from "../../daemon/conversation-store.js";
 import { canonicalizeTimeZone } from "../../daemon/date-context.js";
 import {
+  buildScanFirstMessage,
   getCannedFirstGreeting,
   isWakeUpGreeting,
 } from "../../daemon/first-greeting.js";
@@ -51,6 +46,7 @@ import {
   preactivateHostProxySkills,
   shouldAttachHostProxyForCapability,
 } from "../../daemon/host-proxy-preactivation.js";
+import { getAssistantName } from "../../daemon/identity-helpers.js";
 import type { ServerMessage } from "../../daemon/message-protocol.js";
 import type {
   HostProxyTransportMetadata,
@@ -75,7 +71,7 @@ import {
 } from "../../memory/canonical-guardian-store.js";
 import {
   addMessage,
-  getLastAssistantTimestampBefore,
+  getConversation,
   getMessages,
   getMessagesPaginated,
   hasMessages,
@@ -101,11 +97,9 @@ import { writeOnboardingSection } from "../../prompts/persona-resolver.js";
 import { getConfiguredProvider } from "../../providers/provider-send-message.js";
 import type { Provider } from "../../providers/types.js";
 import { checkIngressForSecrets } from "../../security/secret-ingress.js";
+import { getSubagentManager } from "../../subagent/index.js";
 import { getLogger } from "../../util/logger.js";
-import {
-  getInterfacesDir,
-  getWorkspacePromptPath,
-} from "../../util/platform.js";
+import { getWorkspacePromptPath } from "../../util/platform.js";
 import { silentlyWithLog } from "../../util/silently.js";
 import { assistantEventHub, broadcastMessage } from "../assistant-event-hub.js";
 import { DAEMON_INTERNAL_ASSISTANT_ID } from "../assistant-scope.js";
@@ -135,6 +129,7 @@ const log = getLogger("conversation-routes");
 
 /** Matches the `<no_response/>` sentinel used by channel delivery suppression. */
 const NO_RESPONSE_INLINE_RE = /<no_response\s*\/?>/g;
+const ATTACHMENT_ENTRY_RE = /^attachment:(\d+)$/;
 
 const SUGGESTION_CACHE_MAX = 100;
 const VALID_RISK_THRESHOLDS = ["none", "low", "medium", "high"] as const;
@@ -147,31 +142,67 @@ function isValidRiskThreshold(value: unknown): value is RiskThreshold {
   );
 }
 
+/**
+ * True when a message's persisted metadata explicitly flags it as hidden.
+ * Used to suppress internal scaffolding messages from UI history while
+ * leaving them in the LLM-side context.
+ */
+function isHiddenMessage(metadata: string | null): boolean {
+  if (!metadata) return false;
+  try {
+    const meta = JSON.parse(metadata) as { hidden?: unknown };
+    return meta?.hidden === true;
+  } catch {
+    return false;
+  }
+}
+
 function buildSlackHistoryMessage(
   slackMeta: SlackMessageMetadata | null,
+  opts?: { role?: string; assistantDisplayName?: string },
 ): RuntimeMessagePayload["slackMessage"] | undefined {
   if (!slackMeta) return undefined;
 
   const slackConfig = getConfig().slack;
+  const replyThreadTs =
+    slackMeta.threadTs && slackMeta.threadTs !== slackMeta.channelTs
+      ? slackMeta.threadTs
+      : undefined;
   const messageLink = buildSlackMessageDeepLinks({
     teamId: slackConfig?.teamId,
     teamUrl: slackConfig?.teamUrl,
     channelId: slackMeta.channelId,
     messageTs: slackMeta.channelTs,
+    ...(replyThreadTs ? { threadTs: replyThreadTs } : {}),
   });
-  const threadLink = slackMeta.threadTs
+  const threadLink = replyThreadTs
     ? buildSlackMessageDeepLinks({
         teamId: slackConfig?.teamId,
         teamUrl: slackConfig?.teamUrl,
         channelId: slackMeta.channelId,
-        messageTs: slackMeta.threadTs,
+        messageTs: replyThreadTs,
       })
     : undefined;
+  const assistantDisplayName =
+    opts?.role === "assistant" ? opts.assistantDisplayName : undefined;
+  const senderDisplayName =
+    slackMeta.displayName?.trim() || assistantDisplayName;
 
   return {
     channelId: slackMeta.channelId,
+    ...(slackMeta.channelName ? { channelName: slackMeta.channelName } : {}),
     channelTs: slackMeta.channelTs,
     ...(slackMeta.threadTs ? { threadTs: slackMeta.threadTs } : {}),
+    ...(senderDisplayName || slackMeta.actorExternalUserId
+      ? {
+          sender: {
+            ...(senderDisplayName ? { displayName: senderDisplayName } : {}),
+            ...(slackMeta.actorExternalUserId
+              ? { externalUserId: slackMeta.actorExternalUserId }
+              : {}),
+          },
+        }
+      : {}),
     ...(messageLink ? { messageLink } : {}),
     ...(threadLink ? { threadLink } : {}),
   };
@@ -389,32 +420,9 @@ async function tryConsumeCanonicalGuardianReply(params: {
   return { consumed: true, messageId };
 }
 
-function getInterfaceFilesWithMtimes(
-  interfacesDir: string | null,
-): Array<{ path: string; mtimeMs: number }> {
-  if (!interfacesDir || !existsSync(interfacesDir)) return [];
-  const results: Array<{ path: string; mtimeMs: number }> = [];
-  const scan = (dir: string): void => {
-    for (const entry of readdirSync(dir, { withFileTypes: true })) {
-      const fullPath = join(dir, entry.name);
-      if (entry.isDirectory()) {
-        scan(fullPath);
-      } else {
-        results.push({
-          path: relative(interfacesDir, fullPath),
-          mtimeMs: statSync(fullPath).mtimeMs,
-        });
-      }
-    }
-  };
-  scan(interfacesDir);
-  return results;
-}
-
-export function handleListMessages(
-  { queryParams }: RouteHandlerArgs,
-  interfacesDir: string | null,
-): Record<string, unknown> {
+export function handleListMessages({
+  queryParams,
+}: RouteHandlerArgs): Record<string, unknown> {
   const conversationId = queryParams?.conversationId;
   const conversationKey = queryParams?.conversationKey;
 
@@ -422,8 +430,20 @@ export function handleListMessages(
   if (conversationId) {
     resolvedConversationId = conversationId;
   } else if (conversationKey) {
+    // Dual lookup, key-first: prefer the `conversation_keys` table — the
+    // canonical channel/external → internal-id mapping — so legacy or
+    // externally-sourced keys keep their explicit mapping precedence and
+    // never collide with an unrelated `conversations.id`. Fall back to a
+    // direct id lookup only when no mapping exists, which covers
+    // background/scheduled conversations bootstrapped without a
+    // `conversation_keys` row (web clients use the conversation list's
+    // `id` as `conversationKey` for those).
     const mapping = getConversationByKey(conversationKey);
-    resolvedConversationId = mapping?.conversationId;
+    if (mapping) {
+      resolvedConversationId = mapping.conversationId;
+    } else if (getConversation(conversationKey)) {
+      resolvedConversationId = conversationKey;
+    }
   } else {
     throw new BadRequestError(
       "conversationKey or conversationId query parameter is required",
@@ -479,16 +499,30 @@ export function handleListMessages(
   let rawMessages: MessageRow[];
   let hasMore = false;
 
+  // Drop messages flagged as hidden in metadata (e.g. internal scaffolding
+  // like retrospective instructions). The LLM-side history loader
+  // (`getMessages` in memory/conversation-crud.ts) intentionally does not
+  // filter — hidden messages remain in agent context but are suppressed from
+  // the UI list. Filtering is pushed into the paginated query so `hasMore`
+  // and the cursor reflect visible rows; otherwise a fully-hidden page would
+  // return `hasMore: true` with no cursor and stall the web client.
+  // Hidden tool_use/tool_result pairs must be hidden together — if a hidden
+  // assistant message has tool_use blocks but its matching user tool_result
+  // is left visible, the result will render as a standalone orphan because
+  // `mergeToolResultsIntoAssistantMessages` has nothing to merge it into.
+  const visibleFilter = (m: MessageRow) => !isHiddenMessage(m.metadata);
+
   if (isPaginated) {
     const result = getMessagesPaginated(
       resolvedConversationId,
       limit,
       beforeTimestamp,
+      visibleFilter,
     );
     rawMessages = result.messages;
     hasMore = result.hasMore;
   } else {
-    rawMessages = getMessages(resolvedConversationId);
+    rawMessages = getMessages(resolvedConversationId).filter(visibleFilter);
   }
 
   // During streaming, tool_use (assistant) and tool_result (user) events are
@@ -507,6 +541,7 @@ export function handleListMessages(
   // (consecutive tool refs grouped together).
   const { messages: consolidatedMessages, mergedIdMap } =
     mergeConsecutiveAssistantMessages(mergedMessages);
+  const assistantSlackDisplayName = getAssistantName()?.trim() || undefined;
 
   // Parse content blocks and extract text + tool calls
   const parsed = consolidatedMessages.map((msg) => {
@@ -558,6 +593,10 @@ export function handleListMessages(
     }
     const slackMessage = buildSlackHistoryMessage(
       readSlackMetadataFromMessageMetadata(msg.metadata),
+      {
+        role: msg.role,
+        assistantDisplayName: assistantSlackDisplayName,
+      },
     );
 
     // Strip <no_response/> markers from assistant messages so web/API
@@ -598,6 +637,7 @@ export function handleListMessages(
         textSegments: filteredSegments,
         contentOrder: filteredContentOrder,
         surfaces: rendered.surfaces,
+        attachmentRefs: rendered.attachments,
         slackMessage,
         ...(rendered.thinkingSegments.length > 0
           ? { thinkingSegments: rendered.thinkingSegments }
@@ -617,6 +657,7 @@ export function handleListMessages(
       textSegments: rendered.textSegments,
       contentOrder: rendered.contentOrder,
       surfaces: rendered.surfaces,
+      attachmentRefs: rendered.attachments,
       slackMessage,
       ...(rendered.thinkingSegments.length > 0
         ? { thinkingSegments: rendered.thinkingSegments }
@@ -626,15 +667,6 @@ export function handleListMessages(
     };
   });
 
-  const interfaceFiles = getInterfaceFilesWithMtimes(interfacesDir);
-
-  let prevAssistantTimestamp = 0;
-  if (isPaginated && rawMessages.length > 0) {
-    prevAssistantTimestamp = getLastAssistantTimestampBefore(
-      resolvedConversationId!,
-      rawMessages[0].createdAt,
-    );
-  }
   const messages: RuntimeMessagePayload[] = parsed.map((m) => {
     let msgAttachments: RuntimeAttachmentMetadata[] = [];
     if (m.id) {
@@ -681,19 +713,76 @@ export function handleListMessages(
       }
     }
 
-    let interfaces: string[] | undefined;
-    if (m.role === "assistant") {
-      const msgTimestamp = new Date(m.timestamp).getTime();
-      const dirtied = interfaceFiles
-        .filter(
-          (f) =>
-            f.mtimeMs > prevAssistantTimestamp && f.mtimeMs <= msgTimestamp,
-        )
-        .map((f) => f.path);
-      if (dirtied.length > 0) {
-        interfaces = dirtied;
+    // Align msgAttachments order with the file-block order captured by
+    // renderHistoryContent. When a file block was persisted with
+    // `_attachmentId`, we can join on that id to position the chip inline
+    // (the `attachment:N` entries in contentOrder index into msgAttachments).
+    // DB rows without a matching ref go to the tail as orphan chips;
+    // unmatched refs drop their contentOrder entry and trigger a remap.
+    let alignedContentOrder = m.contentOrder;
+    if (
+      m.attachmentRefs.length > 0 &&
+      msgAttachments.length > 0 &&
+      m.contentOrder.length > 0
+    ) {
+      const byId = new Map<string, number>();
+      msgAttachments.forEach((att, idx) => {
+        if (att.id) byId.set(att.id, idx);
+      });
+      const consumed = new Set<number>();
+      const orderedRowIdx: Array<number | null> = m.attachmentRefs.map(
+        (ref) => {
+          if (!ref.attachmentId) return null;
+          const idx = byId.get(ref.attachmentId);
+          if (idx === undefined || consumed.has(idx)) return null;
+          consumed.add(idx);
+          return idx;
+        },
+      );
+      const matchedRows = orderedRowIdx.filter(
+        (idx): idx is number => idx !== null,
+      );
+      if (matchedRows.length > 0) {
+        const orphanRows: number[] = [];
+        for (let i = 0; i < msgAttachments.length; i++) {
+          if (!consumed.has(i)) orphanRows.push(i);
+        }
+        msgAttachments = [
+          ...matchedRows.map((i) => msgAttachments[i]),
+          ...orphanRows.map((i) => msgAttachments[i]),
+        ];
+        const refToNewIdx = new Map<number, number>();
+        let nextIdx = 0;
+        orderedRowIdx.forEach((rowIdx, refIdx) => {
+          if (rowIdx !== null) {
+            refToNewIdx.set(refIdx, nextIdx);
+            nextIdx++;
+          }
+        });
+        alignedContentOrder = m.contentOrder
+          .map((entry) => {
+            const match = entry.match(ATTACHMENT_ENTRY_RE);
+            if (!match) return entry;
+            const remapped = refToNewIdx.get(Number(match[1]));
+            return remapped !== undefined
+              ? `attachment:${remapped}`
+              : undefined;
+          })
+          .filter((e): e is string => e !== undefined);
+      } else {
+        // No refs carried an attachmentId we could match — strip any
+        // attachment:N entries so the client doesn't try to position
+        // attachments inline against a misaligned array.
+        alignedContentOrder = m.contentOrder.filter(
+          (entry) => !ATTACHMENT_ENTRY_RE.test(entry),
+        );
       }
-      prevAssistantTimestamp = msgTimestamp;
+    } else if (m.attachmentRefs.length > 0 && msgAttachments.length === 0) {
+      // Refs were captured but no DB rows came back — drop the
+      // contentOrder entries to avoid out-of-bounds renders.
+      alignedContentOrder = m.contentOrder.filter(
+        (entry) => !ATTACHMENT_ENTRY_RE.test(entry),
+      );
     }
 
     // Use sentAt (actual event time) for the display timestamp when
@@ -716,13 +805,14 @@ export function handleListMessages(
       timestamp: new Date(displayTimestamp).toISOString(),
       attachments: msgAttachments,
       ...(m.toolCalls.length > 0 ? { toolCalls: m.toolCalls } : {}),
-      ...(interfaces ? { interfaces } : {}),
       ...(m.surfaces.length > 0 ? { surfaces: m.surfaces } : {}),
       ...(m.textSegments.length > 0 ? { textSegments: m.textSegments } : {}),
       ...(m.thinkingSegments?.length
         ? { thinkingSegments: m.thinkingSegments }
         : {}),
-      ...(m.contentOrder.length > 0 ? { contentOrder: m.contentOrder } : {}),
+      ...(alignedContentOrder.length > 0
+        ? { contentOrder: alignedContentOrder }
+        : {}),
       ...(m.subagentNotification
         ? { subagentNotification: m.subagentNotification }
         : {}),
@@ -1091,6 +1181,10 @@ export function persistOnboardingArtifacts(onboarding: {
   tone: string;
   userName?: string;
   assistantName?: string;
+  priorAssistants?: string[];
+  cohort?: string;
+  websiteUrl?: string;
+  contentSourceUrl?: string;
 }): void {
   writeOnboardingSidecar(onboarding);
 
@@ -1168,6 +1262,10 @@ export async function handleSendMessage(
       assistantName?: string;
       googleConnected?: boolean;
       googleScopes?: string[];
+      priorAssistants?: string[];
+      cohort?: string;
+      websiteUrl?: string;
+      contentSourceUrl?: string;
     };
   };
 
@@ -1492,13 +1590,41 @@ export async function handleSendMessage(
   // that can service host_browser_request events; we restore that single
   // proxy explicitly below without relaxing `hasNoClient`.
   conversation.updateClient(broadcastMessage, !isInteractive);
+  if (isInteractive) {
+    getSubagentManager().updateParentSender(
+      mapping.conversationId,
+      broadcastMessage,
+    );
+  }
 
-  // ── Canned first-greeting fast path ──
-  // On a completely fresh workspace, skip LLM inference for the macOS
-  // wake-up greeting and return a pre-written response. When onboarding
-  // context is present the greeting is personalized using the selections;
-  // otherwise a generic greeting is served. Both paths are instant.
-  if (isWakeUpGreeting(trimmedContent, conversation.getMessages().length)) {
+  // ── URL scan path: rewrite first message for scan onboarding ──
+  // When onboarding provides a websiteUrl or contentSourceUrl and the
+  // first message is the macOS wake-up greeting, bypass the canned
+  // greeting and rewrite the user message to a scan instruction so real
+  // LLM inference runs against the URL.
+  const sanitizeUrl = (u?: string) =>
+    u?.trim().replace(/[\r\n\t]/g, "") || undefined;
+  const websiteUrl = sanitizeUrl(body.onboarding?.websiteUrl);
+  const contentSourceUrl = sanitizeUrl(body.onboarding?.contentSourceUrl);
+  const scanUrl = websiteUrl || contentSourceUrl;
+  const isWakeUp = isWakeUpGreeting(
+    trimmedContent,
+    conversation.getMessages().length,
+  );
+  const isScanPath = !!scanUrl && isWakeUp;
+
+  let effectiveContent: string | undefined;
+  if (isScanPath) {
+    const scanVariant = websiteUrl
+      ? ("website" as const)
+      : ("content-source" as const);
+    effectiveContent = buildScanFirstMessage(scanUrl, scanVariant);
+    // Fall through to normal inference path below
+  } else if (isWakeUp && body.onboarding?.cohort === "content-automation") {
+    effectiveContent = "I want to write articles that rank better in GEO";
+    // Fall through to normal inference path — the bootstrap template
+    // and geo-writing skill handle this message.
+  } else if (isWakeUp) {
     const cannedGreeting = getCannedFirstGreeting(body.onboarding ?? undefined);
 
     conversation.processing = true;
@@ -1562,6 +1688,7 @@ export async function handleSendMessage(
             tone: body.onboarding!.tone,
             googleConnected: body.onboarding!.googleConnected,
             googleScopes: body.onboarding!.googleScopes,
+            priorAssistants: body.onboarding!.priorAssistants,
           });
         } catch (err) {
           log.warn({ err }, "Failed to record onboarding telemetry event");
@@ -1616,12 +1743,18 @@ export async function handleSendMessage(
         tone: body.onboarding!.tone,
         googleConnected: body.onboarding!.googleConnected,
         googleScopes: body.onboarding!.googleScopes,
+        priorAssistants: body.onboarding!.priorAssistants,
       });
     } catch (err) {
       log.warn({ err }, "Failed to record onboarding telemetry event");
     }
   }
 
+  // When the scan path rewrote the first message, prefer the rewritten
+  // content for all downstream consumers (guardian reply, enqueue, agent
+  // loop) so they see the scan instruction rather than the wake-up greeting.
+  const contentAfterScan = effectiveContent ?? content ?? "";
+
   const attachments = hasAttachments
     ? smDeps.resolveAttachments(attachmentIds)
     : [];
@@ -1641,7 +1774,7 @@ export async function handleSendMessage(
       conversationId: mapping.conversationId,
       sourceChannel,
       sourceInterface,
-      content: content ?? "",
+      content: contentAfterScan,
       attachments,
       conversation,
       onEvent: broadcastMessage,
@@ -1675,7 +1808,7 @@ export async function handleSendMessage(
     // Queue the message so it's processed when the current turn completes
     const requestId = crypto.randomUUID();
     const enqueueResult = conversation.enqueueMessage(
-      content ?? "",
+      contentAfterScan,
       attachments,
       broadcastMessage,
       requestId,
@@ -1747,6 +1880,7 @@ export async function handleSendMessage(
       accepted: true,
       queued: true,
       conversationId: mapping.conversationId,
+      requestId,
     };
   }
 
@@ -1794,7 +1928,9 @@ export async function handleSendMessage(
   await conversation.ensureActorScopedHistory();
 
   // Resolve slash commands before persisting or running the agent loop.
-  const rawContent = content ?? "";
+  // `contentAfterScan` already carries the scan-rewritten content when
+  // applicable; reuse it here for consistency.
+  const rawContent = contentAfterScan;
   const slashContext = buildSlashContextForContent(rawContent, {
     conversationId: mapping.conversationId,
     messageCount: conversation.getMessages().length,
@@ -2005,6 +2141,82 @@ export async function handleSendMessage(
     };
   }
 
+  if (slashResult.kind === "clean") {
+    conversation.processing = true;
+    const provenance = provenanceFromTrustContext(conversation.trustContext);
+    const channelMeta = {
+      ...provenance,
+      userMessageChannel: sourceChannel,
+      assistantMessageChannel: sourceChannel,
+      userMessageInterface: sourceInterface,
+      assistantMessageInterface: sourceInterface,
+    };
+    const cleanMsg = createUserMessage(rawContent, attachments);
+    const persisted = await addMessage(
+      mapping.conversationId,
+      "user",
+      JSON.stringify(cleanMsg.content),
+      channelMeta,
+    );
+    conversation.getMessages().push(cleanMsg);
+
+    const conversationId = mapping.conversationId;
+
+    let assistantMessagePersisted = false;
+    try {
+      broadcastMessage({
+        type: "user_message_echo",
+        text: rawContent,
+        conversationId,
+        messageId: persisted.id,
+        clientMessageId,
+      });
+      publishConversationMessagesChanged(conversationId);
+
+      const result = await conversation.forceClean();
+      const responseText = formatCleanResult(result);
+
+      const assistantMsg = createAssistantMessage(responseText);
+      await addMessage(
+        conversationId,
+        "assistant",
+        JSON.stringify(assistantMsg.content),
+        channelMeta,
+      );
+      assistantMessagePersisted = true;
+      conversation.getMessages().push(assistantMsg);
+
+      broadcastMessage({
+        type: "assistant_text_delta",
+        text: responseText,
+        conversationId,
+      });
+      broadcastMessage({ type: "message_complete", conversationId });
+      publishConversationMessagesChanged(conversationId);
+    } catch (err) {
+      if (assistantMessagePersisted) {
+        publishConversationMessagesChanged(conversationId);
+      }
+      log.error({ err, conversationId }, "Clean command failed");
+      broadcastMessage({
+        type: "conversation_error",
+        conversationId,
+        code: "UNKNOWN",
+        userMessage: `Clean failed: ${err instanceof Error ? err.message : String(err)}`,
+        retryable: true,
+      });
+    } finally {
+      conversation.processing = false;
+      silentlyWithLog(conversation.drainQueue(), "clean-command queue drain");
+    }
+
+    return {
+      accepted: true,
+      messageId: persisted.id,
+      conversationId,
+    };
+  }
+
   const resolvedContent = slashResult.content;
 
   const requestId = crypto.randomUUID();
@@ -2362,7 +2574,7 @@ export const ROUTES: RouteDefinition[] = [
         .optional()
         .describe("ID of the oldest message in this page"),
     }),
-    handler: (args) => handleListMessages(args, getInterfacesDir()),
+    handler: (args) => handleListMessages(args),
   },
   {
     operationId: "messages_post",