@vellumai/assistant 0.5.1 → 0.5.3

package/src/daemon/conversation-lifecycle.ts

@@ -68,6 +68,39 @@ function filterMessagesForUntrustedActor(messages: MessageRow[]): MessageRow[] {
   });
 }
 
+/**
+ * Re-inject image source path annotations into message content blocks.
+ *
+ * When the desktop client attaches images from local files, the source paths
+ * are stored in `metadata.imageSourcePaths` (keyed by filename). The LLM-facing
+ * content omits these paths at persistence time, so we re-inject them when
+ * loading history from the DB. Only user messages are annotated.
+ */
+export function reinjectImageSourcePaths(
+  content: ContentBlock[],
+  role: string,
+  metadataJson: string | null,
+): ContentBlock[] {
+  if (role !== "user" || !metadataJson) return content;
+  try {
+    const meta = JSON.parse(metadataJson);
+    if (!meta.imageSourcePaths || typeof meta.imageSourcePaths !== "object") {
+      return content;
+    }
+    const paths = Object.values(meta.imageSourcePaths).filter(
+      (v): v is string => typeof v === "string",
+    );
+    if (paths.length === 0) return content;
+    const annotation = paths
+      .map((p) => `[Attached image source: ${p}]`)
+      .join("\n");
+    return [...content, { type: "text" as const, text: annotation }];
+  } catch {
+    // metadata parse failure — skip annotation, not critical
+    return content;
+  }
+}
+
 // ── Context Interfaces ───────────────────────────────────────────────
 
 export interface LoadFromDbContext {
@@ -78,7 +111,6 @@ export interface LoadFromDbContext {
   contextCompactedAt: number | null;
   trustContext?: { trustClass: TrustClass };
   loadedHistoryTrustClass?: TrustClass;
-  hasAttachments?: boolean;
 }
 
 export interface AbortContext {
@@ -91,8 +123,19 @@ export interface AbortContext {
   surfaceActionRequestIds: Set<string>;
   surfaceState: Map<
     string,
-    { surfaceType: SurfaceType; data: SurfaceData; title?: string }
+    {
+      surfaceType: SurfaceType;
+      data: SurfaceData;
+      title?: string;
+      actions?: Array<{
+        id: string;
+        label: string;
+        style?: string;
+        data?: Record<string, unknown>;
+      }>;
+    }
   >;
+  accumulatedSurfaceState: Map<string, Record<string, unknown>>;
   readonly queue: MessageQueue;
 }
 
@@ -151,6 +194,9 @@ export async function loadFromDb(ctx: LoadFromDbContext): Promise<void> {
         );
         content = [{ type: "text", text: m.content }];
       }
+
+      content = reinjectImageSourcePaths(content, role, m.metadata);
+
       return { role, content };
     });
 
@@ -182,20 +228,6 @@ export async function loadFromDb(ctx: LoadFromDbContext): Promise<void> {
 
   ctx.loadedHistoryTrustClass = trustClass;
 
-  // Scan ALL db messages (including compacted ones) for attachments so that
-  // asset tools remain available after context compaction.
-  if (
-    ctx.contextCompactedMessageCount > 0 &&
-    dbMessages.some(
-      (m) =>
-        m.role === "user" &&
-        (m.content.includes('"type":"image"') ||
-          m.content.includes('"type":"file"')),
-    )
-  ) {
-    ctx.hasAttachments = true;
-  }
-
   log.info(
     { conversationId: ctx.conversationId, count: ctx.messages.length },
     "Loaded messages from DB",
@@ -216,6 +248,7 @@ export function abortConversation(ctx: AbortContext): void {
     ctx.pendingSurfaceActions.clear();
     ctx.surfaceActionRequestIds.clear();
     ctx.surfaceState.clear();
+    ctx.accumulatedSurfaceState.clear();
     unregisterWatchNotifiers(ctx.conversationId);
     for (const queued of ctx.queue) {
       queued.onEvent({
@@ -247,6 +280,7 @@ export function disposeConversation(ctx: DisposeContext): void {
   ctx.pendingSurfaceActions.clear();
   ctx.surfaceActionRequestIds.clear();
   ctx.surfaceState.clear();
+  ctx.accumulatedSurfaceState.clear();
   ctx.lastSurfaceAction.clear();
   ctx.workspaceTopLevelContext = null;
 }

package/src/daemon/conversation-messaging.ts

@@ -7,6 +7,7 @@
 
 import { v4 as uuid } from "uuid";
 
+import { enrichMessageWithSourcePaths } from "../agent/attachments.js";
 import { createUserMessage } from "../agent/message-types.js";
 import type {
   TurnChannelContext,
@@ -14,17 +15,23 @@ import type {
 } from "../channels/types.js";
 import { parseChannelId, parseInterfaceId } from "../channels/types.js";
 import {
+  attachInlineAttachmentToMessage,
+  attachmentExists,
   AttachmentUploadError,
   linkAttachmentToMessage,
-  uploadAttachment,
   validateAttachmentUpload,
 } from "../memory/attachments-store.js";
 import {
   addMessage,
+  getConversation,
   provenanceFromTrustContext,
   setConversationOriginChannelIfUnset,
   setConversationOriginInterfaceIfUnset,
 } from "../memory/conversation-crud.js";
+import {
+  syncMessageToDisk,
+  updateMetaFile,
+} from "../memory/conversation-disk-view.js";
 import type { SecretPrompter } from "../permissions/secret-prompter.js";
 import type { Message } from "../providers/types.js";
 import { getLogger } from "../util/logger.js";
@@ -276,17 +283,20 @@ export async function persistUserMessage(
   ctx.processing = true;
   ctx.abortController = new AbortController();
 
-  const userMessage = createUserMessage(
-    content,
-    attachments.map((attachment) => ({
-      id: attachment.id,
-      filename: attachment.filename,
-      mimeType: attachment.mimeType,
-      data: attachment.data,
-      extractedText: attachment.extractedText,
-    })),
+  const attachmentInputs = attachments.map((attachment) => ({
+    id: attachment.id,
+    filename: attachment.filename,
+    mimeType: attachment.mimeType,
+    data: attachment.data,
+    extractedText: attachment.extractedText,
+    filePath: attachment.filePath,
+  }));
+  const cleanMessage = createUserMessage(content, attachmentInputs);
+  const llmMessage = enrichMessageWithSourcePaths(
+    cleanMessage,
+    attachmentInputs,
   );
-  ctx.messages.push(userMessage);
+  ctx.messages.push(llmMessage);
 
   try {
     const turnCtx =
@@ -294,6 +304,14 @@ export async function persistUserMessage(
     const turnIfCtx =
       extractTurnInterfaceContext(metadata) ?? ctx.getTurnInterfaceContext();
     const provenance = provenanceFromTrustContext(ctx.trustContext);
+    const imageSourcePaths: Record<string, string> = {};
+    for (let i = 0; i < attachments.length; i++) {
+      const a = attachments[i];
+      if (a.filePath && a.mimeType.toLowerCase().startsWith("image/")) {
+        imageSourcePaths[`${i}:${a.filename}`] = a.filePath;
+      }
+    }
+
     const mergedMetadata = {
       ...(metadata ?? {}),
       ...provenance,
@@ -309,6 +327,7 @@ export async function persistUserMessage(
             assistantMessageInterface: turnIfCtx.assistantMessageInterface,
           }
         : {}),
+      ...(Object.keys(imageSourcePaths).length > 0 ? { imageSourcePaths } : {}),
     };
 
     // When displayContent is provided (e.g. original text before recording
@@ -317,18 +336,9 @@ export async function persistUserMessage(
     // the stripped content.
     const contentToPersist = displayContent
       ? JSON.stringify(
-          createUserMessage(
-            displayContent,
-            attachments.map((a) => ({
-              id: a.id,
-              filename: a.filename,
-              mimeType: a.mimeType,
-              data: a.data,
-              extractedText: a.extractedText,
-            })),
-          ).content,
+          createUserMessage(displayContent, attachmentInputs).content,
         )
-      : JSON.stringify(userMessage.content);
+      : JSON.stringify(cleanMessage.content);
     const persistedUserMessage = await addMessage(
       ctx.conversationId,
       "user",
@@ -349,15 +359,33 @@ export async function persistUserMessage(
       );
     }
 
+    // Rewrite meta.json so the on-disk metadata reflects the origin channel
+    if (turnCtx || turnIfCtx) {
+      const convForMeta = getConversation(ctx.conversationId);
+      if (convForMeta) {
+        updateMetaFile(convForMeta);
+      }
+    }
+
     if (!persistedUserMessage.id) {
       throw new Error("Failed to persist user message");
     }
 
-    // Index user attachments in the attachments table so asset_search can find them.
+    // Index user attachments in the attachments table for later retrieval.
     for (let i = 0; i < attachments.length; i++) {
       const a = attachments[i];
-      if (!a.data) continue;
       try {
+        // If the attachment already exists in the store (e.g. file-backed
+        // attachments uploaded separately), link it directly without
+        // re-uploading. This handles the case where data is empty because
+        // the attachment content lives on disk.
+        if (a.id && attachmentExists(a.id)) {
+          linkAttachmentToMessage(persistedUserMessage.id, a.id, i);
+          continue;
+        }
+
+        if (!a.data) continue;
+
         const validation = validateAttachmentUpload(a.filename, a.mimeType);
         if (!validation.ok) {
           log.warn(
@@ -366,8 +394,14 @@ export async function persistUserMessage(
           );
           continue;
         }
-        const stored = uploadAttachment(a.filename, a.mimeType, a.data);
-        linkAttachmentToMessage(persistedUserMessage.id, stored.id, i);
+        attachInlineAttachmentToMessage(
+          persistedUserMessage.id,
+          i,
+          a.filename,
+          a.mimeType,
+          a.data,
+          { sourcePath: a.filePath },
+        );
       } catch (err) {
         if (err instanceof AttachmentUploadError) {
           log.warn(
@@ -383,6 +417,16 @@ export async function persistUserMessage(
       }
     }
 
+    // Sync the persisted user message (with attachments) to the disk view
+    const conv = getConversation(ctx.conversationId);
+    if (conv) {
+      syncMessageToDisk(
+        ctx.conversationId,
+        persistedUserMessage.id,
+        conv.createdAt,
+      );
+    }
+
     return persistedUserMessage.id;
   } catch (err) {
     ctx.messages.pop();

package/src/daemon/conversation-process.ts

@@ -6,6 +6,7 @@
  * used by conversation-history.ts.
  */
 
+import { enrichMessageWithSourcePaths } from "../agent/attachments.js";
 import {
   createAssistantMessage,
   createUserMessage,
@@ -25,18 +26,14 @@ import {
 } from "../memory/conversation-crud.js";
 import { extractPreferences } from "../notifications/preference-extractor.js";
 import { createPreference } from "../notifications/preferences-store.js";
-import { getConfiguredProviders } from "../providers/provider-availability.js";
 import type { Message } from "../providers/types.js";
 import { routeGuardianReply } from "../runtime/guardian-reply-router.js";
 import { getLogger } from "../util/logger.js";
 import type { MessageQueue } from "./conversation-queue-manager.js";
 import type { QueueDrainReason } from "./conversation-queue-manager.js";
 import type { TrustContext } from "./conversation-runtime-assembly.js";
-import {
-  isProviderShortcut,
-  resolveSlash,
-  type SlashContext,
-} from "./conversation-slash.js";
+import { resolveSlash, type SlashContext } from "./conversation-slash.js";
+import { getModelInfo } from "./handlers/config-model.js";
 import type {
   ServerMessage,
   UsageStats,
@@ -49,23 +46,12 @@ const log = getLogger("conversation-process");
 
 /** Build a model_info event with fresh config data. */
 export async function buildModelInfoEvent(): Promise<ServerMessage> {
-  const config = getConfig();
-  return {
-    type: "model_info",
-    model: config.services.inference.model,
-    provider: config.services.inference.provider,
-    configuredProviders: await getConfiguredProviders(),
-  };
+  return { type: "model_info", ...(await getModelInfo()) };
 }
 
-/** True when the trimmed content is a /model or /models slash command. */
+/** True when the trimmed content is the /models slash command. */
 export function isModelSlashCommand(content: string): boolean {
-  const trimmed = content.trim();
-  return (
-    trimmed === "/model" ||
-    trimmed === "/models" ||
-    trimmed.startsWith("/model ")
-  );
+  return content.trim() === "/models";
 }
 
 // ── Context Interface ────────────────────────────────────────────────
@@ -196,6 +182,7 @@ function buildSlashContext(
   conversation: ProcessConversationContext,
 ): SlashContext {
   const config = getConfig();
+  const turnInterface = conversation.getTurnInterfaceContext();
   return {
     messageCount: conversation.messages.length,
     inputTokens: conversation.usageStats.inputTokens,
@@ -204,6 +191,7 @@ function buildSlashContext(
     model: config.services.inference.model,
     provider: config.services.inference.provider,
     estimatedCost: conversation.usageStats.estimatedCost,
+    userMessageInterface: turnInterface?.userMessageInterface,
   };
 }
 
@@ -308,6 +296,13 @@ export async function drainQueue(
       const drainProvenance = provenanceFromTrustContext(
         conversation.trustContext,
       );
+      const drainImageSourcePaths: Record<string, string> = {};
+      for (let i = 0; i < next.attachments.length; i++) {
+        const a = next.attachments[i];
+        if (a.filePath && a.mimeType.toLowerCase().startsWith("image/")) {
+          drainImageSourcePaths[`${i}:${a.filename}`] = a.filePath;
+        }
+      }
       const drainChannelMeta = {
         ...drainProvenance,
         ...(queuedTurnCtx
@@ -324,8 +319,15 @@ export async function drainQueue(
             }
           : {}),
         ...(next.metadata?.automated ? { automated: true } : {}),
+        ...(Object.keys(drainImageSourcePaths).length > 0
+          ? { imageSourcePaths: drainImageSourcePaths }
+          : {}),
       };
-      const userMsg = createUserMessage(next.content, next.attachments);
+      const cleanUserMsg = createUserMessage(next.content, next.attachments);
+      const llmUserMsg = enrichMessageWithSourcePaths(
+        cleanUserMsg,
+        next.attachments,
+      );
       // When displayContent is provided (e.g. original text before recording
       // intent stripping), persist that to DB so users see the full message.
       // The in-memory userMessage (sent to the LLM) still uses the stripped content.
@@ -333,14 +335,14 @@ export async function drainQueue(
         ? JSON.stringify(
             createUserMessage(next.displayContent, next.attachments).content,
           )
-        : JSON.stringify(userMsg.content);
+        : JSON.stringify(cleanUserMsg.content);
       await addMessage(
         conversation.conversationId,
         "user",
         contentToPersist,
         drainChannelMeta,
       );
-      conversation.messages.push(userMsg);
+      conversation.messages.push(llmUserMsg);
 
       const assistantMsg = createAssistantMessage(slashResult.message);
       await addMessage(
@@ -366,10 +368,7 @@ export async function drainQueue(
 
       // Emit fresh model info before the text delta so the client has
       // up-to-date configuredProviders when rendering /model or /models UI.
-      if (
-        isModelSlashCommand(next.content) ||
-        isProviderShortcut(next.content)
-      ) {
+      if (isModelSlashCommand(next.content)) {
         next.onEvent(await buildModelInfoEvent());
       }
       next.onEvent({ type: "assistant_text_delta", text: slashResult.message });
@@ -602,6 +601,13 @@ export async function processMessage(
 
     if (routerResult.consumed) {
       const guardianIfCtx = conversation.getTurnInterfaceContext();
+      const guardianImageSourcePaths: Record<string, string> = {};
+      for (let i = 0; i < attachments.length; i++) {
+        const a = attachments[i];
+        if (a.filePath && a.mimeType.toLowerCase().startsWith("image/")) {
+          guardianImageSourcePaths[`${i}:${a.filename}`] = a.filePath;
+        }
+      }
       const routerChannelMeta = {
         userMessageChannel: "vellum" as const,
         assistantMessageChannel: "vellum" as const,
@@ -609,16 +615,23 @@ export async function processMessage(
         assistantMessageInterface:
           guardianIfCtx?.assistantMessageInterface ?? "vellum",
         provenanceTrustClass: "guardian" as const,
+        ...(Object.keys(guardianImageSourcePaths).length > 0
+          ? { imageSourcePaths: guardianImageSourcePaths }
+          : {}),
       };
 
-      const userMsg = createUserMessage(content, attachments);
+      const cleanUserMsg = createUserMessage(content, attachments);
+      const llmUserMsg = enrichMessageWithSourcePaths(
+        cleanUserMsg,
+        attachments,
+      );
       const persisted = await addMessage(
         conversation.conversationId,
         "user",
-        JSON.stringify(userMsg.content),
+        JSON.stringify(cleanUserMsg.content),
         routerChannelMeta,
       );
-      conversation.messages.push(userMsg);
+      conversation.messages.push(llmUserMsg);
 
       const replyText =
         routerResult.replyText ??
@@ -666,6 +679,13 @@ export async function processMessage(
     const pmTurnCtx = conversation.getTurnChannelContext();
     const pmInterfaceCtx = conversation.getTurnInterfaceContext();
     const pmProvenance = provenanceFromTrustContext(conversation.trustContext);
+    const pmImageSourcePaths: Record<string, string> = {};
+    for (let i = 0; i < attachments.length; i++) {
+      const a = attachments[i];
+      if (a.filePath && a.mimeType.toLowerCase().startsWith("image/")) {
+        pmImageSourcePaths[`${i}:${a.filename}`] = a.filePath;
+      }
+    }
     const pmChannelMeta = {
       ...pmProvenance,
       ...(pmTurnCtx
@@ -680,21 +700,25 @@ export async function processMessage(
             assistantMessageInterface: pmInterfaceCtx.assistantMessageInterface,
           }
         : {}),
+      ...(Object.keys(pmImageSourcePaths).length > 0
+        ? { imageSourcePaths: pmImageSourcePaths }
+        : {}),
     };
-    const userMsg = createUserMessage(content, attachments);
+    const cleanUserMsg = createUserMessage(content, attachments);
+    const llmUserMsg = enrichMessageWithSourcePaths(cleanUserMsg, attachments);
     // When displayContent is provided (e.g. original text before recording
     // intent stripping), persist that to DB so users see the full message.
     // The in-memory userMessage (sent to the LLM) still uses the stripped content.
     const contentToPersist = displayContent
       ? JSON.stringify(createUserMessage(displayContent, attachments).content)
-      : JSON.stringify(userMsg.content);
+      : JSON.stringify(cleanUserMsg.content);
     const persisted = await addMessage(
       conversation.conversationId,
       "user",
       contentToPersist,
       pmChannelMeta,
     );
-    conversation.messages.push(userMsg);
+    conversation.messages.push(llmUserMsg);
 
     const assistantMsg = createAssistantMessage(slashResult.message);
     await addMessage(
@@ -720,7 +744,7 @@ export async function processMessage(
 
     // Emit fresh model info before the text delta so the client has
     // up-to-date configuredProviders when rendering /model or /models UI.
-    if (isModelSlashCommand(content) || isProviderShortcut(content)) {
+    if (isModelSlashCommand(content)) {
       onEvent(await buildModelInfoEvent());
     }
     onEvent({ type: "assistant_text_delta", text: slashResult.message });

package/src/daemon/conversation-runtime-assembly.ts

@@ -15,7 +15,7 @@ import {
   type TurnChannelContext,
   type TurnInterfaceContext,
 } from "../channels/types.js";
-import { getAppsDir, listAppFiles } from "../memory/app-store.js";
+import { getAppDirPath, listAppFiles } from "../memory/app-store.js";
 import type { Message } from "../providers/types.js";
 import type { ActorTrustContext } from "../runtime/actor-trust-resolver.js";
 import { channelStatusToMemberStatus } from "../runtime/routes/inbound-stages/acl-enforcement.js";
@@ -266,6 +266,8 @@ export interface ActiveSurfaceContext {
   /** When set, the surface is backed by a persisted app. */
   appId?: string;
   appName?: string;
+  /** Filesystem directory/slug for the app (used to construct file paths). */
+  appDirName?: string;
   appSchemaJson?: string;
   /** Additional pages keyed by filename (e.g. "settings.html" → HTML content). */
   appPages?: Record<string, string>;
@@ -297,19 +299,18 @@ export function injectActiveSurfaceContext(
 
   if (ctx.appId) {
     // ── App-backed surface ──
+    const slug = ctx.appDirName ?? ctx.appId;
     lines.push(
-      `The user is viewing app "${ctx.appName ?? "Untitled"}" (app_id: "${ctx.appId}") in workspace mode.`,
+      `The user is viewing app "${ctx.appName ?? "Untitled"}" (app_id: "${ctx.appId}", slug: "${slug}") in workspace mode.`,
       "",
-      'PREREQUISITE: If `app_*` tools (e.g. `app_file_edit`, `app_file_write`) are not yet available, call `skill_load` with `id: "app-builder"` first to load them.',
+      'PREREQUISITE: If `app_refresh` is not yet available, call `skill_load` with `id: "app-builder"` first to load it.',
       "",
       "RULES FOR WORKSPACE MODIFICATION:",
-      `1. Use \`app_file_edit\` with app_id "${ctx.appId}" for surgical changes.`,
-      "   Provide old_string (exact match) and new_string (replacement).",
-      '   Include a short `status` message describing what you\'re doing (e.g. "adding dark mode styles").',
-      "2. Use `app_file_write` to create new files or fully rewrite files. Include `status`.",
-      "3. Use `app_file_read` to read any file with line numbers before editing.",
-      "4. Use `app_file_list` to see all files in the app.",
-      "5. The surface refreshes automatically after file edits — do NOT call app_update, ui_show, or ui_update.",
+      `1. Use \`file_edit\` to make surgical changes to app files. The file path is \`${getAppDirPath(ctx.appId)}/<path>\`.`,
+      "2. Use `file_write` to create new files or rewrite files.",
+      "3. Use `file_read` to read any file with line numbers before editing.",
+      "4. Use `bash ls` to see all files in the app directory.",
+      `5. Call \`app_refresh\` with app_id "${ctx.appId}" ONCE after all changes are complete.`,
       "6. NEVER respond with only text — the user expects a visual update.",
       "7. Make ONLY the changes the user requested. Preserve existing content/styling.",
       "8. Keep your text response to 1 brief sentence confirming what you changed.",
@@ -323,7 +324,7 @@ export function injectActiveSurfaceContext(
     for (const filePath of displayFiles) {
       let sizeLabel: string;
       try {
-        const bytes = statSync(join(getAppsDir(), ctx.appId, filePath)).size;
+        const bytes = statSync(join(getAppDirPath(ctx.appId), filePath)).size;
         sizeLabel =
           bytes < 1000 ? `${bytes} B` : `${(bytes / 1024).toFixed(1)} KB`;
       } catch {
@@ -634,6 +635,15 @@ export function buildTurnContextBlock(
       lines.push(`assistant_message_channel: ${assistant}`);
       lines.push(`conversation_origin_channel: ${origin}`);
     }
+    // Only inject response discretion for external channels (Slack, Telegram,
+    // etc.) where the assistant may receive thread replies not directed at it.
+    // The "vellum" channel is the web/desktop interface where every message is
+    // intentionally directed at the assistant.
+    if (user !== "vellum") {
+      lines.push(
+        `response_discretion: Not every message in a channel thread requires your response. If a message is clearly not directed at you (e.g. people talking among themselves, acknowledgements, reactions), output exactly <no_response/> as your entire reply to stay silent.`,
+      );
+    }
   }
 
   lines.push("</turn_context>");
@@ -951,6 +961,7 @@ const RUNTIME_INJECTION_PREFIXES = [
   "<inbound_actor_context>",
   "<interface_turn_context>",
   "<turn_context>",
+  "<memory_brief>",
   "<memory_context __injected>",
   "<memory_context>", // backward-compat: strip legacy blocks from pre-__injected history
   "<voice_call_control>",
@@ -1126,30 +1137,3 @@ export function applyRuntimeInjections(
 
   return result;
 }
-
-// ---------------------------------------------------------------------------
-// Attachment detection
-// ---------------------------------------------------------------------------
-
-/** Content block types that indicate user-uploaded attachments. */
-const ATTACHMENT_CONTENT_TYPES = new Set(["image", "file"]);
-
-/**
- * Scan conversation messages for user-uploaded attachment content blocks
- * (image or file). Returns true as soon as any attachment is found.
- *
- * Used to set the one-way `hasAttachments` flag on Conversation so that asset
- * tools (asset_search, asset_materialize) are included in tool definitions
- * only when the conversation contains attachments.
- */
-export function messagesContainAttachments(messages: Message[]): boolean {
-  for (const message of messages) {
-    if (message.role !== "user") continue;
-    for (const block of message.content) {
-      if (ATTACHMENT_CONTENT_TYPES.has(block.type)) {
-        return true;
-      }
-    }
-  }
-  return false;
-}