@vellumai/assistant 0.5.1 → 0.5.3

package/src/memory/conversation-directories.ts

@@ -0,0 +1,125 @@
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+
+import { getConversationsDir } from "../util/platform.js";
+
+function getConversationDirTimestamp(createdAtMs: number): string {
+  return new Date(createdAtMs).toISOString().replace(/:/g, "-");
+}
+
+export function getLegacyConversationDirName(
+  id: string,
+  createdAtMs: number,
+): string {
+  return `${id}_${getConversationDirTimestamp(createdAtMs)}`;
+}
+
+/**
+ * Build a filesystem-safe directory name for a conversation.
+ * Format: `{isoDate}_{id}` where colons in the ISO date are replaced with
+ * hyphens so the name is valid on all platforms (Windows forbids colons).
+ */
+export function getConversationDirName(
+  id: string,
+  createdAtMs: number,
+): string {
+  return `${getConversationDirTimestamp(createdAtMs)}_${id}`;
+}
+
+/**
+ * Return the absolute path to a conversation's timestamp-first disk-view
+ * directory.
+ */
+export function getConversationDirPath(
+  id: string,
+  createdAtMs: number,
+): string {
+  return join(getConversationsDir(), getConversationDirName(id, createdAtMs));
+}
+
+export function getLegacyConversationDirPath(
+  id: string,
+  createdAtMs: number,
+): string {
+  return join(
+    getConversationsDir(),
+    getLegacyConversationDirName(id, createdAtMs),
+  );
+}
+
+export interface ResolvedConversationDirectoryPaths {
+  canonicalDirPath: string;
+  canonicalDirName: string;
+  legacyDirPath: string;
+  legacyDirName: string;
+  resolvedDirPath: string;
+  resolvedDirName: string;
+  hasCanonicalDir: boolean;
+  hasLegacyDir: boolean;
+}
+
+export function resolveConversationDirectoryPaths(
+  id: string,
+  createdAtMs: number,
+  conversationsDir: string = getConversationsDir(),
+): ResolvedConversationDirectoryPaths {
+  const canonicalDirName = getConversationDirName(id, createdAtMs);
+  const canonicalDirPath = join(conversationsDir, canonicalDirName);
+  const hasCanonicalDir = existsSync(canonicalDirPath);
+
+  const legacyDirName = getLegacyConversationDirName(id, createdAtMs);
+  const legacyDirPath = join(conversationsDir, legacyDirName);
+  const hasLegacyDir = existsSync(legacyDirPath);
+
+  const resolvedDirPath = hasCanonicalDir
+    ? canonicalDirPath
+    : hasLegacyDir
+      ? legacyDirPath
+      : canonicalDirPath;
+  const resolvedDirName = hasCanonicalDir
+    ? canonicalDirName
+    : hasLegacyDir
+      ? legacyDirName
+      : canonicalDirName;
+
+  return {
+    canonicalDirPath,
+    canonicalDirName,
+    legacyDirPath,
+    legacyDirName,
+    resolvedDirPath,
+    resolvedDirName,
+    hasCanonicalDir,
+    hasLegacyDir,
+  };
+}
+
+/**
+ * Resolve the active conversation directory path:
+ * 1) prefer timestamp-first when it exists;
+ * 2) otherwise reuse legacy sibling when present;
+ * 3) otherwise fall back to timestamp-first as the creation target.
+ */
+export function getResolvedConversationDirPath(
+  id: string,
+  createdAtMs: number,
+): string {
+  return resolveConversationDirectoryPaths(id, createdAtMs).resolvedDirPath;
+}
+
+export function getResolvedConversationDirName(
+  id: string,
+  createdAtMs: number,
+): string {
+  return resolveConversationDirectoryPaths(id, createdAtMs).resolvedDirName;
+}
+
+export function getConversationAttachmentsDirPath(
+  conversationId: string,
+  createdAtMs: number,
+): string {
+  return join(
+    getResolvedConversationDirPath(conversationId, createdAtMs),
+    "attachments",
+  );
+}

package/src/memory/conversation-disk-view.ts

@@ -0,0 +1,390 @@
+/**
+ * Write-through disk view for conversations.
+ *
+ * Projects conversation metadata, messages, and attachments to a browsable
+ * filesystem layout under ~/.vellum/workspace/conversations/. This enables
+ * the assistant to search/read/manipulate conversation data using standard
+ * file tools.
+ *
+ * All disk writes are best-effort — failures are logged but never thrown,
+ * so the disk view cannot break DB operations.
+ */
+
+import {
+  appendFileSync,
+  existsSync,
+  mkdirSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { basename, dirname, extname, join } from "node:path";
+
+import { getLogger } from "../util/logger.js";
+import {
+  getAttachmentContent,
+  getAttachmentMetadataForMessage,
+  getFilePathForAttachment,
+} from "./attachments-store.js";
+import {
+  getConversation,
+  getMessageById,
+  getMessages,
+} from "./conversation-crud.js";
+import {
+  getConversationDirName,
+  getConversationDirPath,
+  getLegacyConversationDirPath,
+  getResolvedConversationDirPath,
+} from "./conversation-directories.js";
+
+const log = getLogger("conversation-disk-view");
+
+// ---------------------------------------------------------------------------
+// Directory helpers
+// ---------------------------------------------------------------------------
+
+export {
+  getConversationDirName,
+  getConversationDirPath,
+  getResolvedConversationDirPath,
+};
+
+function ensureConversationDirPath(id: string, createdAtMs: number): string {
+  const dirPath = getResolvedConversationDirPath(id, createdAtMs);
+  mkdirSync(dirPath, { recursive: true });
+  return dirPath;
+}
+
+// ---------------------------------------------------------------------------
+// Write operations
+// ---------------------------------------------------------------------------
+
+/**
+ * Create the conversation directory and write the initial meta.json.
+ */
+export function initConversationDir(conv: {
+  id: string;
+  title: string | null;
+  createdAt: number;
+  conversationType: string;
+  originChannel: string | null;
+}): void {
+  try {
+    const dirPath = ensureConversationDirPath(conv.id, conv.createdAt);
+
+    const meta = {
+      id: conv.id,
+      title: conv.title,
+      type: conv.conversationType,
+      channel: conv.originChannel,
+      createdAt: new Date(conv.createdAt).toISOString(),
+      updatedAt: new Date(conv.createdAt).toISOString(),
+    };
+
+    writeFileSync(
+      join(dirPath, "meta.json"),
+      JSON.stringify(meta, null, 2) + "\n",
+    );
+  } catch (err) {
+    log.warn(
+      { err, conversationId: conv.id },
+      "Failed to init conversation dir",
+    );
+  }
+}
+
+/**
+ * Rewrite meta.json with updated fields.
+ */
+export function updateMetaFile(conv: {
+  id: string;
+  title: string | null;
+  createdAt: number;
+  updatedAt: number;
+  conversationType: string;
+  originChannel: string | null;
+}): void {
+  try {
+    const dirPath = ensureConversationDirPath(conv.id, conv.createdAt);
+
+    const meta = {
+      id: conv.id,
+      title: conv.title,
+      type: conv.conversationType,
+      channel: conv.originChannel,
+      createdAt: new Date(conv.createdAt).toISOString(),
+      updatedAt: new Date(conv.updatedAt).toISOString(),
+    };
+
+    writeFileSync(
+      join(dirPath, "meta.json"),
+      JSON.stringify(meta, null, 2) + "\n",
+    );
+  } catch (err) {
+    log.warn({ err, conversationId: conv.id }, "Failed to update meta file");
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Content flattening
+// ---------------------------------------------------------------------------
+
+interface ContentBlock {
+  type: string;
+  text?: string;
+  name?: string;
+  input?: unknown;
+  content?: unknown;
+}
+
+interface FlattenedContent {
+  content: string;
+  toolCalls: Array<{ name: string; input: unknown }>;
+  toolResults: Array<{ content: unknown }>;
+}
+
+/**
+ * Parse the message `content` JSON string (ContentBlock[]) and extract
+ * text, tool_use, and tool_result blocks into flat fields.
+ */
+export function flattenContentBlocks(rawContent: string): FlattenedContent {
+  const result: FlattenedContent = {
+    content: "",
+    toolCalls: [],
+    toolResults: [],
+  };
+
+  let blocks: ContentBlock[];
+  try {
+    const parsed = JSON.parse(rawContent);
+    if (!Array.isArray(parsed)) {
+      // Plain text content (not block array)
+      return { ...result, content: rawContent };
+    }
+    blocks = parsed;
+  } catch {
+    // Not valid JSON — treat as plain text
+    return { ...result, content: rawContent };
+  }
+
+  const textParts: string[] = [];
+
+  for (const block of blocks) {
+    switch (block.type) {
+      case "text":
+        if (typeof block.text === "string") {
+          textParts.push(block.text);
+        }
+        break;
+      case "tool_use":
+        if (typeof block.name === "string") {
+          result.toolCalls.push({ name: block.name, input: block.input });
+        }
+        break;
+      case "tool_result":
+        result.toolResults.push({ content: block.content });
+        break;
+      // Skip "image" and "file" blocks — represented via attachments
+    }
+  }
+
+  result.content = textParts.join("\n");
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// Attachment projection
+// ---------------------------------------------------------------------------
+
+/**
+ * Resolve a unique filename within a directory, handling collisions by
+ * appending a suffix (e.g., `photo-2.png`, `photo-3.png`).
+ */
+export function resolveUniqueFilename(dir: string, filename: string): string {
+  const sanitized = basename(filename);
+  if (!existsSync(join(dir, sanitized))) return sanitized;
+
+  const ext = extname(sanitized);
+  const base = basename(sanitized, ext);
+  let counter = 2;
+  let candidate: string;
+  do {
+    candidate = `${base}-${counter}${ext}`;
+    counter++;
+  } while (existsSync(join(dir, candidate)));
+
+  return candidate;
+}
+
+/**
+ * Ensure an attachment is present in the conversation's attachments/
+ * subdirectory and return the filename recorded in the disk view.
+ */
+function writeAttachmentFile(
+  conversationDirPath: string,
+  attachmentId: string,
+  originalFilename: string,
+): string | null {
+  try {
+    const attachDir = join(conversationDirPath, "attachments");
+    mkdirSync(attachDir, { recursive: true });
+
+    const existingPath = getFilePathForAttachment(attachmentId);
+    if (
+      existingPath &&
+      existsSync(existingPath) &&
+      dirname(existingPath) === attachDir
+    ) {
+      return basename(existingPath);
+    }
+
+    const content = getAttachmentContent(attachmentId);
+    if (!content) return null;
+
+    const resolvedName = resolveUniqueFilename(attachDir, originalFilename);
+    writeFileSync(join(attachDir, resolvedName), content);
+    return resolvedName;
+  } catch (err) {
+    log.warn(
+      { err, attachmentId, originalFilename },
+      "Failed to write attachment file to disk",
+    );
+    return null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Message sync
+// ---------------------------------------------------------------------------
+
+/**
+ * Read a message and its attachments from DB, flatten content, and append
+ * a JSONL line to `messages.jsonl` in the conversation's disk-view directory.
+ * Attachment filenames are recorded from the conversation's attachments/
+ * subdirectory, materializing legacy rows there only when needed.
+ *
+ * Requires `createdAtMs` of the conversation to resolve the directory path.
+ */
+export function syncMessageToDisk(
+  conversationId: string,
+  messageId: string,
+  createdAtMs: number,
+): void {
+  try {
+    const message = getMessageById(messageId, conversationId);
+    if (!message) {
+      log.warn(
+        { conversationId, messageId },
+        "syncMessageToDisk: message not found",
+      );
+      return;
+    }
+
+    const dirPath = ensureConversationDirPath(conversationId, createdAtMs);
+    const { content, toolCalls, toolResults } = flattenContentBlocks(
+      message.content,
+    );
+
+    // Project attachments to disk
+    const attachmentMeta = getAttachmentMetadataForMessage(messageId);
+    const attachmentFilenames: string[] = [];
+    for (const att of attachmentMeta) {
+      const resolved = writeAttachmentFile(
+        dirPath,
+        att.id,
+        att.originalFilename,
+      );
+      if (resolved) {
+        attachmentFilenames.push(resolved);
+      }
+    }
+
+    // Build JSONL record
+    const record: Record<string, unknown> = {
+      role: message.role,
+      ts: new Date(message.createdAt).toISOString(),
+    };
+
+    if (content) record.content = content;
+    if (toolCalls.length > 0) record.toolCalls = toolCalls;
+    if (toolResults.length > 0) record.toolResults = toolResults;
+    if (attachmentFilenames.length > 0)
+      record.attachments = attachmentFilenames;
+
+    appendFileSync(
+      join(dirPath, "messages.jsonl"),
+      JSON.stringify(record) + "\n",
+    );
+  } catch (err) {
+    log.warn(
+      { err, conversationId, messageId },
+      "Failed to sync message to disk",
+    );
+  }
+}
+
+/**
+ * Rebuild a single conversation's disk view from current DB state.
+ *
+ * This rewrites append-only `messages.jsonl` and replays all persisted messages
+ * in DB order so disk data matches post-mutation state (e.g., after assistant-
+ * message consolidation). Existing attachment files are preserved to avoid
+ * losing file-backed rows where base64 payloads were already compacted out.
+ */
+export function rebuildConversationDiskViewFromDbState(
+  conversationId: string,
+): void {
+  try {
+    const conv = getConversation(conversationId);
+    if (!conv) {
+      log.warn(
+        { conversationId },
+        "rebuildConversationDiskViewFromDbState: conversation not found",
+      );
+      return;
+    }
+
+    const dirPath = ensureConversationDirPath(conversationId, conv.createdAt);
+    const messagesPath = join(dirPath, "messages.jsonl");
+
+    rmSync(messagesPath, { force: true });
+    writeFileSync(messagesPath, "");
+    // Preserve attachment files: many attachment rows are file-backed with
+    // data_base64 cleared, so deleting attachments/ can make content
+    // unrecoverable for replay.
+    mkdirSync(join(dirPath, "attachments"), { recursive: true });
+
+    const convMessages = getMessages(conversationId);
+    for (const msg of convMessages) {
+      syncMessageToDisk(conversationId, msg.id, conv.createdAt);
+    }
+
+    updateMetaFile(conv);
+  } catch (err) {
+    log.warn(
+      { err, conversationId },
+      "Failed to rebuild conversation disk view from DB state",
+    );
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Removal
+// ---------------------------------------------------------------------------
+
+/**
+ * Remove a conversation's disk-view directory entirely.
+ */
+export function removeConversationDir(id: string, createdAtMs: number): void {
+  try {
+    const dirPaths = new Set([
+      getConversationDirPath(id, createdAtMs),
+      getLegacyConversationDirPath(id, createdAtMs),
+    ]);
+    for (const dirPath of dirPaths) {
+      rmSync(dirPath, { recursive: true, force: true });
+    }
+  } catch (err) {
+    log.warn({ err, conversationId: id }, "Failed to remove conversation dir");
+  }
+}

package/src/memory/conversation-key-store.ts

@@ -10,6 +10,7 @@
 import { eq } from "drizzle-orm";
 import { v4 as uuid } from "uuid";
 
+import { initConversationDir } from "./conversation-disk-view.js";
 import { GENERATING_TITLE } from "./conversation-title-service.js";
 import { getDb } from "./db.js";
 import { conversationKeys, conversations } from "./schema.js";
@@ -138,7 +139,7 @@ export function getOrCreateConversation(
   const db = getDb();
   const conversationType = opts?.conversationType ?? "standard";
 
-  return db.transaction((tx) => {
+  const result = db.transaction((tx) => {
     const existing = tx
       .select()
       .from(conversationKeys)
@@ -146,7 +147,7 @@ export function getOrCreateConversation(
       .get();
 
     if (existing) {
-      return { conversationId: existing.conversationId, created: false };
+      return { conversationId: existing.conversationId, created: false as const };
     }
 
     // Check if the conversationKey itself is an existing conversation ID.
@@ -167,18 +168,19 @@ export function getOrCreateConversation(
           createdAt: Date.now(),
         })
         .run();
-      return { conversationId: existingConversation.id, created: false };
+      return { conversationId: existingConversation.id, created: false as const };
     }
 
     const now = Date.now();
     const conversationId = uuid();
+    const title = GENERATING_TITLE;
     const memoryScopeId =
       conversationType === "private" ? `private:${conversationId}` : "default";
 
     tx.insert(conversations)
       .values({
         id: conversationId,
-        title: GENERATING_TITLE,
+        title,
         createdAt: now,
         updatedAt: now,
         totalInputTokens: 0,
@@ -201,6 +203,16 @@ export function getOrCreateConversation(
       })
       .run();
 
-    return { conversationId, created: true };
+    return {
+      conversationId,
+      created: true as const,
+      conversation: { id: conversationId, title, createdAt: now, conversationType },
+    };
   });
+
+  if (result.created) {
+    initConversationDir({ ...result.conversation, originChannel: null });
+  }
+
+  return { conversationId: result.conversationId, created: result.created };
 }

package/src/memory/conversation-queries.ts

@@ -101,7 +101,11 @@ export function isLastUserMessageToolResult(conversationId: string): boolean {
       parsed.every(
         (block: Record<string, unknown>) =>
           block.type === "tool_result" ||
-          block.type === "web_search_tool_result",
+          block.type === "web_search_tool_result" ||
+          (block.type === "text" &&
+            typeof block.text === "string" &&
+            block.text.startsWith("<system_notice>") &&
+            block.text.endsWith("</system_notice>")),
       )
     ) {
       return true;

package/src/memory/conversation-title-service.ts

@@ -10,6 +10,7 @@
 import { getConfig } from "../config/loader.js";
 import { getConfiguredProvider } from "../providers/provider-send-message.js";
 import type { Provider } from "../providers/types.js";
+import { runBtwSidechain } from "../runtime/btw-sidechain.js";
 import { getLogger } from "../util/logger.js";
 import { truncate } from "../util/truncate.js";
 import {
@@ -129,30 +130,16 @@ export async function generateAndPersistConversationTitle(
 
   const config = getConfig();
   const prompt = buildTitlePrompt(context, userMessage, assistantResponse);
-  const timeoutSignal = AbortSignal.timeout(10_000);
-  const combinedSignal = signal
-    ? AbortSignal.any([signal, timeoutSignal])
-    : timeoutSignal;
-
-  const response = await provider.sendMessage(
-    [{ role: "user", content: [{ type: "text", text: prompt }] }],
-    [],
-    undefined,
-    {
-      config: {
-        max_tokens: config.daemon.titleGenerationMaxTokens,
-        modelIntent: "latency-optimized",
-      },
-      signal: combinedSignal,
-    },
-  );
-  const textBlock = response.content.find((b) => b.type === "text");
-  if (textBlock && textBlock.type === "text") {
-    let title = normalizeTitle(textBlock.text);
-    if (!title) {
-      title = deriveFallbackTitle(context) ?? UNTITLED_FALLBACK;
-    }
-
+  const result = await runBtwSidechain({
+    content: prompt,
+    provider,
+    maxTokens: config.daemon.titleGenerationMaxTokens,
+    modelIntent: "latency-optimized",
+    signal,
+    timeoutMs: 10_000,
+  });
+  const title = normalizeTitle(result.text);
+  if (title) {
     // Re-check replaceability before persisting (race guard)
     const current = getConversation(conversationId);
     if (current && !isReplaceableTitle(current.title)) {
@@ -246,31 +233,16 @@ export async function regenerateConversationTitle(
 
   const prompt = buildRegenerationPrompt(recentMessages);
   const config = getConfig();
-  const timeoutSignal = AbortSignal.timeout(10_000);
-  const combinedSignal = signal
-    ? AbortSignal.any([signal, timeoutSignal])
-    : timeoutSignal;
-
-  const response = await provider.sendMessage(
-    [{ role: "user", content: [{ type: "text", text: prompt }] }],
-    [],
-    undefined,
-    {
-      config: {
-        max_tokens: config.daemon.titleGenerationMaxTokens,
-        modelIntent: "latency-optimized",
-      },
-      signal: combinedSignal,
-    },
-  );
-
-  const textBlock = response.content.find((b) => b.type === "text");
-  if (textBlock && textBlock.type === "text") {
-    const title = normalizeTitle(textBlock.text);
-    if (!title) {
-      return { title: conversation.title ?? UNTITLED_FALLBACK, updated: false };
-    }
-
+  const result = await runBtwSidechain({
+    content: prompt,
+    provider,
+    maxTokens: config.daemon.titleGenerationMaxTokens,
+    modelIntent: "latency-optimized",
+    signal,
+    timeoutMs: 10_000,
+  });
+  const title = normalizeTitle(result.text);
+  if (title) {
     // Re-check isAutoTitle before persisting (race guard against manual rename)
     const current = getConversation(conversationId);
     if (!current || !current.isAutoTitle) {