@vellumai/assistant 0.8.5 → 0.8.7

package/src/memory/conversation-crud.ts

@@ -25,7 +25,10 @@ import { parseChannelId, parseInterfaceId } from "../channels/types.js";
 import { CHANNEL_IDS, isChannelId } from "../channels/types.js";
 import { getConfig } from "../config/loader.js";
 import { findDisplayTurnEndIndex } from "../conversations/message-consolidation.js";
+import { conversationMetadataSyncTag } from "../daemon/message-types/sync.js";
 import type { TrustContext } from "../daemon/trust-context.js";
+import { clearAllConversationIds } from "../home/feed-writer.js";
+import { publishSyncInvalidation } from "../runtime/sync/sync-publisher.js";
 import { UserError } from "../util/errors.js";
 import { safeParseRecord } from "../util/json.js";
 import { getLogger } from "../util/logger.js";
@@ -181,6 +184,24 @@ export function provenanceFromTrustContext(
   };
 }
 
+/** Extract image file paths from resolved attachments for message metadata. */
+export function extractImageSourcePaths(
+  attachments: ReadonlyArray<{
+    filename: string;
+    mimeType: string;
+    filePath?: string;
+  }>,
+): Record<string, string> | undefined {
+  const paths: Record<string, string> = {};
+  for (let i = 0; i < attachments.length; i++) {
+    const a = attachments[i];
+    if (a.filePath && a.mimeType.toLowerCase().startsWith("image/")) {
+      paths[`${i}:${a.filename}`] = a.filePath;
+    }
+  }
+  return Object.keys(paths).length > 0 ? paths : undefined;
+}
+
 export interface ConversationRow {
   id: string;
   title: string | null;
@@ -246,6 +267,9 @@ export const parseConversation = createRowMapper<
   lastNotifiedInferenceProfile: "lastNotifiedInferenceProfile",
 });
 
+/** Allowed values for the `role` column on `messages`. */
+export type MessageRole = "user" | "assistant" | "system";
+
 export interface MessageRow {
   id: string;
   conversationId: string;
@@ -280,6 +304,193 @@ function monotonicNow(): number {
   return lastTimestamp;
 }
 
+// ── insertMessageCore ─────────────────────────────────────────────────
+
+/** Shape returned by {@link insertMessageCore} and its public wrappers. */
+interface InsertedMessage {
+  id: string;
+  conversationId: string;
+  role: MessageRole;
+  content: string;
+  createdAt: number;
+  metadata?: string;
+  clientMessageId?: string;
+  deduplicated: boolean;
+}
+
+interface InsertMessageCoreParams {
+  conversationId: string;
+  role: MessageRole;
+  content: string;
+  metadata?: Record<string, unknown>;
+  clientMessageId?: string;
+}
+
+/**
+ * Core message persistence primitive shared by {@link addMessage} and
+ * {@link reserveMessage}.
+ *
+ * Inserts a message row inside a transaction that atomically bumps the
+ * parent conversation's `updatedAt` / `lastMessageAt` timestamps and
+ * conditionally sets the conversation's `originChannel` when the first
+ * channel-originated message arrives.
+ *
+ * When a `clientMessageId` is provided the insert runs inside a
+ * SAVEPOINT. If the partial unique index on
+ * `(conversation_id, client_message_id)` raises
+ * `SQLITE_CONSTRAINT_UNIQUE`, the SAVEPOINT is rolled back, the
+ * existing row is fetched, and returned with `deduplicated: true`.
+ * This makes the operation idempotent for client-generated
+ * correlation nonces.
+ *
+ * Retries up to 3 times on `SQLITE_BUSY*` / `SQLITE_IOERR*` to handle
+ * WAL contention. The timestamp is recomputed each attempt so a late
+ * retry doesn't persist a stale `updatedAt`.
+ */
+async function insertMessageCore(
+  params: InsertMessageCoreParams,
+): Promise<InsertedMessage> {
+  const { conversationId, role, content, metadata, clientMessageId } = params;
+  const db = getDb();
+  const messageId = uuid();
+
+  if (metadata) {
+    const result = messageMetadataSchema.safeParse(metadata);
+    if (!result.success) {
+      log.warn(
+        { conversationId, messageId, issues: result.error.issues },
+        "Invalid message metadata, storing as-is",
+      );
+    }
+  }
+
+  const metadataStr = metadata ? JSON.stringify(metadata) : undefined;
+  const originChannelCandidate =
+    metadata && isChannelId(metadata.userMessageChannel)
+      ? metadata.userMessageChannel
+      : null;
+
+  const MAX_RETRIES = 3;
+  let now!: number;
+  for (let attempt = 0; ; attempt++) {
+    now = monotonicNow();
+    try {
+      const values = {
+        id: messageId,
+        conversationId,
+        role,
+        content,
+        createdAt: now,
+        ...(metadataStr ? { metadata: metadataStr } : {}),
+        ...(clientMessageId ? { clientMessageId } : {}),
+      };
+
+      if (clientMessageId) {
+        // Idempotent insert: skip silently if this clientMessageId was
+        // already persisted for the conversation.
+        const raw = getSqliteFrom(db);
+        raw.exec("SAVEPOINT insert_msg");
+        try {
+          db.insert(messages).values(values).run();
+          if (originChannelCandidate) {
+            db.update(conversations)
+              .set({ originChannel: originChannelCandidate })
+              .where(
+                and(
+                  eq(conversations.id, conversationId),
+                  isNull(conversations.originChannel),
+                ),
+              )
+              .run();
+          }
+          db.update(conversations)
+            .set({ updatedAt: now, lastMessageAt: now })
+            .where(eq(conversations.id, conversationId))
+            .run();
+          raw.exec("RELEASE insert_msg");
+        } catch (insertErr) {
+          raw.exec("ROLLBACK TO insert_msg");
+          raw.exec("RELEASE insert_msg");
+          const code = (insertErr as { code?: string }).code ?? "";
+          if (code === "SQLITE_CONSTRAINT_UNIQUE") {
+            // Duplicate clientMessageId — return the existing row.
+            const existing = db
+              .select()
+              .from(messages)
+              .where(
+                and(
+                  eq(messages.conversationId, conversationId),
+                  eq(messages.clientMessageId, clientMessageId),
+                ),
+              )
+              .get();
+            if (existing) {
+              return {
+                id: existing.id,
+                conversationId: existing.conversationId,
+                role: existing.role as MessageRole,
+                content: existing.content,
+                createdAt: existing.createdAt,
+                ...(existing.metadata ? { metadata: existing.metadata } : {}),
+                clientMessageId: existing.clientMessageId ?? undefined,
+                deduplicated: true,
+              };
+            }
+          }
+          throw insertErr;
+        }
+      } else {
+        // No clientMessageId — standard insert inside a transaction.
+        db.transaction((tx) => {
+          tx.insert(messages).values(values).run();
+          if (originChannelCandidate) {
+            tx.update(conversations)
+              .set({ originChannel: originChannelCandidate })
+              .where(
+                and(
+                  eq(conversations.id, conversationId),
+                  isNull(conversations.originChannel),
+                ),
+              )
+              .run();
+          }
+          tx.update(conversations)
+            .set({ updatedAt: now, lastMessageAt: now })
+            .where(eq(conversations.id, conversationId))
+            .run();
+        });
+      }
+      break;
+    } catch (err) {
+      const errCode = (err as { code?: string }).code ?? "";
+      if (
+        attempt < MAX_RETRIES &&
+        (errCode.startsWith("SQLITE_BUSY") ||
+          errCode.startsWith("SQLITE_IOERR"))
+      ) {
+        log.warn(
+          { attempt, conversationId, code: errCode },
+          "insertMessageCore: transient SQLite error, retrying",
+        );
+        await Bun.sleep(50 * (attempt + 1));
+        continue;
+      }
+      throw err;
+    }
+  }
+
+  return {
+    id: messageId,
+    conversationId,
+    role,
+    content,
+    createdAt: now,
+    ...(metadataStr ? { metadata: metadataStr } : {}),
+    ...(clientMessageId ? { clientMessageId } : {}),
+    deduplicated: false,
+  };
+}
+
 export function createConversation(
   titleOrOpts?:
     | string
@@ -576,6 +787,20 @@ export function forkConversation(params: {
     throw new UserError(`Conversation ${conversationId} not found`);
   }
   const sourceMessages = getMessages(conversationId);
+  if (throughMessageId != null) {
+    // `getMessages` orders by `createdAt` only; when rows share an identical
+    // millisecond timestamp the tie order is unspecified. Callers that pin the
+    // fork to a cutoff choose it from a `(createdAt, id)` cursor (e.g. the
+    // memory-retrospective job, via `getMessagesAfter`), so slicing through
+    // `throughMessageId` under the unstable order could include same-timestamp
+    // siblings the cursor considers *after* the cutoff (reprocessed next run)
+    // or exclude ones it considers *before* it (skipped forever). Re-sort on
+    // `(createdAt, id)` so the slice agrees with the cutoff. The unpinned full
+    // fork copies every row regardless of order, so it keeps source order.
+    sourceMessages.sort(
+      (a, b) => a.createdAt - b.createdAt || a.id.localeCompare(b.id),
+    );
+  }
 
   if (sourceMessages.length === 0) {
     throw new UserError(
@@ -970,94 +1195,44 @@ export function wipeConversation(id: string): WipeConversationResult {
   };
 }
 
+/** Options for {@link addMessage}. Only `skipIndexing` and `clientMessageId`
+ *  have defaults; `metadata` is genuinely optional. */
+export interface AddMessageOptions {
+  metadata?: Record<string, unknown>;
+  skipIndexing?: boolean;
+  /** Client-generated nonce for idempotent inserts. When provided,
+   *  duplicate inserts for the same `(conversationId, clientMessageId)`
+   *  pair are silently skipped. */
+  clientMessageId?: string;
+}
+
+/**
+ * Persist a message and run post-insert side effects (memory indexing,
+ * attention projection). Delegates the core insert + retry logic to
+ * {@link insertMessageCore}.
+ */
 export async function addMessage(
   conversationId: string,
-  role: string,
+  role: MessageRole,
   content: string,
-  metadata?: Record<string, unknown>,
-  opts?: { skipIndexing?: boolean },
+  options?: AddMessageOptions,
 ) {
-  const db = getDb();
-  const messageId = uuid();
-
-  if (metadata) {
-    const result = messageMetadataSchema.safeParse(metadata);
-    if (!result.success) {
-      log.warn(
-        { conversationId, messageId, issues: result.error.issues },
-        "Invalid message metadata, storing as-is",
-      );
-    }
-  }
-
-  const metadataStr = metadata ? JSON.stringify(metadata) : undefined;
-  const originChannelCandidate =
-    metadata && isChannelId(metadata.userMessageChannel)
-      ? metadata.userMessageChannel
-      : null;
-  // Wrap insert + updatedAt bump in a transaction so they're atomic.
-  // Retry on SQLITE_BUSY* and SQLITE_IOERR* — covers WAL contention variants
-  // (SQLITE_BUSY_SNAPSHOT, SQLITE_BUSY_RECOVERY) and transient disk I/O errors.
-  // Timestamp is recomputed each attempt so a late retry doesn't persist a stale updatedAt.
-  const MAX_RETRIES = 3;
-  let now!: number;
-  for (let attempt = 0; ; attempt++) {
-    now = monotonicNow();
-    try {
-      const values = {
-        id: messageId,
-        conversationId,
-        role,
-        content,
-        createdAt: now,
-        ...(metadataStr ? { metadata: metadataStr } : {}),
-      };
-      db.transaction((tx) => {
-        tx.insert(messages).values(values).run();
-        if (originChannelCandidate) {
-          tx.update(conversations)
-            .set({ originChannel: originChannelCandidate })
-            .where(
-              and(
-                eq(conversations.id, conversationId),
-                isNull(conversations.originChannel),
-              ),
-            )
-            .run();
-        }
-        tx.update(conversations)
-          .set({ updatedAt: now, lastMessageAt: now })
-          .where(eq(conversations.id, conversationId))
-          .run();
-      });
-      break;
-    } catch (err) {
-      const errCode = (err as { code?: string }).code ?? "";
-      if (
-        attempt < MAX_RETRIES &&
-        (errCode.startsWith("SQLITE_BUSY") ||
-          errCode.startsWith("SQLITE_IOERR"))
-      ) {
-        log.warn(
-          { attempt, conversationId, code: errCode },
-          "addMessage: transient SQLite error, retrying",
-        );
-        await Bun.sleep(50 * (attempt + 1));
-        continue;
-      }
-      throw err;
-    }
-  }
-  const message = {
-    id: messageId,
+  const { metadata, skipIndexing, clientMessageId } = options ?? {};
+  const inserted = await insertMessageCore({
     conversationId,
     role,
     content,
-    createdAt: now,
-    ...(metadataStr ? { metadata: metadataStr } : {}),
-  };
+    metadata,
+    clientMessageId,
+  });
 
-  if (!opts?.skipIndexing) {
+  if (inserted.deduplicated) {
+    return inserted;
+  }
+
+  const message = inserted;
+
+  if (!skipIndexing) {
     try {
       const config = getConfig();
       const parsed = metadata
@@ -1090,11 +1265,16 @@ export async function addMessage(
 
   if (role === "assistant") {
     try {
-      projectAssistantMessage({
+      const attentionStateChanged = projectAssistantMessage({
         conversationId,
         messageId: message.id,
         messageAt: message.createdAt,
       });
+      if (attentionStateChanged) {
+        void publishSyncInvalidation([
+          conversationMetadataSyncTag(conversationId),
+        ]);
+      }
     } catch (err) {
       log.warn(
         { err, conversationId, messageId: message.id },
@@ -1288,11 +1468,30 @@ export function hasMessages(conversationId: string): boolean {
 interface PaginatedMessagesResult {
   messages: MessageRow[];
   hasMore: boolean;
+  /**
+   * Position of the last row scanned when the loop stops on
+   * `PAGINATION_SCAN_CAP` rather than DB exhaustion. Callers derive their
+   * client cursor from the visible page's oldest row, but a cap-truncated
+   * page can be empty (a contiguous block of filtered-out rows longer than
+   * the cap), leaving nothing to resume from. Surfacing the last scanned
+   * `(createdAt, id)` lets the caller hand the client a cursor so it can
+   * request the next window and keep draining instead of stalling.
+   */
+  nextCursor?: { createdAt: number; id: string };
 }
 
 const PAGINATION_CHUNK_MIN = 50;
 const PAGINATION_SCAN_CAP = 10_000;
 
+// Test-only override for PAGINATION_SCAN_CAP so tests can exercise the
+// cap-truncation branch with a small cap instead of seeding >10k rows (which
+// makes the suite slow and the post-test DELETE flaky under parallel CI load).
+// `undefined` restores the production cap.
+let paginationScanCapOverride: number | undefined;
+export function _setPaginationScanCapForTesting(cap: number | undefined): void {
+  paginationScanCapOverride = cap;
+}
+
 export function getMessagesPaginated(
   conversationId: string,
   limit: number | undefined,
@@ -1333,8 +1532,19 @@ export function getMessagesPaginated(
   // row — otherwise a pathological filter against a huge conversation would
   // tie up a connection for thousands of roundtrips.
   let rowsScanned = 0;
-
-  while (visible.length < limit + 1 && rowsScanned < PAGINATION_SCAN_CAP) {
+  // Distinguish "stopped because we hit the scan cap" from "stopped because the
+  // DB ran out of rows". On a cap-truncated stop there may be more visible rows
+  // past the scanned window, so `hasMore` must stay true and we record the last
+  // scanned position as a resume cursor (the visible page may be empty).
+  let scanCapTruncated = false;
+  let lastScanned: { createdAt: number; id: string } | undefined;
+  const scanCap = paginationScanCapOverride ?? PAGINATION_SCAN_CAP;
+
+  while (visible.length < limit + 1) {
+    if (rowsScanned >= scanCap) {
+      scanCapTruncated = true;
+      break;
+    }
     const cursorPredicate =
       cursorCreatedAt === undefined
         ? undefined
@@ -1367,15 +1577,25 @@ export function getMessagesPaginated(
 
     if (chunk.length < chunkSize) break;
     const lastRow = chunk[chunk.length - 1];
+    lastScanned = { createdAt: lastRow.createdAt, id: lastRow.id };
     cursorCreatedAt = lastRow.createdAt;
     cursorMessageId = lastRow.id;
   }
 
-  const hasMore = visible.length > limit;
-  if (hasMore) visible.splice(limit);
+  const filledPage = visible.length > limit;
+  // A cap-truncated stop means the DB may still hold older visible rows past
+  // the scanned window, so report `hasMore: true` to keep the client draining
+  // — returning `false` here is the stall this loop exists to prevent.
+  const hasMore = filledPage || scanCapTruncated;
+  if (filledPage) visible.splice(limit);
   visible.reverse();
 
-  return { messages: visible, hasMore };
+  // Only hand back a resume cursor when the cap (not DB exhaustion) cut the
+  // search short; callers fall back to it when the visible page came back
+  // empty and has no oldest row to anchor the next request.
+  const nextCursor = scanCapTruncated ? lastScanned : undefined;
+
+  return { messages: visible, hasMore, nextCursor };
 }
 
 export function getLastUserTimestampBefore(
@@ -1399,6 +1619,31 @@ export function getLastUserTimestampBefore(
   return row?.createdAt ?? 0;
 }
 
+/**
+ * Most recent user-message timestamp (epoch ms) across all conversations, or
+ * `0` when no user message exists.
+ *
+ * Ordered by `created_at` rather than insertion order, because `forkConversation`
+ * copies a parent's messages into the fork while preserving their original
+ * `created_at`. Those copies receive fresh row ids, so an insertion-order scan
+ * could surface an old forked turn as if it were the latest activity and let
+ * maintenance run while the user is in fact active. The `(role, created_at)`
+ * index (`idx_messages_role_created_at`) makes this an indexed seek to the
+ * newest `role = "user"` row rather than a scan of the whole (potentially
+ * multi-GB) table.
+ */
+export function getLastUserMessageTimestamp(): number {
+  const db = getDb();
+  const row = db
+    .select({ createdAt: messages.createdAt })
+    .from(messages)
+    .where(eq(messages.role, "user"))
+    .orderBy(desc(messages.createdAt))
+    .limit(1)
+    .get();
+  return row?.createdAt ?? 0;
+}
+
 /** Fetch a single message by ID, optionally scoped to a specific conversation. */
 export function getMessageById(
   messageId: string,
@@ -1821,6 +2066,8 @@ export async function clearAll(): Promise<{
     log.warn({ err }, "clearAll: failed to reset conversations directory");
   }
 
+  void clearAllConversationIds();
+
   return { conversations: convCount, messages: msgCount };
 }
 
@@ -1913,6 +2160,29 @@ interface WipeConversationResult extends DeletedMemoryIds {
   cancelledJobCount: number;
 }
 
+/**
+ * Reserve an empty message row so the agent loop can stamp outbound
+ * streaming events with a stable identity before content is produced.
+ *
+ * Intentionally skips Qdrant indexing and attention projection — an empty
+ * placeholder is meaningless for either. The caller writes final content
+ * via {@link updateMessageContent} and handles indexing/projection itself.
+ *
+ * Delegates the core insert + retry logic to {@link insertMessageCore}.
+ */
+export async function reserveMessage(
+  conversationId: string,
+  role: MessageRole,
+  metadata?: Record<string, unknown>,
+) {
+  return insertMessageCore({
+    conversationId,
+    role,
+    content: "[]",
+    metadata,
+  });
+}
+
 /**
  * Update the content of an existing message. Used when consolidating
  * multiple assistant messages into one.
@@ -2186,7 +2456,7 @@ export function batchSetDisplayOrders(
   updates: Array<{
     id: string;
     displayOrder: number | null;
-    isPinned: boolean;
+    isPinned?: boolean;
     groupId?: string | null;
   }>,
 ): void {
@@ -2219,35 +2489,36 @@ export function batchSetDisplayOrders(
           safeGroupId,
           update.id,
         );
+      } else if (update.isPinned === undefined) {
+        // Only displayOrder provided — preserve existing pin state and group.
+        rawRun(
+          "UPDATE conversations SET display_order = ? WHERE id = ?",
+          update.displayOrder,
+          update.id,
+        );
+      } else if (update.isPinned) {
+        rawRun(
+          "UPDATE conversations SET display_order = ?, is_pinned = 1, group_id = 'system:pinned' WHERE id = ?",
+          update.displayOrder,
+          update.id,
+        );
       } else {
-        // Old client: no groupId in payload
-        // isPinned true -> set group_id = system:pinned
-        // isPinned false -> clear group_id ONLY IF currently system:pinned
-        //                   otherwise preserve existing group_id
-        if (update.isPinned) {
-          rawRun(
-            "UPDATE conversations SET display_order = ?, is_pinned = 1, group_id = 'system:pinned' WHERE id = ?",
-            update.displayOrder,
-            update.id,
-          );
-        } else {
-          // Restore system group from source/conversationType when old clients
-          // unpin, instead of clearing to NULL (which would lose provenance).
-          rawRun(
-            `UPDATE conversations SET display_order = ?, is_pinned = 0,
-             group_id = CASE WHEN group_id = 'system:pinned' THEN
-               CASE
-                 WHEN source IN ('schedule', 'reminder') THEN 'system:scheduled'
-                 WHEN source IN ('heartbeat', 'task') THEN 'system:background'
-                 WHEN conversation_type = 'background' AND COALESCE(source, '') != 'notification' THEN 'system:background'
-                 ELSE 'system:all'
-               END
-             ELSE group_id END
-             WHERE id = ?`,
-            update.displayOrder,
-            update.id,
-          );
-        }
+        // Restore system group from source/conversationType when unpinning,
+        // instead of clearing to NULL (which would lose provenance).
+        rawRun(
+          `UPDATE conversations SET display_order = ?, is_pinned = 0,
+           group_id = CASE WHEN group_id = 'system:pinned' THEN
+             CASE
+               WHEN source IN ('schedule', 'reminder') THEN 'system:scheduled'
+               WHEN source IN ('heartbeat', 'task') THEN 'system:background'
+               WHEN conversation_type = 'background' AND COALESCE(source, '') != 'notification' THEN 'system:background'
+               ELSE 'system:all'
+             END
+           ELSE group_id END
+           WHERE id = ?`,
+          update.displayOrder,
+          update.id,
+        );
       }
     }
     rawExec("COMMIT");