@vellumai/assistant 0.5.16 → 0.6.1

package/src/memory/conversation-group-migration.ts

@@ -0,0 +1,157 @@
+/**
+ * Runtime migration for the conversation_groups table and group_id column
+ * on the conversations table. Follows the same lazy-initialization pattern
+ * as conversation-display-order-migration.ts.
+ *
+ * group_id is display/organizational metadata, consistent with how
+ * display_order and is_pinned are handled via runtime migration rather
+ * than the formal migrations/ pipeline.
+ */
+
+import { getLogger } from "../util/logger.js";
+import { ensureDisplayOrderMigration } from "./conversation-display-order-migration.js";
+import { rawExec, rawGet, rawRun } from "./db.js";
+
+const log = getLogger("conversation-store");
+
+function isDuplicateColumnError(err: unknown): boolean {
+  return err instanceof Error && /duplicate column name:/i.test(err.message);
+}
+
+let migrated = false;
+
+/**
+ * Uses raw BEGIN/COMMIT for the one-time backfill. Must NOT be called
+ * for the first time inside a Drizzle db.transaction() block — SQLite
+ * does not support nested transactions. In practice this is safe because
+ * the migration is triggered by early startup queries (listConversations,
+ * batchSetDisplayOrders) before any transaction-wrapped paths run, and
+ * the `migrated` flag makes subsequent calls no-ops.
+ */
+export function ensureGroupMigration(): void {
+  if (migrated) return;
+
+  // 1. Create groups table if not exists
+  rawExec(`
+    CREATE TABLE IF NOT EXISTS conversation_groups (
+      id TEXT PRIMARY KEY,
+      name TEXT NOT NULL,
+      sort_position REAL NOT NULL DEFAULT 0,
+      is_system_group BOOLEAN NOT NULL DEFAULT FALSE,
+      created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')),
+      updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now'))
+    )
+  `);
+
+  // 2. Add group_id column if not exists
+  // Match existing error-handling pattern from conversation-display-order-migration.ts:
+  // only swallow duplicate-column errors, log+throw everything else.
+  try {
+    rawRun(
+      "ALTER TABLE conversations ADD COLUMN group_id TEXT REFERENCES conversation_groups(id) ON DELETE SET NULL",
+    );
+  } catch (err) {
+    if (!isDuplicateColumnError(err)) {
+      log.error({ err }, "Failed to add group_id column");
+      throw err;
+    }
+  }
+
+  // 3. Seed system groups (three: pinned, scheduled, background)
+  rawExec(`
+    INSERT OR IGNORE INTO conversation_groups (id, name, sort_position, is_system_group)
+    VALUES
+      ('system:pinned', 'Pinned', 0, TRUE),
+      ('system:scheduled', 'Scheduled', 1, TRUE),
+      ('system:background', 'Background', 2, TRUE)
+  `);
+
+  // 4. One-time backfill (guard: persistent marker prevents re-running on restart)
+  //
+  // The backfill sets group_id on existing conversations based on their attributes.
+  // It must only run once — re-running would overwrite user-explicit ungrouping
+  // (e.g., user removes a conversation from a group → group_id = NULL → next restart
+  // would backfill it back into system:background). A persistent marker row in
+  // conversation_groups tracks whether backfill has already completed.
+  const backfillDone = rawGet<{ id: string }>(
+    "SELECT id FROM conversation_groups WHERE id = '_backfill_complete'",
+  );
+
+  if (!backfillDone) {
+    ensureDisplayOrderMigration(); // ensure is_pinned column exists first
+
+    // Canonical classification rules (full decision tree with precedence):
+    //
+    //   1. Pinned:     is_pinned = TRUE
+    //                  → system:pinned (always wins, checked first)
+    //
+    //   2. Scheduled:  source IN ('schedule', 'reminder') OR schedule_job_id IS NOT NULL
+    //                  → system:scheduled (checked second)
+    //
+    //   3. Background: (conversation_type = 'background' AND COALESCE(source, '') != 'notification')
+    //                  OR source IN ('heartbeat', 'task')
+    //                  AND COALESCE(source, '') NOT IN ('schedule', 'reminder')  ← safety + NULL handling
+    //                  AND schedule_job_id IS NULL                               ← safety: prevents overlap with step 2
+    //                  → system:background
+    //
+    //   4. Else:       ungrouped (group_id = NULL)
+    //
+    // Each backfill step uses AND group_id IS NULL to avoid reassignment.
+    // The exclusion guards in step 3 are belt-and-suspenders — step 2's
+    // group_id IS NULL guard already prevents overlap, but the explicit
+    // exclusions make the SQL self-documenting and migration-order-independent.
+    // COALESCE handles NULL source values (legacy rows).
+
+    // Wrap all backfill steps in a transaction so a partial failure
+    // (e.g. crash mid-backfill) doesn't leave conversations half-classified
+    // with the marker row missing, which would cause a re-run to skip
+    // already-classified rows (group_id IS NULL guard).
+    try {
+      rawExec("BEGIN");
+
+      // Step A: Pinned -> system:pinned (runs first, always wins)
+      rawExec(`
+        UPDATE conversations SET group_id = 'system:pinned'
+        WHERE is_pinned = 1 AND group_id IS NULL
+      `);
+
+      // Step B: Scheduled -> system:scheduled (schedule/reminder source or has schedule_job_id)
+      rawExec(`
+        UPDATE conversations SET group_id = 'system:scheduled'
+        WHERE (source IN ('schedule', 'reminder') OR schedule_job_id IS NOT NULL OR conversation_type = 'scheduled')
+        AND group_id IS NULL
+      `);
+
+      // Step C: Background -> system:background (background type, heartbeat, or task — excluding notifications)
+      // Explicit exclusion of schedule sources + schedule_job_id as a safety net.
+      // Step B already catches those via group_id IS NULL guard, but the explicit exclusion
+      // makes the SQL self-documenting and protects against future migration ordering changes.
+      // Note: source can be NULL for legacy rows. NULL != 'notification' evaluates to NULL (not TRUE)
+      // in SQL, so use COALESCE to treat NULL source as empty string for the exclusion checks.
+      rawExec(`
+        UPDATE conversations SET group_id = 'system:background'
+        WHERE (
+          (conversation_type = 'background' AND COALESCE(source, '') != 'notification')
+          OR source IN ('heartbeat', 'task')
+        )
+        AND COALESCE(source, '') NOT IN ('schedule', 'reminder')
+        AND schedule_job_id IS NULL
+        AND group_id IS NULL
+      `);
+
+      // Mark backfill as complete so it won't re-run on future process restarts
+      rawExec(`
+        INSERT OR IGNORE INTO conversation_groups (id, name, sort_position, is_system_group)
+        VALUES ('_backfill_complete', '_backfill_complete', -1, TRUE)
+      `);
+
+      rawExec("COMMIT");
+    } catch (err) {
+      rawExec("ROLLBACK");
+      log.error({ err }, "Group backfill transaction failed, rolled back");
+      throw err;
+    }
+  }
+
+  migrated = true;
+}

package/src/memory/conversation-queries.ts

@@ -4,6 +4,7 @@ import { getLogger } from "../util/logger.js";
 import type { ConversationRow } from "./conversation-crud.js";
 import { parseConversation } from "./conversation-crud.js";
 import { ensureDisplayOrderMigration } from "./conversation-display-order-migration.js";
+import { ensureGroupMigration } from "./conversation-group-migration.js";
 import { getDb, rawAll } from "./db.js";
 import { conversations, messages } from "./schema.js";
 
@@ -13,7 +14,22 @@ const log = getLogger("conversation-store");
  * Build an FTS5 MATCH query string from natural text by extracting tokens.
  * Used for messages_fts full-text search over conversation content.
  */
-function buildFtsMatchQuery(text: string): string | null {
+export function buildFtsMatchQuery(
+  text: string,
+  opts?: { allowFts5Syntax?: boolean },
+): string | null {
+  // If the query already contains FTS5 operators, pass it through directly
+  // so callers (e.g. the archive recall tool) can use exact-phrase, AND, OR,
+  // NOT, NEAR syntax. Only enabled when the caller explicitly opts in —
+  // user-facing search should always go through normal tokenization to avoid
+  // FTS5 boolean semantics leaking into sidebar/global search.
+  if (
+    opts?.allowFts5Syntax &&
+    /\bAND\b|\bOR\b|\bNOT\b|\bNEAR\s*\(|"[^"]+"/.test(text)
+  ) {
+    return text.trim();
+  }
+
   const tokens = text
     .toLowerCase()
     .split(/[^a-z0-9_]+/g)
@@ -21,7 +37,8 @@ function buildFtsMatchQuery(text: string): string | null {
     .filter((token) => token.length >= 2);
   if (tokens.length === 0) return null;
   const unique = [...new Set(tokens)].slice(0, 24);
-  return unique.map((token) => `"${token.replace(/"/g, '""')}"`).join(" OR ");
+  // Space-separated quoted tokens are implicit AND in FTS5.
+  return unique.map((token) => `"${token.replace(/"/g, '""')}"`).join(" ");
 }
 
 export function listConversations(
@@ -30,25 +47,52 @@ export function listConversations(
   offset = 0,
 ): ConversationRow[] {
   ensureDisplayOrderMigration();
+  ensureGroupMigration();
   const db = getDb();
   const where = backgroundOnly
-    ? sql`${conversations.conversationType} = 'background'`
-    : sql`${conversations.conversationType} NOT IN ('background', 'private')`;
+    ? sql`${conversations.conversationType} IN ('background', 'scheduled') AND (${conversations.source} IS NULL OR ${conversations.source} != 'subagent')`
+    : sql`${conversations.conversationType} NOT IN ('background', 'private', 'scheduled')`;
   const query = db
     .select()
     .from(conversations)
     .where(where)
-    .orderBy(desc(conversations.updatedAt))
+    .orderBy(
+      desc(
+        sql`COALESCE(${conversations.lastMessageAt}, ${conversations.updatedAt})`,
+      ),
+    )
     .limit(limit ?? 100)
     .offset(offset);
   return query.all().map(parseConversation);
 }
 
+export function listPinnedConversations(): ConversationRow[] {
+  ensureDisplayOrderMigration();
+  ensureGroupMigration();
+  const db = getDb();
+  const query = db
+    .select()
+    .from(conversations)
+    .where(
+      and(
+        sql`${conversations.conversationType} NOT IN ('background', 'private', 'scheduled')`,
+        sql`is_pinned = 1`,
+      ),
+    )
+    .orderBy(
+      sql`COALESCE(display_order, 999999) ASC`,
+      desc(
+        sql`COALESCE(${conversations.lastMessageAt}, ${conversations.updatedAt})`,
+      ),
+    );
+  return query.all().map(parseConversation);
+}
+
 export function countConversations(backgroundOnly = false): number {
   const db = getDb();
   const where = backgroundOnly
-    ? sql`${conversations.conversationType} = 'background'`
-    : sql`${conversations.conversationType} NOT IN ('background', 'private')`;
+    ? sql`${conversations.conversationType} IN ('background', 'scheduled') AND (${conversations.source} IS NULL OR ${conversations.source} != 'subagent')`
+    : sql`${conversations.conversationType} NOT IN ('background', 'private', 'scheduled')`;
   const [{ total }] = db
     .select({ total: count() })
     .from(conversations)
@@ -63,9 +107,13 @@ export function getLatestConversation(): ConversationRow | null {
     .select()
     .from(conversations)
     .where(
-      sql`${conversations.conversationType} NOT IN ('background', 'private')`,
+      sql`${conversations.conversationType} NOT IN ('background', 'private', 'scheduled')`,
+    )
+    .orderBy(
+      desc(
+        sql`COALESCE(${conversations.lastMessageAt}, ${conversations.updatedAt})`,
+      ),
     )
-    .orderBy(desc(conversations.updatedAt))
     .limit(1)
     .get();
   return row ? parseConversation(row) : null;
@@ -170,7 +218,7 @@ export function searchConversations(
         FROM messages_fts f
         JOIN messages m ON m.id = f.message_id
         JOIN conversations c ON c.id = m.conversation_id
-        WHERE messages_fts MATCH ? AND c.conversation_type NOT IN ('background', 'private')
+        WHERE messages_fts MATCH ? AND c.conversation_type NOT IN ('background', 'private', 'scheduled')
         LIMIT 1000
       `,
         ftsMatch,
@@ -195,7 +243,7 @@ export function searchConversations(
       SELECT DISTINCT m.conversation_id
       FROM messages m
       JOIN conversations c ON c.id = m.conversation_id
-      WHERE m.content LIKE ? ESCAPE '\\' AND c.conversation_type NOT IN ('background', 'private')
+      WHERE m.content LIKE ? ESCAPE '\\' AND c.conversation_type NOT IN ('background', 'private', 'scheduled')
       LIMIT 1000
     `,
       likePattern,
@@ -209,7 +257,7 @@ export function searchConversations(
     .from(conversations)
     .where(
       and(
-        sql`${conversations.conversationType} NOT IN ('background', 'private')`,
+        sql`${conversations.conversationType} NOT IN ('background', 'private', 'scheduled')`,
         sql`${conversations.title} LIKE ${titlePattern} ESCAPE '\\'`,
       ),
     )
@@ -309,7 +357,7 @@ export function searchConversations(
  * occurrence of `query`. The content may be JSON (content blocks) or plain
  * text; we extract a readable snippet in either case.
  */
-function buildExcerpt(rawContent: string, query: string): string {
+export function buildExcerpt(rawContent: string, query: string): string {
   // Try to extract plain text from JSON content blocks first.
   let text = rawContent;
   try {

package/src/memory/conversation-title-service.ts

@@ -34,6 +34,7 @@ export type TitleOrigin =
   | "subagent"
   | "sequence"
   | "heartbeat"
+  | "filing"
   | "local"
   | "task_submit"
   | "misc";