@vellumai/assistant 0.5.16 → 0.6.0

package/src/memory/conversation-crud.ts

@@ -41,17 +41,15 @@ import {
   updateMetaFile,
 } from "./conversation-disk-view.js";
 import { ensureDisplayOrderMigration } from "./conversation-display-order-migration.js";
-import { getDb, rawAll, rawExec, rawGet, rawRun } from "./db.js";
+import { ensureGroupMigration } from "./conversation-group-migration.js";
+import { getDb, rawExec, rawGet, rawRun } from "./db.js";
 import { indexMessageNow } from "./indexer.js";
-import { enqueueMemoryJob } from "./jobs-store.js";
 import {
   channelInboundEvents,
   conversations,
   conversationStarters,
   llmRequestLogs,
   memoryEmbeddings,
-  memoryItems,
-  memoryItemSources,
   memorySegments,
   memorySummaries,
   messageAttachments,
@@ -110,7 +108,7 @@ export type MessageMetadata = z.infer<typeof messageMetadataSchema>;
 
 function cloneForkMessageMetadata(
   metadata: string | null,
-  sourceMessageId: string,
+  sourceMessageId: string
 ): string {
   if (!metadata) {
     return JSON.stringify({ forkSourceMessageId: sourceMessageId });
@@ -143,7 +141,7 @@ function cloneForkMessageMetadata(
  * callers with actual guardian trust should always supply a real context.
  */
 export function provenanceFromTrustContext(
-  ctx: TrustContext | null | undefined,
+  ctx: TrustContext | null | undefined
 ): Record<string, unknown> {
   if (!ctx) return { provenanceTrustClass: "unknown" };
   return {
@@ -244,19 +242,27 @@ export function createConversation(
         conversationType?: "standard" | "private" | "background";
         source?: string;
         scheduleJobId?: string;
-      },
+        groupId?: string;
+      }
 ) {
   const db = getDb();
   const now = Date.now();
   const opts =
     typeof titleOrOpts === "string"
       ? { title: titleOrOpts }
-      : (titleOrOpts ?? {});
+      : titleOrOpts ?? {};
   const conversationType = opts.conversationType ?? "standard";
   const source = opts.source ?? "user";
+  const groupId = opts.groupId;
   const id = uuid();
   const memoryScopeId =
     conversationType === "private" ? `private:${id}` : "default";
+
+  // Ensure group_id column exists for deterministic schema readiness,
+  // even when this conversation has no groupId (a subsequent query or
+  // reorder may reference the column).
+  ensureGroupMigration();
+
   const conversation = {
     id,
     title: opts.title ?? null,
@@ -276,6 +282,12 @@ export function createConversation(
 
   // Retry on SQLITE_BUSY and SQLITE_IOERR — transient disk I/O errors or WAL
   // contention can cause the first attempt to fail even under normal load.
+  // INSERT and group_id UPDATE are retried independently so a transient failure
+  // on the UPDATE doesn't re-execute the already-succeeded INSERT (which would
+  // hit a unique constraint violation).
+  // No explicit BEGIN/COMMIT here — callers that need atomicity (e.g.
+  // forkConversation) wrap in their own transaction, and nesting raw BEGIN
+  // inside Drizzle's db.transaction() would crash SQLite.
   const MAX_RETRIES = 3;
   for (let attempt = 0; ; attempt++) {
     try {
@@ -289,10 +301,8 @@ export function createConversation(
       ) {
         log.warn(
           { attempt, conversationId: id, code },
-          "createConversation: transient SQLite error, retrying",
+          "createConversation: INSERT transient error, retrying"
         );
-        // Synchronous sleep — createConversation is synchronous and the
-        // retry window is short (50-150ms), so Bun.sleepSync is appropriate.
         Bun.sleepSync(50 * (attempt + 1));
         continue;
       }
@@ -300,6 +310,35 @@ export function createConversation(
     }
   }
 
+  // group_id is NOT in the Drizzle schema (raw-query-only pattern).
+  // Set via raw SQL after the INSERT succeeds.
+  if (groupId) {
+    for (let attempt = 0; ; attempt++) {
+      try {
+        rawRun(
+          "UPDATE conversations SET group_id = ? WHERE id = ?",
+          groupId,
+          id
+        );
+        break;
+      } catch (err) {
+        const code = (err as { code?: string }).code ?? "";
+        if (
+          attempt < MAX_RETRIES &&
+          (code.startsWith("SQLITE_BUSY") || code.startsWith("SQLITE_IOERR"))
+        ) {
+          log.warn(
+            { attempt, conversationId: id, code },
+            "createConversation: group_id UPDATE transient error, retrying"
+          );
+          Bun.sleepSync(50 * (attempt + 1));
+          continue;
+        }
+        throw err;
+      }
+    }
+  }
+
   initConversationDir({ ...conversation, originChannel: null });
 
   return conversation;
@@ -321,18 +360,18 @@ export function getConversation(id: string): ConversationRow | null {
  * (i.e. no other conversations still reference it).
  */
 export function countConversationsByScheduleJobId(
-  scheduleJobId: string,
+  scheduleJobId: string
 ): number {
   return (
     rawGet<{ c: number }>(
       "SELECT COUNT(*) AS c FROM conversations WHERE schedule_job_id = ?",
-      scheduleJobId,
+      scheduleJobId
     )?.c ?? 0
   );
 }
 
 export function getConversationType(
-  conversationId: string,
+  conversationId: string
 ): "standard" | "private" {
   const conv = getConversation(conversationId);
   const raw = conv?.conversationType;
@@ -344,6 +383,20 @@ export function getConversationMemoryScopeId(conversationId: string): string {
   return conv?.memoryScopeId ?? "default";
 }
 
+/**
+ * Fetch group_id for a conversation via raw SQL. group_id is NOT in the
+ * Drizzle schema (raw-query-only pattern), so ConversationRow doesn't
+ * include it. This helper is used by forkConversation to inherit group_id.
+ */
+export function getConversationGroupId(conversationId: string): string | null {
+  ensureGroupMigration();
+  const row = rawGet<{ group_id: string | null }>(
+    "SELECT group_id FROM conversations WHERE id = ?",
+    conversationId
+  );
+  return row?.group_id ?? null;
+}
+
 export function forkConversation(params: {
   conversationId: string;
   throughMessageId?: string;
@@ -363,7 +416,7 @@ export function forkConversation(params: {
 
   if (sourceMessages.length === 0) {
     throw new UserError(
-      `Conversation ${conversationId} has no persisted messages to fork`,
+      `Conversation ${conversationId} has no persisted messages to fork`
     );
   }
 
@@ -374,7 +427,7 @@ export function forkConversation(params: {
 
   if (throughMessageId != null && copyBoundaryIndex === -1) {
     throw new UserError(
-      `Message ${throughMessageId} does not belong to conversation ${conversationId}`,
+      `Message ${throughMessageId} does not belong to conversation ${conversationId}`
     );
   }
 
@@ -382,8 +435,8 @@ export function forkConversation(params: {
     0,
     Math.min(
       sourceConversation.contextCompactedMessageCount,
-      sourceMessages.length,
-    ),
+      sourceMessages.length
+    )
   );
   const preserveSourceCompactionState =
     copyBoundaryIndex >= visibleWindowStartIndex;
@@ -407,10 +460,14 @@ export function forkConversation(params: {
   // (linkAttachmentToMessage, relinkAttachments, seedForkedConversationAttention)
   // use the same underlying bun:sqlite connection, so their writes participate
   // in this transaction automatically.
+  // Inherit group_id from parent via raw SQL helper (group_id is not in Drizzle schema)
+  const parentGroupId = getConversationGroupId(conversationId);
+
   const forkedConversation = db.transaction(() => {
     const fc = createConversation({
       title: forkTitle,
       conversationType: "standard",
+      groupId: parentGroupId ?? undefined,
     });
 
     db.update(conversations)
@@ -473,7 +530,7 @@ export function forkConversation(params: {
         .orderBy(messageAttachments.position)
         .all();
       const uncachedAttachmentLinks = attachmentLinks.filter(
-        (link) => !attachmentIdMap.has(link.attachmentId),
+        (link) => !attachmentIdMap.has(link.attachmentId)
       );
       const stagingMessageId =
         uncachedAttachmentLinks.length > 0 ? uuid() : null;
@@ -509,7 +566,7 @@ export function forkConversation(params: {
         const scopedAttachmentId = linkAttachmentToMessage(
           stagingMessageId ?? forkedMessageId,
           link.attachmentId,
-          link.position,
+          link.position
         );
         attachmentIdMap.set(link.attachmentId, scopedAttachmentId);
       }
@@ -544,7 +601,7 @@ export function forkConversation(params: {
   const persistedFork = getConversation(forkedConversation.id);
   if (!persistedFork) {
     throw new Error(
-      `Failed to load forked conversation ${forkedConversation.id} after creation`,
+      `Failed to load forked conversation ${forkedConversation.id} after creation`
     );
   }
 
@@ -589,16 +646,6 @@ export function deleteConversation(id: string): DeletedMemoryIds {
         .all();
       result.segmentIds = linkedSegments.map((r) => r.id);
 
-      // Collect memory item IDs linked to these messages before cascade.
-      const linkedItems = tx
-        .select({ memoryItemId: memoryItemSources.memoryItemId })
-        .from(memoryItemSources)
-        .where(inArray(memoryItemSources.messageId, messageIds))
-        .all();
-      const candidateItemIds = [
-        ...new Set(linkedItems.map((r) => r.memoryItemId)),
-      ];
-
       // Delete non-cascading tables first.
       tx.delete(llmRequestLogs)
         .where(eq(llmRequestLogs.conversationId, id))
@@ -606,7 +653,7 @@ export function deleteConversation(id: string): DeletedMemoryIds {
       tx.delete(toolInvocations)
         .where(eq(toolInvocations.conversationId, id))
         .run();
-      // Cascade deletes memory_segments, memory_item_sources, message_attachments.
+      // Cascade deletes memory_segments, message_attachments.
       tx.delete(messages).where(eq(messages.conversationId, id)).run();
 
       // Clean up segment embeddings.
@@ -615,39 +662,11 @@ export function deleteConversation(id: string): DeletedMemoryIds {
           .where(
             and(
               eq(memoryEmbeddings.targetType, "segment"),
-              inArray(memoryEmbeddings.targetId, result.segmentIds),
-            ),
+              inArray(memoryEmbeddings.targetId, result.segmentIds)
+            )
           )
           .run();
       }
-
-      // Clean up orphaned memory items whose only sources were in this conversation.
-      if (candidateItemIds.length > 0) {
-        const surviving = tx
-          .select({ memoryItemId: memoryItemSources.memoryItemId })
-          .from(memoryItemSources)
-          .where(inArray(memoryItemSources.memoryItemId, candidateItemIds))
-          .all();
-        const survivingIds = new Set(surviving.map((r) => r.memoryItemId));
-        const orphanedIds = candidateItemIds.filter(
-          (itemId) => !survivingIds.has(itemId),
-        );
-        result.orphanedItemIds = orphanedIds;
-
-        if (orphanedIds.length > 0) {
-          tx.delete(memoryEmbeddings)
-            .where(
-              and(
-                eq(memoryEmbeddings.targetType, "item"),
-                inArray(memoryEmbeddings.targetId, orphanedIds),
-              ),
-            )
-            .run();
-          tx.delete(memoryItems)
-            .where(inArray(memoryItems.id, orphanedIds))
-            .run();
-        }
-      }
     } else {
       // No messages — just clean up non-message tables.
       tx.delete(llmRequestLogs)
@@ -659,35 +678,6 @@ export function deleteConversation(id: string): DeletedMemoryIds {
     }
 
     if (isPrivateScope && memoryScopeId) {
-      // Sweep remaining memory items with this private scopeId.
-      const scopeItems = tx
-        .select({ id: memoryItems.id })
-        .from(memoryItems)
-        .where(eq(memoryItems.scopeId, memoryScopeId))
-        .all();
-      const alreadyDeleted = new Set(result.orphanedItemIds);
-      const scopeItemIds = scopeItems
-        .map((r) => r.id)
-        .filter((id) => !alreadyDeleted.has(id));
-
-      if (scopeItemIds.length > 0) {
-        tx.delete(memoryEmbeddings)
-          .where(
-            and(
-              eq(memoryEmbeddings.targetType, "item"),
-              inArray(memoryEmbeddings.targetId, scopeItemIds),
-            ),
-          )
-          .run();
-        tx.delete(memoryItemSources)
-          .where(inArray(memoryItemSources.memoryItemId, scopeItemIds))
-          .run();
-        tx.delete(memoryItems)
-          .where(inArray(memoryItems.id, scopeItemIds))
-          .run();
-        result.orphanedItemIds.push(...scopeItemIds);
-      }
-
       // Sweep memory summaries with this private scopeId.
       const scopeSummaries = tx
         .select({ id: memorySummaries.id })
@@ -701,8 +691,8 @@ export function deleteConversation(id: string): DeletedMemoryIds {
           .where(
             and(
               eq(memoryEmbeddings.targetType, "summary"),
-              inArray(memoryEmbeddings.targetId, scopeSummaryIds),
-            ),
+              inArray(memoryEmbeddings.targetId, scopeSummaryIds)
+            )
           )
           .run();
         tx.delete(memorySummaries)
@@ -747,65 +737,6 @@ export function wipeConversation(id: string): WipeConversationResult {
   // the cancellation queries join on `messages`).
   const cancelledJobCount = cancelPendingJobsForConversation(id);
 
-  // Step B — Un-supersede memory items with explicit `supersededBy` links.
-  // Find memory items whose `superseded_by` points to an item sourced
-  // exclusively from this conversation.
-  const explicitSuperseded = rawAll<{ oldItemId: string }>(
-    `SELECT DISTINCT mi_old.id AS oldItemId
-     FROM memory_items mi_old
-     JOIN memory_items mi_new ON mi_old.superseded_by = mi_new.id
-     WHERE mi_old.status = 'superseded'
-       AND mi_new.id IN (
-         SELECT mis.memory_item_id
-         FROM memory_item_sources mis
-         JOIN messages m ON m.id = mis.message_id
-         WHERE m.conversation_id = ?
-       )
-       AND NOT EXISTS (
-         SELECT 1 FROM memory_item_sources mis2
-         JOIN messages m2 ON m2.id = mis2.message_id
-         WHERE mis2.memory_item_id = mi_new.id
-           AND m2.conversation_id != ?
-       )
-       AND NOT EXISTS (
-         SELECT 1 FROM memory_items mi_active
-         WHERE mi_active.kind = mi_old.kind
-           AND mi_active.subject = mi_old.subject
-           AND mi_active.scope_id = mi_old.scope_id
-           AND mi_active.status = 'active'
-           AND mi_active.id != mi_old.id
-           -- Exclude items sourced exclusively from the conversation being
-           -- wiped — deleteConversation will remove them, so they should not
-           -- block restoration of mi_old.
-           AND NOT (
-             EXISTS (
-               SELECT 1 FROM memory_item_sources mis_a
-               JOIN messages m_a ON m_a.id = mis_a.message_id
-               WHERE mis_a.memory_item_id = mi_active.id
-                 AND m_a.conversation_id = ?
-             )
-             AND NOT EXISTS (
-               SELECT 1 FROM memory_item_sources mis_b
-               JOIN messages m_b ON m_b.id = mis_b.message_id
-               WHERE mis_b.memory_item_id = mi_active.id
-                 AND m_b.conversation_id != ?
-             )
-           )
-       )`,
-    id,
-    id,
-    id,
-    id,
-  );
-  for (const { oldItemId } of explicitSuperseded) {
-    rawRun(
-      "UPDATE memory_items SET status = 'active', superseded_by = NULL WHERE id = ?",
-      oldItemId,
-    );
-    enqueueMemoryJob("embed_item", { itemId: oldItemId });
-    unsupersededItemIds.push(oldItemId);
-  }
-
   // Step C — Delete conversation-scoped memory summaries and their embeddings.
   const summaryRows = db
     .select({ id: memorySummaries.id })
@@ -813,8 +744,8 @@ export function wipeConversation(id: string): WipeConversationResult {
     .where(
       and(
         eq(memorySummaries.scope, "conversation"),
-        eq(memorySummaries.scopeKey, id),
-      ),
+        eq(memorySummaries.scopeKey, id)
+      )
     )
     .all();
   const summaryIds = summaryRows.map((r) => r.id);
@@ -823,8 +754,8 @@ export function wipeConversation(id: string): WipeConversationResult {
       .where(
         and(
           eq(memoryEmbeddings.targetType, "summary"),
-          inArray(memoryEmbeddings.targetId, summaryIds),
-        ),
+          inArray(memoryEmbeddings.targetId, summaryIds)
+        )
       )
       .run();
     db.delete(memorySummaries)
@@ -833,79 +764,12 @@ export function wipeConversation(id: string): WipeConversationResult {
   }
   deletedSummaryIds.push(...summaryIds);
 
-  // Step D — Get the conversation's memoryScopeId before deletion.
-  const scopeId = getConversationMemoryScopeId(id);
-
-  // Step D.5 — Collect kind + subject pairs of items that will be orphaned
-  // by deleteConversation. These are items sourced from this conversation's
-  // messages that have NO sources from any other conversation. We need this
-  // before deletion so we can scope Step F to only restore superseded items
-  // matching the specific kind + subject pairs that just lost their active
-  // replacement.
-  const orphanedKindSubjects = rawAll<{ kind: string; subject: string }>(
-    `SELECT DISTINCT mi.kind, mi.subject
-     FROM memory_items mi
-     JOIN memory_item_sources mis ON mis.memory_item_id = mi.id
-     JOIN messages m ON m.id = mis.message_id
-     WHERE m.conversation_id = ?
-       AND NOT EXISTS (
-         SELECT 1 FROM memory_item_sources mis2
-         JOIN messages m2 ON m2.id = mis2.message_id
-         WHERE mis2.memory_item_id = mi.id
-           AND m2.conversation_id != ?
-       )`,
-    id,
-    id,
-  );
-
-  // Step E — Delegate to deleteConversation which handles messages (cascade
-  // segments, item_sources, attachments), llmRequestLogs, toolInvocations,
-  // orphaned memory items + embeddings, and the conversation row.
+  // Step D — Delegate to deleteConversation which handles messages (cascade
+  // segments, attachments), llmRequestLogs, toolInvocations,
+  // embeddings, and the conversation row.
   const deletedMemoryIds = deleteConversation(id);
 
-  // Step F — Restore orphaned subject-match superseded items. After
-  // deleteConversation removes superseding items, find superseded items
-  // with no supersededBy link where no active item with the same
-  // kind + subject + scope_id exists. Scoped to only the kind + subject
-  // pairs of items that were just orphaned by deleteConversation, so we
-  // don't accidentally restore items superseded by unrelated conversations.
-  let orphanedSuperseded: Array<{ id: string }> = [];
-  if (orphanedKindSubjects.length > 0) {
-    const placeholders = orphanedKindSubjects.map(() => "(?, ?)").join(", ");
-    const params: Array<string> = [scopeId];
-    for (const { kind, subject } of orphanedKindSubjects) {
-      params.push(kind, subject);
-    }
-    orphanedSuperseded = rawAll<{ id: string }>(
-      `SELECT id FROM (
-         SELECT id, ROW_NUMBER() OVER (
-           PARTITION BY kind, subject, scope_id
-           ORDER BY last_seen_at DESC
-         ) AS rn
-         FROM memory_items
-         WHERE status = 'superseded'
-           AND superseded_by IS NULL
-           AND scope_id = ?
-           AND (kind, subject) IN (VALUES ${placeholders})
-           AND NOT EXISTS (
-             SELECT 1 FROM memory_items mi2
-             WHERE mi2.kind = memory_items.kind
-               AND mi2.subject = memory_items.subject
-               AND mi2.scope_id = memory_items.scope_id
-               AND mi2.status = 'active'
-               AND mi2.id != memory_items.id
-           )
-       ) WHERE rn = 1`,
-      ...params,
-    );
-  }
-  for (const { id: itemId } of orphanedSuperseded) {
-    rawRun("UPDATE memory_items SET status = 'active' WHERE id = ?", itemId);
-    enqueueMemoryJob("embed_item", { itemId });
-    unsupersededItemIds.push(itemId);
-  }
-
-  // Step G — Return the combined result.
+  // Step E — Return the combined result.
   return {
     ...deletedMemoryIds,
     unsupersededItemIds,
@@ -970,7 +834,7 @@ export async function addMessage(
   role: string,
   content: string,
   metadata?: Record<string, unknown>,
-  opts?: { skipIndexing?: boolean },
+  opts?: { skipIndexing?: boolean }
 ) {
   const db = getDb();
   const messageId = uuid();
@@ -980,7 +844,7 @@ export async function addMessage(
     if (!result.success) {
       log.warn(
         { conversationId, messageId, issues: result.error.issues },
-        "Invalid message metadata, storing as-is",
+        "Invalid message metadata, storing as-is"
       );
     }
   }
@@ -1015,8 +879,8 @@ export async function addMessage(
             .where(
               and(
                 eq(conversations.id, conversationId),
-                isNull(conversations.originChannel),
-              ),
+                isNull(conversations.originChannel)
+              )
             )
             .run();
         }
@@ -1035,7 +899,7 @@ export async function addMessage(
       ) {
         log.warn(
           { attempt, conversationId, code: errCode },
-          "addMessage: transient SQLite error, retrying",
+          "addMessage: transient SQLite error, retrying"
         );
         await Bun.sleep(50 * (attempt + 1));
         continue;
@@ -1074,12 +938,12 @@ export async function addMessage(
           provenanceTrustClass,
           automated,
         },
-        config.memory,
+        config.memory
       );
     } catch (err) {
       log.warn(
         { err, conversationId, messageId: message.id },
-        "Failed to index message for memory",
+        "Failed to index message for memory"
       );
     }
   }
@@ -1094,7 +958,7 @@ export async function addMessage(
     } catch (err) {
       log.warn(
         { err, conversationId, messageId: message.id },
-        "Failed to project assistant message for attention tracking",
+        "Failed to project assistant message for attention tracking"
       );
     }
   }
@@ -1121,7 +985,7 @@ export interface PaginatedMessagesResult {
 export function getMessagesPaginated(
   conversationId: string,
   limit: number | undefined,
-  beforeTimestamp?: number,
+  beforeTimestamp?: number
 ): PaginatedMessagesResult {
   const db = getDb();
 
@@ -1165,7 +1029,7 @@ export function getMessagesPaginated(
 
 export function getLastAssistantTimestampBefore(
   conversationId: string,
-  beforeTimestamp: number,
+  beforeTimestamp: number
 ): number {
   const db = getDb();
   const row = db
@@ -1175,8 +1039,8 @@ export function getLastAssistantTimestampBefore(
       and(
         eq(messages.conversationId, conversationId),
         eq(messages.role, "assistant"),
-        lt(messages.createdAt, beforeTimestamp),
-      ),
+        lt(messages.createdAt, beforeTimestamp)
+      )
     )
     .orderBy(desc(messages.createdAt))
     .limit(1)
@@ -1187,7 +1051,7 @@ export function getLastAssistantTimestampBefore(
 /** Fetch a single message by ID, optionally scoped to a specific conversation. */
 export function getMessageById(
   messageId: string,
-  conversationId?: string,
+  conversationId?: string
 ): MessageRow | null {
   const db = getDb();
   const conditions = [eq(messages.id, messageId)];
@@ -1205,7 +1069,7 @@ export function getMessageById(
 export function updateConversationTitle(
   id: string,
   title: string,
-  isAutoTitle?: number,
+  isAutoTitle?: number
 ): void {
   const db = getDb();
   const set: Record<string, unknown> = { title, updatedAt: Date.now() };
@@ -1223,7 +1087,7 @@ export function updateConversationUsage(
   id: string,
   totalInputTokens: number,
   totalOutputTokens: number,
-  totalEstimatedCost: number,
+  totalEstimatedCost: number
 ): void {
   const db = getDb();
   db.update(conversations)
@@ -1240,7 +1104,7 @@ export function updateConversationUsage(
 export function updateConversationContextWindow(
   id: string,
   contextSummary: string,
-  contextCompactedMessageCount: number,
+  contextCompactedMessageCount: number
 ): void {
   const db = getDb();
   db.update(conversations)
@@ -1265,18 +1129,16 @@ export function clearAll(): { conversations: number; messages: number } {
   const convCount =
     rawGet<{ c: number }>("SELECT COUNT(*) AS c FROM conversations")?.c ?? 0;
 
-  // Delete in dependency order. Cascades handle memory_segments,
-  // memory_item_sources, and tool_invocations, but we explicitly
-  // clear non-cascading memory tables too.
+  // Delete in dependency order. Cascades handle memory_segments and
+  // tool_invocations, but we explicitly clear non-cascading memory
+  // tables too.
   //
   // FTS virtual tables are cleared before their base tables. If an FTS
   // table is corrupted, the DELETE will fail — we drop the associated
   // triggers so that the subsequent base-table DELETEs don't also fail
   // (SQLite triggers are atomic with the triggering statement, so a
   // corrupted FTS table would roll back every base-table DELETE).
-  rawExec("DELETE FROM memory_item_sources");
   rawExec("DELETE FROM memory_segments");
-  rawExec("DELETE FROM memory_items");
   rawExec("DELETE FROM memory_summaries");
   rawExec("DELETE FROM memory_embeddings");
   rawExec("DELETE FROM memory_jobs");
@@ -1292,7 +1154,7 @@ export function clearAll(): { conversations: number; messages: number } {
   } catch (err) {
     log.warn(
       { err },
-      "clearAll: failed to clear messages_fts — dropping triggers so base-table cleanup can proceed",
+      "clearAll: failed to clear messages_fts — dropping triggers so base-table cleanup can proceed"
     );
     rawExec("DROP TRIGGER IF EXISTS messages_fts_ai");
     rawExec("DROP TRIGGER IF EXISTS messages_fts_ad");
@@ -1308,7 +1170,7 @@ export function clearAll(): { conversations: number; messages: number } {
     `INSERT INTO lifecycle_events (id, event_name, created_at) VALUES (?, ?, ?)`,
     uuid(),
     "conversations_clear_all",
-    Date.now(),
+    Date.now()
   );
 
   // Rebuild corrupted FTS tables and restore triggers after all base-table
@@ -1318,16 +1180,16 @@ export function clearAll(): { conversations: number; messages: number } {
   if (messagesFtsCorrupted) {
     rawExec("DROP TABLE IF EXISTS messages_fts");
     rawExec(
-      `CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(message_id UNINDEXED, content)`,
+      `CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(message_id UNINDEXED, content)`
     );
     rawExec(
-      `CREATE TRIGGER IF NOT EXISTS messages_fts_ai AFTER INSERT ON messages BEGIN INSERT INTO messages_fts(message_id, content) VALUES (new.id, new.content); END`,
+      `CREATE TRIGGER IF NOT EXISTS messages_fts_ai AFTER INSERT ON messages BEGIN INSERT INTO messages_fts(message_id, content) VALUES (new.id, new.content); END`
     );
     rawExec(
-      `CREATE TRIGGER IF NOT EXISTS messages_fts_ad AFTER DELETE ON messages BEGIN DELETE FROM messages_fts WHERE message_id = old.id; END`,
+      `CREATE TRIGGER IF NOT EXISTS messages_fts_ad AFTER DELETE ON messages BEGIN DELETE FROM messages_fts WHERE message_id = old.id; END`
     );
     rawExec(
-      `CREATE TRIGGER IF NOT EXISTS messages_fts_au AFTER UPDATE ON messages BEGIN DELETE FROM messages_fts WHERE message_id = old.id; INSERT INTO messages_fts(message_id, content) VALUES (new.id, new.content); END`,
+      `CREATE TRIGGER IF NOT EXISTS messages_fts_au AFTER UPDATE ON messages BEGIN DELETE FROM messages_fts WHERE message_id = old.id; INSERT INTO messages_fts(message_id, content) VALUES (new.id, new.content); END`
     );
   }
 
@@ -1352,8 +1214,8 @@ export function deleteLastExchange(conversationId: string): number {
     .where(
       and(
         eq(messages.conversationId, conversationId),
-        eq(messages.role, "user"),
-      ),
+        eq(messages.role, "user")
+      )
     )
     .orderBy(sql`rowid DESC`)
     .limit(1)
@@ -1367,7 +1229,7 @@ export function deleteLastExchange(conversationId: string): number {
   const rowidSubquery = sql`(SELECT rowid FROM messages WHERE id = ${lastUserMsg.id})`;
   const condition = and(
     eq(messages.conversationId, conversationId),
-    sql`rowid >= ${rowidSubquery}`,
+    sql`rowid >= ${rowidSubquery}`
   );
 
   const [{ deleted }] = db
@@ -1431,7 +1293,7 @@ export interface WipeConversationResult extends DeletedMemoryIds {
  */
 export function updateMessageContent(
   messageId: string,
-  newContent: string,
+  newContent: string
 ): void {
   const db = getDb();
   db.update(messages)
@@ -1440,6 +1302,27 @@ export function updateMessageContent(
     .run();
 }
 
+/**
+ * Merge `updates` into the metadata JSON of an existing message.
+ * Reads the current metadata, shallow-merges the new fields, and writes back.
+ */
+export function updateMessageMetadata(
+  messageId: string,
+  updates: Record<string, unknown>,
+): void {
+  const db = getDb();
+  const row = db
+    .select({ metadata: messages.metadata })
+    .from(messages)
+    .where(eq(messages.id, messageId))
+    .get();
+  const existing = row?.metadata ? JSON.parse(row.metadata) : {};
+  db.update(messages)
+    .set({ metadata: JSON.stringify({ ...existing, ...updates }) })
+    .where(eq(messages.id, messageId))
+    .run();
+}
+
 /**
  * Re-link all attachments from a set of source messages to a target message.
  * Used during message consolidation so that attachments linked to deleted
@@ -1447,7 +1330,7 @@ export function updateMessageContent(
  */
 export function relinkAttachments(
   fromMessageIds: string[],
-  toMessageId: string,
+  toMessageId: string
 ): number {
   if (fromMessageIds.length === 0) return 0;
   const db = getDb();
@@ -1475,15 +1358,8 @@ export function relinkAttachments(
  * NULL before the message row is removed, so associated run and event
  * records survive.
  *
- * Also cleans up derived memory_items: if the memory worker has already
- * processed an extract_items job for this message, deleting the message
- * cascades memory_item_sources but leaves the memory_items active.
- * Without cleanup, those items would leak into summaries and recall.
- * We delete any memory_items that become orphaned (no remaining sources)
- * after this message is removed.
- *
- * Returns segment and orphaned item IDs so the caller can clean up the
- * corresponding Qdrant vector entries.
+ * Returns segment IDs so the caller can clean up the corresponding
+ * Qdrant vector entries.
  */
 export function deleteMessageById(messageId: string): DeletedMemoryIds {
   const db = getDb();
@@ -1512,22 +1388,14 @@ export function deleteMessageById(messageId: string): DeletedMemoryIds {
       .all();
     result.segmentIds = linkedSegments.map((r) => r.id);
 
-    // Collect memory item IDs linked to this message before cascade.
-    const linkedItems = tx
-      .select({ memoryItemId: memoryItemSources.memoryItemId })
-      .from(memoryItemSources)
-      .where(eq(memoryItemSources.messageId, messageId))
-      .all();
-    const candidateItemIds = linkedItems.map((r) => r.memoryItemId);
-
     // Detach nullable FK references so the cascade doesn't destroy them.
     tx.update(channelInboundEvents)
       .set({ messageId: null })
       .where(eq(channelInboundEvents.messageId, messageId))
       .run();
 
-    // Now safe to delete — NOT NULL cascades remove memory_item_sources,
-    // memory_segments, and message_attachments.
+    // Now safe to delete — NOT NULL cascades remove memory_segments
+    // and message_attachments.
     tx.delete(messages).where(eq(messages.id, messageId)).run();
 
     // Clean up segment embeddings from SQLite (Qdrant cleanup is the caller's job).
@@ -1536,42 +1404,11 @@ export function deleteMessageById(messageId: string): DeletedMemoryIds {
         .where(
           and(
             eq(memoryEmbeddings.targetType, "segment"),
-            inArray(memoryEmbeddings.targetId, result.segmentIds),
-          ),
+            inArray(memoryEmbeddings.targetId, result.segmentIds)
+          )
         )
         .run();
     }
-
-    // Clean up orphaned memory items whose only source was this message.
-    if (candidateItemIds.length > 0) {
-      // Find which items still have at least one remaining source.
-      const surviving = tx
-        .select({ memoryItemId: memoryItemSources.memoryItemId })
-        .from(memoryItemSources)
-        .where(inArray(memoryItemSources.memoryItemId, candidateItemIds))
-        .all();
-      const survivingIds = new Set(surviving.map((r) => r.memoryItemId));
-      const orphanedIds = candidateItemIds.filter(
-        (id) => !survivingIds.has(id),
-      );
-      result.orphanedItemIds = orphanedIds;
-
-      if (orphanedIds.length > 0) {
-        // Delete embeddings referencing these items.
-        tx.delete(memoryEmbeddings)
-          .where(
-            and(
-              eq(memoryEmbeddings.targetType, "item"),
-              inArray(memoryEmbeddings.targetId, orphanedIds),
-            ),
-          )
-          .run();
-        // Delete the orphaned memory items themselves.
-        tx.delete(memoryItems)
-          .where(inArray(memoryItems.id, orphanedIds))
-          .run();
-      }
-    }
   });
 
   deleteOrphanAttachments(candidateAttachmentIds);
@@ -1581,7 +1418,7 @@ export function deleteMessageById(messageId: string): DeletedMemoryIds {
 
 export function setConversationOriginChannelIfUnset(
   conversationId: string,
-  channel: ChannelId,
+  channel: ChannelId
 ): void {
   const db = getDb();
   db.update(conversations)
@@ -1589,14 +1426,14 @@ export function setConversationOriginChannelIfUnset(
     .where(
       and(
         eq(conversations.id, conversationId),
-        isNull(conversations.originChannel),
-      ),
+        isNull(conversations.originChannel)
+      )
     )
     .run();
 }
 
 export function getConversationOriginChannel(
-  conversationId: string,
+  conversationId: string
 ): ChannelId | null {
   const db = getDb();
   const row = db
@@ -1609,7 +1446,7 @@ export function getConversationOriginChannel(
 
 export function setConversationOriginInterfaceIfUnset(
   conversationId: string,
-  interfaceId: InterfaceId,
+  interfaceId: InterfaceId
 ): void {
   const db = getDb();
   db.update(conversations)
@@ -1617,14 +1454,14 @@ export function setConversationOriginInterfaceIfUnset(
     .where(
       and(
         eq(conversations.id, conversationId),
-        isNull(conversations.originInterface),
-      ),
+        isNull(conversations.originInterface)
+      )
     )
     .run();
 }
 
 export function getConversationOriginInterface(
-  conversationId: string,
+  conversationId: string
 ): InterfaceId | null {
   const db = getDb();
   const row = db
@@ -1644,13 +1481,13 @@ export function getConversationOriginInterface(
  * conversation itself isn't a desktop-origin private conversation).
  */
 export function getConversationRecentProvenanceTrustClass(
-  conversationId: string,
+  conversationId: string
 ): "guardian" | "trusted_contact" | "unknown" | undefined {
   const row = rawGet<{ metadata: string | null }>(
     `SELECT metadata FROM messages
      WHERE conversation_id = ? AND role = 'user' AND metadata IS NOT NULL
      ORDER BY created_at DESC LIMIT 1`,
-    conversationId,
+    conversationId
   );
   if (!row?.metadata) return undefined;
   try {
@@ -1670,18 +1507,66 @@ export function batchSetDisplayOrders(
     id: string;
     displayOrder: number | null;
     isPinned: boolean;
-  }>,
+    groupId?: string | null;
+  }>
 ): void {
   ensureDisplayOrderMigration();
+  ensureGroupMigration();
   rawExec("BEGIN");
   try {
     for (const update of updates) {
-      rawRun(
-        "UPDATE conversations SET display_order = ?, is_pinned = ? WHERE id = ?",
-        update.displayOrder,
-        update.isPinned ? 1 : 0,
-        update.id,
-      );
+      if (update.groupId !== undefined) {
+        // New client: groupId is authoritative.
+        // Derive is_pinned from groupId.
+        // Sanitize: if groupId references a deleted/unknown group, fall back
+        // to NULL to avoid FK violation that would roll back the entire batch.
+        let safeGroupId = update.groupId;
+        if (
+          safeGroupId !== null &&
+          !rawGet<{ id: string }>(
+            "SELECT id FROM conversation_groups WHERE id = ?",
+            safeGroupId
+          )
+        ) {
+          safeGroupId = null;
+        }
+        rawRun(
+          "UPDATE conversations SET display_order = ?, is_pinned = ?, group_id = ? WHERE id = ?",
+          update.displayOrder,
+          safeGroupId === "system:pinned" ? 1 : 0,
+          safeGroupId,
+          update.id
+        );
+      } else {
+        // Old client: no groupId in payload
+        // isPinned true -> set group_id = system:pinned
+        // isPinned false -> clear group_id ONLY IF currently system:pinned
+        //                   otherwise preserve existing group_id
+        if (update.isPinned) {
+          rawRun(
+            "UPDATE conversations SET display_order = ?, is_pinned = 1, group_id = 'system:pinned' WHERE id = ?",
+            update.displayOrder,
+            update.id
+          );
+        } else {
+          // Restore system group from source/conversationType when old clients
+          // unpin, instead of clearing to NULL (which would lose provenance).
+          rawRun(
+            `UPDATE conversations SET display_order = ?, is_pinned = 0,
+             group_id = CASE WHEN group_id = 'system:pinned' THEN
+               CASE
+                 WHEN source IN ('schedule', 'reminder') THEN 'system:scheduled'
+                 WHEN source IN ('heartbeat', 'task') THEN 'system:background'
+                 WHEN conversation_type = 'background' AND COALESCE(source, '') != 'notification' THEN 'system:background'
+                 ELSE NULL
+               END
+             ELSE group_id END
+             WHERE id = ?`,
+            update.displayOrder,
+            update.id
+          );
+        }
+      }
     }
     rawExec("COMMIT");
   } catch (err) {
@@ -1691,22 +1576,31 @@ export function batchSetDisplayOrders(
 }
 
 export function getDisplayMetaForConversations(
-  conversationIds: string[],
-): Map<string, { displayOrder: number | null; isPinned: boolean }> {
+  conversationIds: string[]
+): Map<
+  string,
+  { displayOrder: number | null; isPinned: boolean; groupId: string | null }
+> {
   ensureDisplayOrderMigration();
+  ensureGroupMigration();
   const result = new Map<
     string,
-    { displayOrder: number | null; isPinned: boolean }
+    { displayOrder: number | null; isPinned: boolean; groupId: string | null }
   >();
   if (conversationIds.length === 0) return result;
   for (const id of conversationIds) {
     const row = rawGet<{
       display_order: number | null;
       is_pinned: number | null;
-    }>("SELECT display_order, is_pinned FROM conversations WHERE id = ?", id);
+      group_id: string | null;
+    }>(
+      "SELECT display_order, is_pinned, group_id FROM conversations WHERE id = ?",
+      id
+    );
     result.set(id, {
       displayOrder: row?.display_order ?? null,
       isPinned: (row?.is_pinned ?? 0) === 1,
+      groupId: row?.group_id ?? null,
     });
   }
   return result;
@@ -1730,7 +1624,7 @@ function isToolResultMessage(role: string, content: string): boolean {
       (block: unknown) =>
         block != null &&
         typeof block === "object" &&
-        (block as Record<string, unknown>).type === "tool_result",
+        (block as Record<string, unknown>).type === "tool_result"
     );
   } catch {
     return false;
@@ -1750,7 +1644,7 @@ function isToolResultMessage(role: string, content: string): boolean {
  */
 export function getTurnTimeBounds(
   conversationId: string,
-  messageCreatedAt: number,
+  messageCreatedAt: number
 ): { startTime: number; endTime: number } | null {
   const db = getDb();
 
@@ -1772,8 +1666,8 @@ export function getTurnTimeBounds(
     .where(
       and(
         eq(messages.conversationId, conversationId),
-        sql`rowid <= ${rowidSubquery}`,
-      ),
+        sql`rowid <= ${rowidSubquery}`
+      )
     )
     .orderBy(sql`rowid DESC`)
     .limit(50)
@@ -1804,8 +1698,8 @@ export function getTurnTimeBounds(
     .where(
       and(
         eq(messages.conversationId, conversationId),
-        sql`rowid > ${forwardRowidSubquery}`,
-      ),
+        sql`rowid > ${forwardRowidSubquery}`
+      )
     )
     .orderBy(sql`rowid ASC`)
     .limit(50)
@@ -1846,8 +1740,8 @@ export function getTurnTimeBounds(
         and(
           eq(llmRequestLogs.conversationId, conversationId),
           gte(llmRequestLogs.createdAt, startTime),
-          lte(llmRequestLogs.createdAt, hardCeiling),
-        ),
+          lte(llmRequestLogs.createdAt, hardCeiling)
+        )
       )
       .orderBy(desc(llmRequestLogs.createdAt))
       .limit(1)
@@ -1895,8 +1789,8 @@ export function getAssistantMessageIdsInTurn(messageId: string): string[] {
     .where(
       and(
         eq(messages.conversationId, target.conversationId),
-        lte(messages.createdAt, target.createdAt),
-      ),
+        lte(messages.createdAt, target.createdAt)
+      )
     )
     .orderBy(desc(messages.createdAt))
     .limit(50)
@@ -1933,8 +1827,8 @@ export function getAssistantMessageIdsInTurn(messageId: string): string[] {
     .where(
       and(
         eq(messages.conversationId, target.conversationId),
-        gt(messages.createdAt, target.createdAt),
-      ),
+        gt(messages.createdAt, target.createdAt)
+      )
     )
     .orderBy(asc(messages.createdAt))
     .limit(50)
@@ -1970,8 +1864,8 @@ export function getAssistantMessageIdsInTurn(messageId: string): string[] {
         and(
           eq(messages.conversationId, target.conversationId),
           gt(messages.createdAt, boundaryCreatedAt),
-          lte(messages.createdAt, target.createdAt),
-        ),
+          lte(messages.createdAt, target.createdAt)
+        )
       )
       .orderBy(asc(messages.createdAt))
       .all();
@@ -1994,8 +1888,8 @@ export function getAssistantMessageIdsInTurn(messageId: string): string[] {
     .where(
       and(
         eq(messages.conversationId, target.conversationId),
-        inArray(messages.id, [...idSet]),
-      ),
+        inArray(messages.id, [...idSet])
+      )
     )
     .orderBy(asc(messages.createdAt))
     .all();