@vellumai/assistant 0.5.2 → 0.5.3

package/src/memory/conversation-crud.ts

@@ -45,6 +45,7 @@ import { enqueueMemoryJob } from "./jobs-store.js";
 import {
   channelInboundEvents,
   conversations,
+  conversationStarters,
   llmRequestLogs,
   memoryEmbeddings,
   memoryItems,
@@ -171,6 +172,9 @@ export interface ConversationRow {
   forkParentMessageId: string | null;
   isAutoTitle: number;
   scheduleJobId: string | null;
+  memoryReducedThroughMessageId: string | null;
+  memoryDirtyTailSinceMessageId: string | null;
+  memoryLastReducedAt: number | null;
 }
 
 export const parseConversation = createRowMapper<
@@ -196,6 +200,9 @@ export const parseConversation = createRowMapper<
   forkParentMessageId: "forkParentMessageId",
   isAutoTitle: "isAutoTitle",
   scheduleJobId: "scheduleJobId",
+  memoryReducedThroughMessageId: "memoryReducedThroughMessageId",
+  memoryDirtyTailSinceMessageId: "memoryDirtyTailSinceMessageId",
+  memoryLastReducedAt: "memoryLastReducedAt",
 });
 
 export interface MessageRow {
@@ -375,127 +382,153 @@ export function forkConversation(params: {
       : ([] as MessageRow[]);
   const forkParentMessageId = messagesToCopy.at(-1)?.id ?? null;
   const forkTitle = `${sourceConversation.title ?? "Untitled"} (Fork)`;
-  const forkedConversation = createConversation({
-    title: forkTitle,
-    conversationType: "standard",
-  });
 
-  db.update(conversations)
-    .set({
-      forkParentConversationId: sourceConversation.id,
-      forkParentMessageId,
-      contextSummary: preserveSourceCompactionState
-        ? sourceConversation.contextSummary
-        : null,
-      contextCompactedMessageCount: preserveSourceCompactionState
-        ? sourceConversation.contextCompactedMessageCount
-        : 0,
-      contextCompactedAt: preserveSourceCompactionState
-        ? sourceConversation.contextCompactedAt
-        : null,
-    })
-    .where(eq(conversations.id, forkedConversation.id))
-    .run();
+  // Collect disk-sync work to run after the transaction commits.
+  const diskSyncQueue: Array<{
+    conversationId: string;
+    messageId: string;
+    createdAt: number;
+  }> = [];
+
+  // Wrap all DB mutations in a single transaction so a mid-flight failure
+  // rolls back cleanly instead of leaving a partial fork. Helper functions
+  // (linkAttachmentToMessage, relinkAttachments, seedForkedConversationAttention)
+  // use the same underlying bun:sqlite connection, so their writes participate
+  // in this transaction automatically.
+  const forkedConversation = db.transaction(() => {
+    const fc = createConversation({
+      title: forkTitle,
+      conversationType: "standard",
+    });
 
-  const forkedMessageIds = new Map<string, string>();
-  let latestForkedAssistant: { messageId: string; messageAt: number } | null =
-    null;
-
-  for (const message of messagesToCopy) {
-    const forkedMessageId = uuid();
-    db.insert(messages)
-      .values({
-        id: forkedMessageId,
-        conversationId: forkedConversation.id,
-        role: message.role,
-        content: message.content,
-        createdAt: message.createdAt,
-        metadata: cloneForkMessageMetadata(message.metadata, message.id),
+    db.update(conversations)
+      .set({
+        forkParentConversationId: sourceConversation.id,
+        forkParentMessageId,
+        contextSummary: preserveSourceCompactionState
+          ? sourceConversation.contextSummary
+          : null,
+        contextCompactedMessageCount: preserveSourceCompactionState
+          ? sourceConversation.contextCompactedMessageCount
+          : 0,
+        contextCompactedAt: preserveSourceCompactionState
+          ? sourceConversation.contextCompactedAt
+          : null,
       })
+      .where(eq(conversations.id, fc.id))
       .run();
-    forkedMessageIds.set(message.id, forkedMessageId);
-
-    if (message.role === "assistant") {
-      latestForkedAssistant = {
-        messageId: forkedMessageId,
-        messageAt: message.createdAt,
-      };
-    }
-  }
 
-  const attachmentIdMap = new Map<string, string>();
-  for (const message of messagesToCopy) {
-    const forkedMessageId = forkedMessageIds.get(message.id);
-    if (!forkedMessageId) continue;
+    const forkedMessageIds = new Map<string, string>();
+    let latestForkedAssistant: {
+      messageId: string;
+      messageAt: number;
+    } | null = null;
 
-    const attachmentLinks = db
-      .select({
-        attachmentId: messageAttachments.attachmentId,
-        position: messageAttachments.position,
-      })
-      .from(messageAttachments)
-      .where(eq(messageAttachments.messageId, message.id))
-      .orderBy(messageAttachments.position)
-      .all();
-    const uncachedAttachmentLinks = attachmentLinks.filter(
-      (link) => !attachmentIdMap.has(link.attachmentId),
-    );
-    const stagingMessageId = uncachedAttachmentLinks.length > 0 ? uuid() : null;
-
-    if (stagingMessageId) {
+    for (const message of messagesToCopy) {
+      const forkedMessageId = uuid();
       db.insert(messages)
         .values({
-          id: stagingMessageId,
-          conversationId: forkedConversation.id,
+          id: forkedMessageId,
+          conversationId: fc.id,
           role: message.role,
-          content: "",
+          content: message.content,
           createdAt: message.createdAt,
-          metadata: null,
+          metadata: cloneForkMessageMetadata(message.metadata, message.id),
         })
         .run();
+      forkedMessageIds.set(message.id, forkedMessageId);
+
+      if (message.role === "assistant") {
+        latestForkedAssistant = {
+          messageId: forkedMessageId,
+          messageAt: message.createdAt,
+        };
+      }
     }
 
-    for (const link of attachmentLinks) {
-      const cachedAttachmentId = attachmentIdMap.get(link.attachmentId);
-      if (cachedAttachmentId) {
-        db.insert(messageAttachments)
+    const attachmentIdMap = new Map<string, string>();
+    for (const message of messagesToCopy) {
+      const forkedMessageId = forkedMessageIds.get(message.id);
+      if (!forkedMessageId) continue;
+
+      const attachmentLinks = db
+        .select({
+          attachmentId: messageAttachments.attachmentId,
+          position: messageAttachments.position,
+        })
+        .from(messageAttachments)
+        .where(eq(messageAttachments.messageId, message.id))
+        .orderBy(messageAttachments.position)
+        .all();
+      const uncachedAttachmentLinks = attachmentLinks.filter(
+        (link) => !attachmentIdMap.has(link.attachmentId),
+      );
+      const stagingMessageId =
+        uncachedAttachmentLinks.length > 0 ? uuid() : null;
+
+      if (stagingMessageId) {
+        db.insert(messages)
           .values({
-            id: uuid(),
-            messageId: forkedMessageId,
-            attachmentId: cachedAttachmentId,
-            position: link.position,
-            createdAt: Date.now(),
+            id: stagingMessageId,
+            conversationId: fc.id,
+            role: message.role,
+            content: "",
+            createdAt: message.createdAt,
+            metadata: null,
           })
           .run();
-        continue;
       }
 
-      const scopedAttachmentId = linkAttachmentToMessage(
-        stagingMessageId ?? forkedMessageId,
-        link.attachmentId,
-        link.position,
-      );
-      attachmentIdMap.set(link.attachmentId, scopedAttachmentId);
-    }
+      for (const link of attachmentLinks) {
+        const cachedAttachmentId = attachmentIdMap.get(link.attachmentId);
+        if (cachedAttachmentId) {
+          db.insert(messageAttachments)
+            .values({
+              id: uuid(),
+              messageId: forkedMessageId,
+              attachmentId: cachedAttachmentId,
+              position: link.position,
+              createdAt: Date.now(),
+            })
+            .run();
+          continue;
+        }
+
+        const scopedAttachmentId = linkAttachmentToMessage(
+          stagingMessageId ?? forkedMessageId,
+          link.attachmentId,
+          link.position,
+        );
+        attachmentIdMap.set(link.attachmentId, scopedAttachmentId);
+      }
+
+      if (stagingMessageId) {
+        relinkAttachments([stagingMessageId], forkedMessageId);
+        db.delete(messages).where(eq(messages.id, stagingMessageId)).run();
+      }
 
-    if (stagingMessageId) {
-      relinkAttachments([stagingMessageId], forkedMessageId);
-      db.delete(messages).where(eq(messages.id, stagingMessageId)).run();
+      diskSyncQueue.push({
+        conversationId: fc.id,
+        messageId: forkedMessageId,
+        createdAt: fc.createdAt,
+      });
     }
 
-    syncMessageToDisk(
-      forkedConversation.id,
-      forkedMessageId,
-      forkedConversation.createdAt,
-    );
-  }
+    seedForkedConversationAttention({
+      conversationId: fc.id,
+      latestAssistantMessageId: latestForkedAssistant?.messageId ?? null,
+      latestAssistantMessageAt: latestForkedAssistant?.messageAt ?? null,
+    });
 
-  seedForkedConversationAttention({
-    conversationId: forkedConversation.id,
-    latestAssistantMessageId: latestForkedAssistant?.messageId ?? null,
-    latestAssistantMessageAt: latestForkedAssistant?.messageAt ?? null,
+    return fc;
   });
 
+  // Disk-view sync runs after commit — file I/O is idempotent and
+  // conversation deletion cleans up orphaned directories.
+  for (const entry of diskSyncQueue) {
+    syncMessageToDisk(entry.conversationId, entry.messageId, entry.createdAt);
+  }
+
   const persistedFork = getConversation(forkedConversation.id);
   if (!persistedFork) {
     throw new Error(
@@ -513,12 +546,18 @@ export function forkConversation(params: {
  */
 export function deleteConversation(id: string): DeletedMemoryIds {
   const db = getDb();
-  const result: DeletedMemoryIds = { segmentIds: [], orphanedItemIds: [] };
+  const result: DeletedMemoryIds = {
+    segmentIds: [],
+    orphanedItemIds: [],
+    deletedSummaryIds: [],
+  };
 
   // Capture createdAt before the transaction deletes the row — needed to
   // resolve the conversation's disk-view directory path after deletion.
   const convBeforeDelete = getConversation(id);
   const createdAtForDiskCleanup = convBeforeDelete?.createdAt;
+  const memoryScopeId = convBeforeDelete?.memoryScopeId;
+  const isPrivateScope = memoryScopeId?.startsWith("private:") ?? false;
 
   db.transaction((tx) => {
     // Collect all message IDs for this conversation.
@@ -607,6 +646,65 @@ export function deleteConversation(id: string): DeletedMemoryIds {
         .run();
     }
 
+    if (isPrivateScope && memoryScopeId) {
+      // Sweep remaining memory items with this private scopeId.
+      const scopeItems = tx
+        .select({ id: memoryItems.id })
+        .from(memoryItems)
+        .where(eq(memoryItems.scopeId, memoryScopeId))
+        .all();
+      const alreadyDeleted = new Set(result.orphanedItemIds);
+      const scopeItemIds = scopeItems
+        .map((r) => r.id)
+        .filter((id) => !alreadyDeleted.has(id));
+
+      if (scopeItemIds.length > 0) {
+        tx.delete(memoryEmbeddings)
+          .where(
+            and(
+              eq(memoryEmbeddings.targetType, "item"),
+              inArray(memoryEmbeddings.targetId, scopeItemIds),
+            ),
+          )
+          .run();
+        tx.delete(memoryItemSources)
+          .where(inArray(memoryItemSources.memoryItemId, scopeItemIds))
+          .run();
+        tx.delete(memoryItems)
+          .where(inArray(memoryItems.id, scopeItemIds))
+          .run();
+        result.orphanedItemIds.push(...scopeItemIds);
+      }
+
+      // Sweep memory summaries with this private scopeId.
+      const scopeSummaries = tx
+        .select({ id: memorySummaries.id })
+        .from(memorySummaries)
+        .where(eq(memorySummaries.scopeId, memoryScopeId))
+        .all();
+      const scopeSummaryIds = scopeSummaries.map((r) => r.id);
+
+      if (scopeSummaryIds.length > 0) {
+        tx.delete(memoryEmbeddings)
+          .where(
+            and(
+              eq(memoryEmbeddings.targetType, "summary"),
+              inArray(memoryEmbeddings.targetId, scopeSummaryIds),
+            ),
+          )
+          .run();
+        tx.delete(memorySummaries)
+          .where(inArray(memorySummaries.id, scopeSummaryIds))
+          .run();
+        result.deletedSummaryIds.push(...scopeSummaryIds);
+      }
+
+      // Sweep conversation starters with this private scopeId.
+      tx.delete(conversationStarters)
+        .where(eq(conversationStarters.scopeId, memoryScopeId))
+        .run();
+    }
+
     tx.delete(conversations).where(eq(conversations.id, id)).run();
   });
 
@@ -799,7 +897,10 @@ export function wipeConversation(id: string): WipeConversationResult {
   return {
     ...deletedMemoryIds,
     unsupersededItemIds,
-    deletedSummaryIds,
+    deletedSummaryIds: [
+      ...deletedSummaryIds,
+      ...deletedMemoryIds.deletedSummaryIds,
+    ],
     cancelledJobCount,
   };
 }
@@ -821,16 +922,25 @@ export function purgePrivateConversations(): {
     .all();
 
   if (privateConvs.length === 0) {
-    return { count: 0, deletedMemory: { segmentIds: [], orphanedItemIds: [] } };
+    return {
+      count: 0,
+      deletedMemory: {
+        segmentIds: [],
+        orphanedItemIds: [],
+        deletedSummaryIds: [],
+      },
+    };
   }
 
   const allSegmentIds: string[] = [];
   const allOrphanedItemIds: string[] = [];
+  const allDeletedSummaryIds: string[] = [];
 
   for (const conv of privateConvs) {
     const deleted = deleteConversation(conv.id);
     allSegmentIds.push(...deleted.segmentIds);
     allOrphanedItemIds.push(...deleted.orphanedItemIds);
+    allDeletedSummaryIds.push(...deleted.deletedSummaryIds);
   }
 
   return {
@@ -838,6 +948,7 @@ export function purgePrivateConversations(): {
     deletedMemory: {
       segmentIds: allSegmentIds,
       orphanedItemIds: allOrphanedItemIds,
+      deletedSummaryIds: allDeletedSummaryIds,
     },
   };
 }
@@ -920,6 +1031,13 @@ export async function addMessage(
       throw err;
     }
   }
+
+  // Mark the conversation dirty for delayed memory reduction. This runs
+  // after the insert transaction succeeds so the reducer knows which
+  // conversations have unprocessed messages. The helper preserves the
+  // earliest unreduced boundary (no-op when already dirty).
+  markConversationMemoryDirty(conversationId, messageId);
+
   const message = {
     id: messageId,
     conversationId,
@@ -1214,11 +1332,11 @@ export function deleteLastExchange(conversationId: string): number {
 export interface DeletedMemoryIds {
   segmentIds: string[];
   orphanedItemIds: string[];
+  deletedSummaryIds: string[];
 }
 
 export interface WipeConversationResult extends DeletedMemoryIds {
   unsupersededItemIds: string[];
-  deletedSummaryIds: string[];
   cancelledJobCount: number;
 }
 
@@ -1284,7 +1402,11 @@ export function relinkAttachments(
  */
 export function deleteMessageById(messageId: string): DeletedMemoryIds {
   const db = getDb();
-  const result: DeletedMemoryIds = { segmentIds: [], orphanedItemIds: [] };
+  const result: DeletedMemoryIds = {
+    segmentIds: [],
+    orphanedItemIds: [],
+    deletedSummaryIds: [],
+  };
 
   // Collect attachment IDs linked to this message before cascade-delete
   // so we can scope orphan cleanup to only those candidates.
@@ -1372,6 +1494,28 @@ export function deleteMessageById(messageId: string): DeletedMemoryIds {
   return result;
 }
 
+/**
+ * Mark a conversation as having unreduced messages starting from the given
+ * message. Sets `memoryDirtyTailSinceMessageId` only when it is currently
+ * null so the earliest unreduced boundary is preserved across multiple
+ * messages — later messages must not clobber the original dirty marker.
+ */
+export function markConversationMemoryDirty(
+  conversationId: string,
+  messageId: string,
+): void {
+  const db = getDb();
+  db.update(conversations)
+    .set({ memoryDirtyTailSinceMessageId: messageId })
+    .where(
+      and(
+        eq(conversations.id, conversationId),
+        isNull(conversations.memoryDirtyTailSinceMessageId),
+      ),
+    )
+    .run();
+}
+
 export function setConversationOriginChannelIfUnset(
   conversationId: string,
   channel: ChannelId,

package/src/memory/db-init.ts

@@ -82,7 +82,10 @@ import {
   migrateInviteContactId,
   migrateLlmRequestLogMessageId,
   migrateLlmRequestLogProvider,
+  migrateMemoryArchiveTables,
+  migrateMemoryBriefState,
   migrateMemoryItemSupersession,
+  migrateMemoryReducerCheckpoints,
   migrateMessagesFtsBackfill,
   migrateNormalizePhoneIdentities,
   migrateNotificationDeliveryThreadDecision,
@@ -484,6 +487,15 @@ export function initializeDb(): void {
   // 84. Add nullable conversation fork lineage columns and parent lookup index
   migrateConversationForkLineage(database);
 
+  // 85. Memory brief state tables (time_contexts, open_loops) for simplified memory system
+  migrateMemoryBriefState(database);
+
+  // 86. Memory archive tables (observations, chunks, episodes) for simplified memory v1
+  migrateMemoryArchiveTables(database);
+
+  // 87. Add memory reducer checkpoint columns to conversations
+  migrateMemoryReducerCheckpoints(database);
+
   validateMigrationState(database);
 
   if (process.env.BUN_TEST === "1") {