@vellumai/assistant 0.5.3 → 0.5.5

package/src/memory/conversation-crud.ts

@@ -47,13 +47,18 @@ import {
   conversations,
   conversationStarters,
   llmRequestLogs,
+  memoryChunks,
   memoryEmbeddings,
+  memoryEpisodes,
   memoryItems,
   memoryItemSources,
+  memoryObservations,
   memorySegments,
   memorySummaries,
   messageAttachments,
   messages,
+  openLoops,
+  timeContexts,
   toolInvocations,
 } from "./schema.js";
 import { cancelPendingJobsForConversation } from "./task-memory-cleanup.js";
@@ -550,6 +555,9 @@ export function deleteConversation(id: string): DeletedMemoryIds {
     segmentIds: [],
     orphanedItemIds: [],
     deletedSummaryIds: [],
+    deletedObservationIds: [],
+    deletedChunkIds: [],
+    deletedEpisodeIds: [],
   };
 
   // Capture createdAt before the transaction deletes the row — needed to
@@ -703,6 +711,75 @@ export function deleteConversation(id: string): DeletedMemoryIds {
       tx.delete(conversationStarters)
         .where(eq(conversationStarters.scopeId, memoryScopeId))
         .run();
+
+      // Sweep brief-state tables scoped to this private conversation.
+      tx.delete(timeContexts)
+        .where(eq(timeContexts.scopeId, memoryScopeId))
+        .run();
+      tx.delete(openLoops).where(eq(openLoops.scopeId, memoryScopeId)).run();
+    }
+
+    // Collect archive table IDs before the cascade delete removes them.
+    // Observations and episodes reference conversations with ON DELETE CASCADE,
+    // and chunks cascade from observations.
+    const observationRows = tx
+      .select({ id: memoryObservations.id })
+      .from(memoryObservations)
+      .where(eq(memoryObservations.conversationId, id))
+      .all();
+    const observationIds = observationRows.map((r) => r.id);
+
+    if (observationIds.length > 0) {
+      // Collect chunk IDs before observations cascade-delete them.
+      const chunkRows = tx
+        .select({ id: memoryChunks.id })
+        .from(memoryChunks)
+        .where(inArray(memoryChunks.observationId, observationIds))
+        .all();
+      const chunkIds = chunkRows.map((r) => r.id);
+
+      // Clean up embeddings for chunks.
+      if (chunkIds.length > 0) {
+        tx.delete(memoryEmbeddings)
+          .where(
+            and(
+              eq(memoryEmbeddings.targetType, "chunk"),
+              inArray(memoryEmbeddings.targetId, chunkIds),
+            ),
+          )
+          .run();
+        result.deletedChunkIds.push(...chunkIds);
+      }
+
+      // Clean up embeddings for observations.
+      tx.delete(memoryEmbeddings)
+        .where(
+          and(
+            eq(memoryEmbeddings.targetType, "observation"),
+            inArray(memoryEmbeddings.targetId, observationIds),
+          ),
+        )
+        .run();
+      result.deletedObservationIds.push(...observationIds);
+    }
+
+    const episodeRows = tx
+      .select({ id: memoryEpisodes.id })
+      .from(memoryEpisodes)
+      .where(eq(memoryEpisodes.conversationId, id))
+      .all();
+    const episodeIds = episodeRows.map((r) => r.id);
+
+    if (episodeIds.length > 0) {
+      tx.delete(memoryEmbeddings)
+        .where(
+          and(
+            eq(memoryEmbeddings.targetType, "episode"),
+            inArray(memoryEmbeddings.targetId, episodeIds),
+          ),
+        )
+        .run();
+      result.deletedEpisodeIds.push(...episodeIds);
     }
 
     tx.delete(conversations).where(eq(conversations.id, id)).run();
@@ -928,6 +1005,9 @@ export function purgePrivateConversations(): {
         segmentIds: [],
         orphanedItemIds: [],
         deletedSummaryIds: [],
+        deletedObservationIds: [],
+        deletedChunkIds: [],
+        deletedEpisodeIds: [],
       },
     };
   }
@@ -935,12 +1015,18 @@ export function purgePrivateConversations(): {
   const allSegmentIds: string[] = [];
   const allOrphanedItemIds: string[] = [];
   const allDeletedSummaryIds: string[] = [];
+  const allDeletedObservationIds: string[] = [];
+  const allDeletedChunkIds: string[] = [];
+  const allDeletedEpisodeIds: string[] = [];
 
   for (const conv of privateConvs) {
     const deleted = deleteConversation(conv.id);
     allSegmentIds.push(...deleted.segmentIds);
     allOrphanedItemIds.push(...deleted.orphanedItemIds);
     allDeletedSummaryIds.push(...deleted.deletedSummaryIds);
+    allDeletedObservationIds.push(...deleted.deletedObservationIds);
+    allDeletedChunkIds.push(...deleted.deletedChunkIds);
+    allDeletedEpisodeIds.push(...deleted.deletedEpisodeIds);
   }
 
   return {
@@ -949,6 +1035,9 @@ export function purgePrivateConversations(): {
       segmentIds: allSegmentIds,
       orphanedItemIds: allOrphanedItemIds,
       deletedSummaryIds: allDeletedSummaryIds,
+      deletedObservationIds: allDeletedObservationIds,
+      deletedChunkIds: allDeletedChunkIds,
+      deletedEpisodeIds: allDeletedEpisodeIds,
     },
   };
 }
@@ -1226,6 +1315,15 @@ export function clearAll(): { conversations: number; messages: number } {
   rawExec("DELETE FROM messages");
   rawExec("DELETE FROM conversations");
 
+  // Record audit event — lifecycle_events is NOT deleted by clearAll(),
+  // so this survives the wipe and provides a permanent trail.
+  rawRun(
+    `INSERT INTO lifecycle_events (id, event_name, created_at) VALUES (?, ?, ?)`,
+    uuid(),
+    "conversations_clear_all",
+    Date.now(),
+  );
+
   // Rebuild corrupted FTS tables and restore triggers after all base-table
   // DELETEs have completed. Dropping the virtual table clears the corruption,
   // and recreating it + triggers means subsequent writes maintain FTS
@@ -1333,6 +1431,9 @@ export interface DeletedMemoryIds {
   segmentIds: string[];
   orphanedItemIds: string[];
   deletedSummaryIds: string[];
+  deletedObservationIds: string[];
+  deletedChunkIds: string[];
+  deletedEpisodeIds: string[];
 }
 
 export interface WipeConversationResult extends DeletedMemoryIds {
@@ -1406,6 +1507,9 @@ export function deleteMessageById(messageId: string): DeletedMemoryIds {
     segmentIds: [],
     orphanedItemIds: [],
     deletedSummaryIds: [],
+    deletedObservationIds: [],
+    deletedChunkIds: [],
+    deletedEpisodeIds: [],
   };
 
   // Collect attachment IDs linked to this message before cascade-delete
@@ -1499,6 +1603,12 @@ export function deleteMessageById(messageId: string): DeletedMemoryIds {
  * message. Sets `memoryDirtyTailSinceMessageId` only when it is currently
  * null so the earliest unreduced boundary is preserved across multiple
  * messages — later messages must not clobber the original dirty marker.
+ *
+ * Also upserts a pending `reduce_conversation_memory` job scheduled at
+ * `now + idleDelayMs`. If a pending job for this conversation already exists,
+ * its `runAfter` is pushed forward (rescheduled) so the reducer waits for
+ * the full idle window after the *latest* message — avoiding premature runs
+ * while the user is still actively typing.
  */
 export function markConversationMemoryDirty(
   conversationId: string,
@@ -1514,6 +1624,106 @@ export function markConversationMemoryDirty(
       ),
     )
     .run();
+
+  // Schedule (or reschedule) a deferred reducer job for this conversation.
+  scheduleReducerJob(conversationId);
+}
+
+/**
+ * Upsert a pending `reduce_conversation_memory` job for the given
+ * conversation, scheduled `idleDelayMs` from now. If one already exists in
+ * pending state, its `runAfter` is pushed forward to restart the idle timer.
+ * This ensures exactly one pending reducer job per conversation — new
+ * messages reschedule rather than duplicate.
+ */
+export function scheduleReducerJob(
+  conversationId: string,
+  runAfter?: number,
+): void {
+  const idleDelayMs = getReducerIdleDelayMs();
+  const scheduledAt = runAfter ?? Date.now() + idleDelayMs;
+
+  const existing = rawGet<{ id: string; status: string }>(
+    `SELECT id, status FROM memory_jobs
+     WHERE type = 'reduce_conversation_memory'
+       AND json_extract(payload, '$.conversationId') = ?
+       AND status = 'pending'
+     LIMIT 1`,
+    conversationId,
+  );
+
+  if (existing) {
+    // Reschedule: push runAfter forward so the idle timer resets.
+    rawRun(
+      `UPDATE memory_jobs SET run_after = ?, updated_at = ? WHERE id = ?`,
+      scheduledAt,
+      Date.now(),
+      existing.id,
+    );
+  } else {
+    enqueueMemoryJob(
+      "reduce_conversation_memory",
+      { conversationId },
+      scheduledAt,
+    );
+  }
+}
+
+/**
+ * Startup sweep: find conversations that are marked dirty and whose tail
+ * message is already older than the idle delay. For these conversations the
+ * reducer should have run but didn't (daemon was down). Enqueue immediate
+ * reducer jobs for each so they are processed on the next worker tick.
+ *
+ * Conversations whose tail is still within the idle window are skipped —
+ * the normal `markConversationMemoryDirty` path will schedule them when
+ * new messages arrive (or on the next conversation interaction).
+ *
+ * Returns the number of jobs enqueued.
+ */
+export function sweepStaleReducerJobs(): number {
+  const idleDelayMs = getReducerIdleDelayMs();
+  const cutoff = Date.now() - idleDelayMs;
+
+  // Find dirty conversations whose latest message is older than the idle
+  // window AND that don't already have a pending reducer job.
+  const stale = rawAll<{ conversationId: string }>(
+    `SELECT c.id AS conversationId
+     FROM conversations c
+     WHERE c.memory_dirty_tail_since_message_id IS NOT NULL
+       AND NOT EXISTS (
+         SELECT 1 FROM memory_jobs mj
+         WHERE mj.type = 'reduce_conversation_memory'
+           AND json_extract(mj.payload, '$.conversationId') = c.id
+           AND mj.status IN ('pending', 'running')
+       )
+       AND (
+         SELECT MAX(m.created_at) FROM messages m
+         WHERE m.conversation_id = c.id
+       ) <= ?`,
+    cutoff,
+  );
+
+  for (const { conversationId } of stale) {
+    enqueueMemoryJob("reduce_conversation_memory", { conversationId });
+  }
+
+  return stale.length;
+}
+
+function getReducerIdleDelayMs(): number {
+  // Some test suites mock getConfig() with partial objects; fall back to the
+  // schema default so reducer scheduling stays stable outside full config load.
+  const config = getConfig() as {
+    memory?: {
+      simplified?: {
+        reducer?: {
+          idleDelayMs?: number;
+        };
+      };
+    };
+  };
+  return config.memory?.simplified?.reducer?.idleDelayMs ?? 30_000;
 }
 
 export function setConversationOriginChannelIfUnset(

package/src/memory/conversation-key-store.ts

@@ -134,6 +134,7 @@ export function getOrCreateConversation(
   opts?: { conversationType?: "standard" | "private" },
 ): {
   conversationId: string;
+  conversationType: string;
   created: boolean;
 } {
   const db = getDb();
@@ -147,7 +148,16 @@ export function getOrCreateConversation(
       .get();
 
     if (existing) {
-      return { conversationId: existing.conversationId, created: false as const };
+      const conv = tx
+        .select({ conversationType: conversations.conversationType })
+        .from(conversations)
+        .where(eq(conversations.id, existing.conversationId))
+        .get();
+      return {
+        conversationId: existing.conversationId,
+        conversationType: conv?.conversationType ?? "standard",
+        created: false as const,
+      };
     }
 
     // Check if the conversationKey itself is an existing conversation ID.
@@ -168,7 +178,16 @@ export function getOrCreateConversation(
           createdAt: Date.now(),
         })
         .run();
-      return { conversationId: existingConversation.id, created: false as const };
+      const conv = tx
+        .select({ conversationType: conversations.conversationType })
+        .from(conversations)
+        .where(eq(conversations.id, existingConversation.id))
+        .get();
+      return {
+        conversationId: existingConversation.id,
+        conversationType: conv?.conversationType ?? "standard",
+        created: false as const,
+      };
     }
 
     const now = Date.now();
@@ -205,8 +224,14 @@ export function getOrCreateConversation(
 
     return {
       conversationId,
+      conversationType,
       created: true as const,
-      conversation: { id: conversationId, title, createdAt: now, conversationType },
+      conversation: {
+        id: conversationId,
+        title,
+        createdAt: now,
+        conversationType,
+      },
     };
   });
 
@@ -214,5 +239,9 @@ export function getOrCreateConversation(
     initConversationDir({ ...result.conversation, originChannel: null });
   }
 
-  return { conversationId: result.conversationId, created: result.created };
+  return {
+    conversationId: result.conversationId,
+    conversationType: result.conversationType,
+    created: result.created,
+  };
 }

package/src/memory/db-init.ts

@@ -111,6 +111,7 @@ import {
   migrateRenameVerificationTable,
   migrateRenameVoiceToPhone,
   migrateScheduleOneShotRouting,
+  migrateScheduleQuietFlag,
   migrateSchemaIndexesAndColumns,
   migrateUsageDashboardIndexes,
   migrateVoiceInviteColumns,
@@ -496,6 +497,9 @@ export function initializeDb(): void {
   // 87. Add memory reducer checkpoint columns to conversations
   migrateMemoryReducerCheckpoints(database);
 
+  // 88. Add quiet flag to schedule jobs
+  migrateScheduleQuietFlag(database);
+
   validateMigrationState(database);
 
   if (process.env.BUN_TEST === "1") {

package/src/memory/embedding-local.ts

@@ -1,6 +1,7 @@
 import { existsSync, unlinkSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 
+import { getIsContainerized } from "../config/env-registry.js";
 import { getLogger } from "../util/logger.js";
 import { getEmbeddingModelsDir, getRootDir } from "../util/platform.js";
 import { PromiseGuard } from "../util/promise-guard.js";
@@ -353,12 +354,17 @@ export class LocalEmbeddingBackend implements EmbeddingBackend {
 
   private static readonly PID_FILENAME = "embed-worker.pid";
 
+  /** PID files are process-local state — store in /tmp when containerized to keep shared volumes clean. */
+  private getPidFilePath(): string {
+    if (getIsContainerized()) {
+      return join("/tmp", LocalEmbeddingBackend.PID_FILENAME);
+    }
+    return join(getRootDir(), LocalEmbeddingBackend.PID_FILENAME);
+  }
+
   private writePidFile(pid: number): void {
     try {
-      writeFileSync(
-        join(getRootDir(), LocalEmbeddingBackend.PID_FILENAME),
-        String(pid),
-      );
+      writeFileSync(this.getPidFilePath(), String(pid));
     } catch {
       // Best-effort — doesn't affect functionality
     }
@@ -366,7 +372,7 @@ export class LocalEmbeddingBackend implements EmbeddingBackend {
 
   private removePidFile(): void {
     try {
-      unlinkSync(join(getRootDir(), LocalEmbeddingBackend.PID_FILENAME));
+      unlinkSync(this.getPidFilePath());
     } catch {
       // Best-effort
     }