@vellumai/assistant 0.5.2 → 0.5.4

package/src/memory/conversation-key-store.ts

@@ -134,6 +134,7 @@ export function getOrCreateConversation(
   opts?: { conversationType?: "standard" | "private" },
 ): {
   conversationId: string;
+  conversationType: string;
   created: boolean;
 } {
   const db = getDb();
@@ -147,7 +148,16 @@ export function getOrCreateConversation(
       .get();
 
     if (existing) {
-      return { conversationId: existing.conversationId, created: false as const };
+      const conv = tx
+        .select({ conversationType: conversations.conversationType })
+        .from(conversations)
+        .where(eq(conversations.id, existing.conversationId))
+        .get();
+      return {
+        conversationId: existing.conversationId,
+        conversationType: conv?.conversationType ?? "standard",
+        created: false as const,
+      };
     }
 
     // Check if the conversationKey itself is an existing conversation ID.
@@ -168,7 +178,16 @@ export function getOrCreateConversation(
           createdAt: Date.now(),
         })
         .run();
-      return { conversationId: existingConversation.id, created: false as const };
+      const conv = tx
+        .select({ conversationType: conversations.conversationType })
+        .from(conversations)
+        .where(eq(conversations.id, existingConversation.id))
+        .get();
+      return {
+        conversationId: existingConversation.id,
+        conversationType: conv?.conversationType ?? "standard",
+        created: false as const,
+      };
     }
 
     const now = Date.now();
@@ -205,8 +224,14 @@ export function getOrCreateConversation(
 
     return {
       conversationId,
+      conversationType,
       created: true as const,
-      conversation: { id: conversationId, title, createdAt: now, conversationType },
+      conversation: {
+        id: conversationId,
+        title,
+        createdAt: now,
+        conversationType,
+      },
     };
   });
 
@@ -214,5 +239,9 @@ export function getOrCreateConversation(
     initConversationDir({ ...result.conversation, originChannel: null });
   }
 
-  return { conversationId: result.conversationId, created: result.created };
+  return {
+    conversationId: result.conversationId,
+    conversationType: result.conversationType,
+    created: result.created,
+  };
 }

package/src/memory/db-init.ts

@@ -82,7 +82,10 @@ import {
   migrateInviteContactId,
   migrateLlmRequestLogMessageId,
   migrateLlmRequestLogProvider,
+  migrateMemoryArchiveTables,
+  migrateMemoryBriefState,
   migrateMemoryItemSupersession,
+  migrateMemoryReducerCheckpoints,
   migrateMessagesFtsBackfill,
   migrateNormalizePhoneIdentities,
   migrateNotificationDeliveryThreadDecision,
@@ -108,6 +111,7 @@ import {
   migrateRenameVerificationTable,
   migrateRenameVoiceToPhone,
   migrateScheduleOneShotRouting,
+  migrateScheduleQuietFlag,
   migrateSchemaIndexesAndColumns,
   migrateUsageDashboardIndexes,
   migrateVoiceInviteColumns,
@@ -484,6 +488,18 @@ export function initializeDb(): void {
   // 84. Add nullable conversation fork lineage columns and parent lookup index
   migrateConversationForkLineage(database);
 
+  // 85. Memory brief state tables (time_contexts, open_loops) for simplified memory system
+  migrateMemoryBriefState(database);
+
+  // 86. Memory archive tables (observations, chunks, episodes) for simplified memory v1
+  migrateMemoryArchiveTables(database);
+
+  // 87. Add memory reducer checkpoint columns to conversations
+  migrateMemoryReducerCheckpoints(database);
+
+  // 88. Add quiet flag to schedule jobs
+  migrateScheduleQuietFlag(database);
+
   validateMigrationState(database);
 
   if (process.env.BUN_TEST === "1") {

package/src/memory/indexer.ts

@@ -5,6 +5,7 @@ import { getConfig } from "../config/loader.js";
 import type { MemoryConfig } from "../config/types.js";
 import type { TrustClass } from "../runtime/actor-trust-resolver.js";
 import { getLogger } from "../util/logger.js";
+import { computeChunkContentHash } from "./archive-store.js";
 import { getDb } from "./db.js";
 import { selectedBackendSupportsMultimodal } from "./embedding-backend.js";
 import { enqueueMemoryJob } from "./jobs-store.js";
@@ -12,7 +13,7 @@ import {
   extractMediaBlockMeta,
   extractTextFromStoredMessageContent,
 } from "./message-content.js";
-import { memorySegments } from "./schema.js";
+import { memoryChunks, memoryObservations, memorySegments } from "./schema.js";
 import { segmentText } from "./segmenter.js";
 
 const log = getLogger("memory-indexer");
@@ -53,7 +54,12 @@ export async function indexMessageNow(
     input.provenanceTrustClass === undefined;
 
   const text = extractTextFromStoredMessageContent(input.content);
-  if (text.length === 0) {
+  const hasText = text.length > 0;
+  const candidateMediaMeta = extractMediaBlockMeta(input.content).filter(
+    (b) => b.type === "image",
+  );
+  const hasMedia = candidateMediaMeta.length > 0;
+  if (!hasText && !hasMedia) {
     enqueueMemoryJob("build_conversation_summary", {
       conversationId: input.conversationId,
     });
@@ -62,11 +68,13 @@ export async function indexMessageNow(
 
   const db = getDb();
   const now = Date.now();
-  const segments = segmentText(
-    text,
-    config.segmentation.targetTokens,
-    config.segmentation.overlapTokens,
-  );
+  const segments = hasText
+    ? segmentText(
+        text,
+        config.segmentation.targetTokens,
+        config.segmentation.overlapTokens,
+      )
+    : [];
   const shouldExtract =
     input.role === "user" ||
     (input.role === "assistant" && config.extraction.extractFromAssistant);
@@ -76,9 +84,6 @@ export async function indexMessageNow(
   // overhead for messages on non-multimodal backends.
   // selectedBackendSupportsMultimodal requires async key resolution, so we
   // skip it entirely for text-only messages.
-  const candidateMediaMeta = extractMediaBlockMeta(input.content).filter(
-    (b) => b.type === "image",
-  );
   const mediaBlocks =
     candidateMediaMeta.length > 0 &&
     (await selectedBackendSupportsMultimodal(getConfig()))
@@ -88,7 +93,10 @@ export async function indexMessageNow(
   // Wrap all segment inserts and job enqueues in a single transaction so they
   // either all succeed or all roll back, preventing partial/orphaned state.
   let skippedEmbedJobs = 0;
+  let skippedChunkEmbedJobs = 0;
+  const scopeId = input.scopeId ?? "default";
   db.transaction((tx) => {
+    // ── Legacy segment path (kept intact for parallel validation) ───
     for (const segment of segments) {
       const segmentId = buildSegmentId(input.messageId, segment.segmentIndex);
       const hash = createHash("sha256").update(segment.text).digest("hex");
@@ -109,7 +117,7 @@ export async function indexMessageNow(
           segmentIndex: segment.segmentIndex,
           text: segment.text,
           tokenEstimate: segment.tokenEstimate,
-          scopeId: input.scopeId ?? "default",
+          scopeId,
           contentHash: hash,
           createdAt: input.createdAt,
           updatedAt: now,
@@ -119,7 +127,7 @@ export async function indexMessageNow(
           set: {
             text: segment.text,
             tokenEstimate: segment.tokenEstimate,
-            scopeId: input.scopeId ?? "default",
+            scopeId,
             contentHash: hash,
             updatedAt: now,
           },
@@ -133,6 +141,65 @@ export async function indexMessageNow(
       }
     }
 
+    // ── Archive chunk dual-write (mirrors segment boundaries) ──────
+    // Create a single observation per message, then create one chunk per
+    // segment using the same segmentation boundaries. Chunks are
+    // deduplicated by (scopeId, contentHash) via onConflictDoNothing so
+    // unchanged content does not enqueue duplicate embed_chunk jobs.
+    const observationId = buildObservationId(input.messageId);
+    tx.insert(memoryObservations)
+      .values({
+        id: observationId,
+        scopeId,
+        conversationId: input.conversationId,
+        messageId: input.messageId,
+        role: input.role,
+        content: hasText ? text : input.content,
+        modality: hasMedia ? "multimodal" : "text",
+        source: null,
+        createdAt: input.createdAt,
+      })
+      .onConflictDoNothing({ target: memoryObservations.id })
+      .run();
+
+    for (const segment of segments) {
+      const chunkId = buildChunkId(input.messageId, segment.segmentIndex);
+      const chunkHash = computeChunkContentHash(scopeId, segment.text);
+
+      // Check if this chunk already exists with the same content hash
+      const existingChunk = tx
+        .select({ contentHash: memoryChunks.contentHash })
+        .from(memoryChunks)
+        .where(eq(memoryChunks.id, chunkId))
+        .get();
+
+      tx.insert(memoryChunks)
+        .values({
+          id: chunkId,
+          scopeId,
+          observationId,
+          content: segment.text,
+          tokenEstimate: segment.tokenEstimate,
+          contentHash: chunkHash,
+          createdAt: input.createdAt,
+        })
+        .onConflictDoUpdate({
+          target: memoryChunks.id,
+          set: {
+            content: segment.text,
+            tokenEstimate: segment.tokenEstimate,
+            contentHash: chunkHash,
+          },
+        })
+        .run();
+
+      if (existingChunk?.contentHash === chunkHash) {
+        skippedChunkEmbedJobs++;
+      } else {
+        enqueueMemoryJob("embed_chunk", { chunkId, scopeId }, Date.now(), tx);
+      }
+    }
+
     // Enqueue embed_attachment jobs for image content blocks when the
     // embedding provider supports multimodal (Gemini only).
     for (const block of mediaBlocks) {
@@ -147,7 +214,7 @@ export async function indexMessageNow(
     if (shouldExtract && isTrustedActor && !input.automated) {
       enqueueMemoryJob(
         "extract_items",
-        { messageId: input.messageId, scopeId: input.scopeId ?? "default" },
+        { messageId: input.messageId, scopeId },
         Date.now(),
         tx,
       );
@@ -166,6 +233,12 @@ export async function indexMessageNow(
     );
   }
 
+  if (skippedChunkEmbedJobs > 0) {
+    log.debug(
+      `Skipped ${skippedChunkEmbedJobs}/${segments.length} embed_chunk jobs (content unchanged)`,
+    );
+  }
+
   if (!isTrustedActor && shouldExtract) {
     log.info(
       `Skipping extraction jobs for untrusted actor (trustClass=${input.provenanceTrustClass})`,
@@ -177,9 +250,11 @@ export async function indexMessageNow(
   }
 
   const extractionGated = !isTrustedActor || !!input.automated;
+  const segmentEmbedJobs = segments.length - skippedEmbedJobs;
+  const chunkEmbedJobs = segments.length - skippedChunkEmbedJobs;
   const enqueuedJobs =
-    segments.length -
-    skippedEmbedJobs +
+    segmentEmbedJobs +
+    chunkEmbedJobs +
     mediaBlocks.length +
     (shouldExtract && !extractionGated ? 2 : 1);
   return {
@@ -213,3 +288,19 @@ export function getRecentSegmentsForConversation(
 function buildSegmentId(messageId: string, segmentIndex: number): string {
   return `${messageId}:${segmentIndex}`;
 }
+
+/**
+ * Deterministic observation ID derived from the messageId so repeated
+ * indexer runs for the same message converge on the same observation row.
+ */
+function buildObservationId(messageId: string): string {
+  return `obs:${messageId}`;
+}
+
+/**
+ * Deterministic chunk ID derived from the messageId and segment index so
+ * the dual-write path mirrors the legacy segment identity scheme exactly.
+ */
+function buildChunkId(messageId: string, segmentIndex: number): string {
+  return `chunk:${messageId}:${segmentIndex}`;
+}