@vellumai/assistant 0.4.49 → 0.4.51

package/src/memory/conversation-crud.ts

@@ -19,7 +19,6 @@ import {
   conversations,
   llmRequestLogs,
   memoryEmbeddings,
-  memoryItemEntities,
   memoryItems,
   memoryItemSources,
   memorySegments,
@@ -503,19 +502,6 @@ export function clearAll(): { conversations: number; messages: number } {
   // triggers so that the subsequent base-table DELETEs don't also fail
   // (SQLite triggers are atomic with the triggering statement, so a
   // corrupted FTS table would roll back every base-table DELETE).
-  let segmentFtsCorrupted = false;
-  try {
-    rawExec("DELETE FROM memory_segment_fts");
-  } catch (err) {
-    log.warn(
-      { err },
-      "clearAll: failed to clear memory_segment_fts — dropping triggers so base-table cleanup can proceed",
-    );
-    rawExec("DROP TRIGGER IF EXISTS memory_segments_ai");
-    rawExec("DROP TRIGGER IF EXISTS memory_segments_ad");
-    rawExec("DROP TRIGGER IF EXISTS memory_segments_au");
-    segmentFtsCorrupted = true;
-  }
   rawExec("DELETE FROM memory_item_sources");
   rawExec("DELETE FROM memory_segments");
   rawExec("DELETE FROM memory_items");
@@ -548,21 +534,6 @@ export function clearAll(): { conversations: number; messages: number } {
   // DELETEs have completed. Dropping the virtual table clears the corruption,
   // and recreating it + triggers means subsequent writes maintain FTS
   // consistency without requiring a daemon restart.
-  if (segmentFtsCorrupted) {
-    rawExec("DROP TABLE IF EXISTS memory_segment_fts");
-    rawExec(
-      `CREATE VIRTUAL TABLE IF NOT EXISTS memory_segment_fts USING fts5(segment_id UNINDEXED, text)`,
-    );
-    rawExec(
-      `CREATE TRIGGER IF NOT EXISTS memory_segments_ai AFTER INSERT ON memory_segments BEGIN INSERT INTO memory_segment_fts(segment_id, text) VALUES (new.id, new.text); END`,
-    );
-    rawExec(
-      `CREATE TRIGGER IF NOT EXISTS memory_segments_ad AFTER DELETE ON memory_segments BEGIN DELETE FROM memory_segment_fts WHERE segment_id = old.id; END`,
-    );
-    rawExec(
-      `CREATE TRIGGER IF NOT EXISTS memory_segments_au AFTER UPDATE ON memory_segments BEGIN DELETE FROM memory_segment_fts WHERE segment_id = old.id; INSERT INTO memory_segment_fts(segment_id, text) VALUES (new.id, new.text); END`,
-    );
-  }
   if (messagesFtsCorrupted) {
     rawExec("DROP TABLE IF EXISTS messages_fts");
     rawExec(
@@ -787,10 +758,6 @@ export function deleteMessageById(messageId: string): DeletedMemoryIds {
       result.orphanedItemIds = orphanedIds;
 
       if (orphanedIds.length > 0) {
-        // Delete memory_item_entities (no FK cascade on this table).
-        tx.delete(memoryItemEntities)
-          .where(inArray(memoryItemEntities.memoryItemId, orphanedIds))
-          .run();
         // Delete embeddings referencing these items.
         tx.delete(memoryEmbeddings)
           .where(

package/src/memory/conversation-queries.ts

@@ -1,15 +1,29 @@
-import { and, asc, count, desc, eq, lt, or, sql } from "drizzle-orm";
+import { and, count, desc, eq, sql } from "drizzle-orm";
 
 import { getLogger } from "../util/logger.js";
-import type { ConversationRow, MessageRow } from "./conversation-crud.js";
-import { parseConversation, parseMessage } from "./conversation-crud.js";
+import type { ConversationRow } from "./conversation-crud.js";
+import { parseConversation } from "./conversation-crud.js";
 import { ensureDisplayOrderMigration } from "./conversation-display-order-migration.js";
 import { getDb, rawAll } from "./db.js";
 import { conversations, messages } from "./schema.js";
-import { buildFtsMatchQuery } from "./search/lexical.js";
 
 const log = getLogger("conversation-store");
 
+/**
+ * Build an FTS5 MATCH query string from natural text by extracting tokens.
+ * Used for messages_fts full-text search over conversation content.
+ */
+function buildFtsMatchQuery(text: string): string | null {
+  const tokens = text
+    .toLowerCase()
+    .split(/[^a-z0-9_]+/g)
+    .map((token) => token.trim())
+    .filter((token) => token.length >= 2);
+  if (tokens.length === 0) return null;
+  const unique = [...new Set(tokens)].slice(0, 24);
+  return unique.map((token) => `"${token.replace(/"/g, '""')}"`).join(" OR ");
+}
+
 export function listConversations(
   limit?: number,
   includeBackground = false,
@@ -55,83 +69,6 @@ export function getLatestConversation(): ConversationRow | null {
   return row ? parseConversation(row) : null;
 }
 
-export interface PaginatedMessagesResult {
-  messages: MessageRow[];
-  /** Whether older messages exist beyond the returned page. */
-  hasMore: boolean;
-}
-
-/**
- * Paginated variant of getMessages. Returns the most recent `limit` messages
- * (optionally before a cursor timestamp), in chronological order.
- *
- * When `limit` is undefined, all matching messages are returned (no pagination).
- * When `beforeMessageId` is provided alongside `beforeTimestamp`, it acts as a
- * tie-breaker to avoid skipping messages that share the same millisecond timestamp
- * at page boundaries.
- */
-export function getMessagesPaginated(
-  conversationId: string,
-  limit: number | undefined,
-  beforeTimestamp?: number,
-  beforeMessageId?: string,
-): PaginatedMessagesResult {
-  const db = getDb();
-  const conditions = [eq(messages.conversationId, conversationId)];
-  if (beforeTimestamp !== undefined) {
-    if (beforeMessageId) {
-      // Proper compound cursor: fetch messages that are strictly older, OR
-      // share the same timestamp but have a smaller ID. This avoids both
-      // duplicates and skipped messages when multiple rows share a timestamp.
-      conditions.push(
-        or(
-          lt(messages.createdAt, beforeTimestamp),
-          and(
-            eq(messages.createdAt, beforeTimestamp),
-            lt(messages.id, beforeMessageId),
-          ),
-        )!,
-      );
-    } else {
-      // Legacy callers without a message ID tie-breaker: use strict lt.
-      // This may skip same-millisecond messages at boundaries, but avoids
-      // re-fetching the boundary message. New callers should prefer the
-      // compound cursor (beforeTimestamp + beforeMessageId).
-      conditions.push(lt(messages.createdAt, beforeTimestamp));
-    }
-  }
-
-  if (limit === undefined) {
-    // Unlimited: return all messages in chronological order, no pagination.
-    const rows = db
-      .select()
-      .from(messages)
-      .where(and(...conditions))
-      .orderBy(asc(messages.createdAt), asc(messages.id))
-      .all()
-      .map(parseMessage);
-    return { messages: rows, hasMore: false };
-  }
-
-  // Fetch limit+1 rows ordered newest-first so we can detect hasMore
-  const rows = db
-    .select()
-    .from(messages)
-    .where(and(...conditions))
-    .orderBy(desc(messages.createdAt), desc(messages.id))
-    .limit(limit + 1)
-    .all()
-    .map(parseMessage);
-
-  const hasMore = rows.length > limit;
-  const page = hasMore ? rows.slice(0, limit) : rows;
-
-  // Return in chronological order (oldest first) for the client
-  page.reverse();
-
-  return { messages: page, hasMore };
-}
-
 /**
  * Check whether the last user message in a conversation is a tool_result-only
  * message (i.e., not a real user-typed message). This is used by undo() to
@@ -160,7 +97,9 @@ export function isLastUserMessageToolResult(conversationId: string): boolean {
       Array.isArray(parsed) &&
       parsed.length > 0 &&
       parsed.every(
-        (block: Record<string, unknown>) => block.type === "tool_result",
+        (block: Record<string, unknown>) =>
+          block.type === "tool_result" ||
+          block.type === "web_search_tool_result",
       )
     ) {
       return true;
@@ -375,7 +314,10 @@ function buildExcerpt(rawContent: string, query: string): string {
         if (typeof block === "object" && block != null) {
           if (block.type === "text" && typeof block.text === "string") {
             parts.push(block.text);
-          } else if (block.type === "tool_result") {
+          } else if (
+            block.type === "tool_result" ||
+            block.type === "web_search_tool_result"
+          ) {
             const inner = Array.isArray(block.content) ? block.content : [];
             for (const ib of inner) {
               if (ib?.type === "text" && typeof ib.text === "string")

package/src/memory/db-init.ts

@@ -37,6 +37,7 @@ import {
   migrateBackfillContactInteractionStats,
   migrateBackfillGuardianPrincipalId,
   migrateBackfillUsageCacheAccounting,
+  migrateCallSessionInviteMetadata,
   migrateCallSessionMode,
   migrateCanonicalGuardianDeliveriesDestinationIndex,
   migrateCanonicalGuardianRequesterChatId,
@@ -49,7 +50,10 @@ import {
   migrateConversationsThreadTypeIndex,
   migrateDropAccountsTable,
   migrateDropAssistantIdColumns,
+  migrateDropConflicts,
+  migrateDropEntityTables,
   migrateDropLegacyMemberGuardianTables,
+  migrateDropMemorySegmentFts,
   migrateDropRemindersTable,
   migrateDropUsageCompositeIndexes,
   migrateFkCascadeRebuilds,
@@ -63,9 +67,13 @@ import {
   migrateGuardianVerificationPurpose,
   migrateGuardianVerificationSessions,
   migrateInviteCodeHashColumn,
+  migrateInviteContactId,
+  migrateMemoryItemSupersession,
   migrateMessagesFtsBackfill,
   migrateNormalizePhoneIdentities,
   migrateNotificationDeliveryThreadDecision,
+  migrateOAuthAppsClientSecretPath,
+  migrateOAuthProvidersPingUrl,
   migrateReminderRoutingIntent,
   migrateRemindersToSchedules,
   migrateRenameGuardianVerificationValues,
@@ -344,6 +352,30 @@ export function initializeDb(): void {
   // 53. OAuth provider/app/connection tables
   createOAuthTables(database);
 
+  // 54. Add explicit client_secret_credential_path to oauth_apps
+  migrateOAuthAppsClientSecretPath(database);
+
+  // 55. Add ping_url column to oauth_providers
+  migrateOAuthProvidersPingUrl(database);
+
+  // 56. Add supersession tracking columns and override confidence to memory_items
+  migrateMemoryItemSupersession(database);
+
+  // 56b. Drop unused entity tables (entity search replaced by hybrid search on item statements)
+  migrateDropEntityTables(database);
+
+  // 57. Drop memory_segment_fts virtual table and triggers (replaced by Qdrant hybrid search)
+  migrateDropMemorySegmentFts(database);
+
+  // 58. Drop memory_item_conflicts table (conflict resolution system removed)
+  migrateDropConflicts(database);
+
+  // 59. Add invite metadata columns to call_sessions for outbound invite call routing
+  migrateCallSessionInviteMetadata(database);
+
+  // 60. Add required contact_id to assistant_ingress_invites and clean up legacy rows
+  migrateInviteContactId(database);
+
   validateMigrationState(database);
 
   if (process.env.BUN_TEST === "1") {

package/src/memory/embedding-backend.ts

@@ -11,14 +11,11 @@ import {
   embeddingInputContentHash,
   type MultimodalEmbeddingInput,
   normalizeEmbeddingInput,
+  type SparseEmbedding,
   type TextEmbeddingInput,
 } from "./embedding-types.js";
 
-export type {
-  EmbeddingInput,
-  MultimodalEmbeddingInput,
-  TextEmbeddingInput,
-};
+export type { EmbeddingInput, MultimodalEmbeddingInput, TextEmbeddingInput };
 export { embeddingInputContentHash, normalizeEmbeddingInput };
 
 const log = getLogger("memory-embeddings");
@@ -412,7 +409,12 @@ export async function embedWithBackend(
 
   // ── In-memory cache check (primary provider only) ──────────────
   const cached: (number[] | null)[] = inputs.map((input) => {
-    const v = getFromVectorCache(primaryProvider, primaryModel, input, vectorExtras);
+    const v = getFromVectorCache(
+      primaryProvider,
+      primaryModel,
+      input,
+      vectorExtras,
+    );
     if (v && v.length === expectedDim) return v;
     return null;
   });
@@ -443,7 +445,8 @@ export async function embedWithBackend(
 
     // Skip text-only backends for multimodal inputs
     const hasNonText = inputsToEmbed.some(
-      (i) => typeof i !== "string" && normalizeEmbeddingInput(i).type !== "text",
+      (i) =>
+        typeof i !== "string" && normalizeEmbeddingInput(i).type !== "text",
     );
     if (backend.provider !== "gemini" && hasNonText) {
       continue;
@@ -502,7 +505,8 @@ export async function embedWithBackend(
   }
   if (!anyBackendAttempted) {
     const hasMultimodal = inputs.some(
-      (i) => typeof i !== "string" && normalizeEmbeddingInput(i).type !== "text",
+      (i) =>
+        typeof i !== "string" && normalizeEmbeddingInput(i).type !== "text",
     );
     if (hasMultimodal) {
       throw new Error(
@@ -614,3 +618,75 @@ function isOllamaConfigured(config: AssistantConfig): boolean {
     Boolean(getOllamaBaseUrlEnv())
   );
 }
+
+// ── TF-IDF sparse embedding ───────────────────────────────────────
+// Simple tokenizer + TF-IDF sparse encoder. Produces a SparseEmbedding
+// with term indices (hashed to a fixed vocabulary) and TF-IDF weights.
+// Can be upgraded to a learned sparse encoder (e.g. SPLADE) later.
+
+const SPARSE_VOCAB_SIZE = 30_000;
+
+/**
+ * Bump this version whenever the sparse embedding algorithm changes
+ * (e.g. hash function fix, tokenizer change) to trigger re-indexing
+ * of existing sparse vectors via the sentinel mismatch mechanism.
+ */
+export const SPARSE_EMBEDDING_VERSION = 2;
+
+/** Tokenize text into lowercase alphanumeric tokens (Unicode-aware). */
+function tokenize(text: string): string[] {
+  return text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
+}
+
+/** Hash a token to a stable index in [0, vocabSize). */
+function tokenHash(token: string, vocabSize: number): number {
+  // FNV-1a 32-bit hash for speed
+  let hash = 0x811c9dc5;
+  for (let i = 0; i < token.length; i++) {
+    hash ^= token.charCodeAt(i);
+    hash = Math.imul(hash, 0x01000193) >>> 0;
+  }
+  return hash % vocabSize;
+}
+
+/**
+ * Generate a TF-IDF-based sparse embedding for the given text.
+ *
+ * Term frequency is computed from the input. IDF is approximated using
+ * sub-linear TF weighting (1 + log(tf)) since we don't have a corpus-level
+ * document frequency table. This still produces useful sparse vectors for
+ * lexical matching via Qdrant's sparse vector support.
+ */
+export function generateSparseEmbedding(text: string): SparseEmbedding {
+  const tokens = tokenize(text);
+  if (tokens.length === 0) {
+    return { indices: [], values: [] };
+  }
+
+  // Count term frequencies per hash bucket
+  const tf = new Map<number, number>();
+  for (const token of tokens) {
+    const idx = tokenHash(token, SPARSE_VOCAB_SIZE);
+    tf.set(idx, (tf.get(idx) ?? 0) + 1);
+  }
+
+  // Convert to sub-linear TF weights: 1 + log(tf)
+  const indices: number[] = [];
+  const values: number[] = [];
+  for (const [idx, count] of tf) {
+    indices.push(idx);
+    values.push(1 + Math.log(count));
+  }
+
+  // L2-normalize the sparse vector so scores are comparable
+  let norm = 0;
+  for (const v of values) norm += v * v;
+  norm = Math.sqrt(norm);
+  if (norm > 0) {
+    for (let i = 0; i < values.length; i++) {
+      values[i] /= norm;
+    }
+  }
+
+  return { indices, values };
+}

package/src/memory/embedding-types.ts

@@ -42,11 +42,19 @@ export type MultimodalEmbeddingInput =
 /** Accepts raw strings as shorthand for text inputs. */
 export type EmbeddingInput = string | MultimodalEmbeddingInput;
 
-export function normalizeEmbeddingInput(input: EmbeddingInput): MultimodalEmbeddingInput {
+export function normalizeEmbeddingInput(
+  input: EmbeddingInput,
+): MultimodalEmbeddingInput {
   if (typeof input === "string") return { type: "text", text: input };
   return input;
 }
 
+/** Sparse vector representation: parallel arrays of term indices and weights. */
+export interface SparseEmbedding {
+  indices: number[];
+  values: number[];
+}
+
 export function embeddingInputContentHash(input: EmbeddingInput): string {
   const normalized = normalizeEmbeddingInput(input);
   const hash = createHash("sha256");

package/src/memory/indexer.ts

@@ -5,24 +5,17 @@ import { getConfig } from "../config/loader.js";
 import type { MemoryConfig } from "../config/types.js";
 import type { TrustClass } from "../runtime/actor-trust-resolver.js";
 import { getLogger } from "../util/logger.js";
-import { getMemoryCheckpoint, setMemoryCheckpoint } from "./checkpoints.js";
 import { getDb } from "./db.js";
 import { selectedBackendSupportsMultimodal } from "./embedding-backend.js";
-import {
-  enqueueMemoryJob,
-  enqueueResolvePendingConflictsForMessageJob,
-} from "./jobs-store.js";
+import { enqueueMemoryJob } from "./jobs-store.js";
 import {
   extractMediaBlocks,
   extractTextFromStoredMessageContent,
 } from "./message-content.js";
-import { bumpMemoryVersion } from "./recall-cache.js";
 import { memorySegments } from "./schema.js";
 import { segmentText } from "./segmenter.js";
 
 const log = getLogger("memory-indexer");
-const SUMMARY_JOB_CHECKPOINT_KEY = "memory:summary_jobs:last_scheduled_at";
-const SUMMARY_SCHEDULE_INTERVAL_MS = 6 * 60 * 60 * 1000;
 
 export interface IndexMessageInput {
   messageId: string;
@@ -34,8 +27,8 @@ export interface IndexMessageInput {
   /**
    * Trust class of the actor who produced this message, captured at
    * persist time. When `'guardian'` or `undefined` (legacy), extraction
-   * and conflict resolution jobs run. Otherwise, the message is segmented
-   * and embedded but no profile mutations are triggered.
+   * jobs run. Otherwise, the message is segmented and embedded but no
+   * profile mutations are triggered.
    */
   provenanceTrustClass?: TrustClass;
 }
@@ -52,7 +45,7 @@ export function indexMessageNow(
   if (!config.enabled) return { indexedSegments: 0, enqueuedJobs: 0 };
 
   // Provenance-based trust gating: only guardian and legacy (undefined) actors
-  // are trusted for extraction and conflict resolution.
+  // are trusted for extraction.
   const isTrustedActor =
     input.provenanceTrustClass === "guardian" ||
     input.provenanceTrustClass === undefined;
@@ -75,9 +68,6 @@ export function indexMessageNow(
   const shouldExtract =
     input.role === "user" ||
     (input.role === "assistant" && config.extraction.extractFromAssistant);
-  const shouldResolveConflicts =
-    input.role === "user" && config.conflicts.enabled;
-
   // Check if the resolved embedding backend supports multimodal input.
   // Only enqueue embed_attachment jobs when it does (currently Gemini only).
   const supportsMultimodal = selectedBackendSupportsMultimodal(getConfig());
@@ -152,13 +142,6 @@ export function indexMessageNow(
         tx,
       );
     }
-    if (shouldResolveConflicts && isTrustedActor) {
-      enqueueResolvePendingConflictsForMessageJob(
-        input.messageId,
-        input.scopeId ?? "default",
-        tx,
-      );
-    }
     enqueueMemoryJob(
       "build_conversation_summary",
       { conversationId: input.conversationId },
@@ -173,27 +156,18 @@ export function indexMessageNow(
     );
   }
 
-  // Invalidate recall cache when synchronous segment writes changed content,
-  // so lexical/recency retrieval doesn't serve stale results during worker lag.
-  if (segments.length - skippedEmbedJobs > 0) {
-    bumpMemoryVersion();
-  }
-
-  if (!isTrustedActor && (shouldExtract || shouldResolveConflicts)) {
+  if (!isTrustedActor && shouldExtract) {
     log.info(
-      `Skipping extraction/conflict jobs for untrusted actor (trustClass=${input.provenanceTrustClass})`,
+      `Skipping extraction jobs for untrusted actor (trustClass=${input.provenanceTrustClass})`,
     );
   }
 
-  enqueueSummaryRollupJobsIfDue();
-
   const extractionGated = !isTrustedActor;
   const enqueuedJobs =
     segments.length -
     skippedEmbedJobs +
     mediaBlocks.length +
-    (shouldExtract && !extractionGated ? 2 : 1) +
-    (shouldResolveConflicts && !extractionGated ? 1 : 0);
+    (shouldExtract && !extractionGated ? 2 : 1);
   return {
     indexedSegments: segments.length,
     enqueuedJobs,
@@ -222,19 +196,6 @@ export function getRecentSegmentsForConversation(
     .all();
 }
 
-function enqueueSummaryRollupJobsIfDue(): void {
-  const now = Date.now();
-  const raw = getMemoryCheckpoint(SUMMARY_JOB_CHECKPOINT_KEY);
-  const last = raw ? Number.parseInt(raw, 10) : 0;
-  if (Number.isFinite(last) && now - last < SUMMARY_SCHEDULE_INTERVAL_MS)
-    return;
-
-  enqueueMemoryJob("refresh_weekly_summary", {});
-  enqueueMemoryJob("refresh_monthly_summary", {});
-  setMemoryCheckpoint(SUMMARY_JOB_CHECKPOINT_KEY, String(now));
-  log.debug("Scheduled periodic global summary jobs");
-}
-
 function buildSegmentId(messageId: string, segmentIndex: number): string {
   return `${messageId}:${segmentIndex}`;
 }

package/src/memory/invite-store.ts

@@ -41,6 +41,8 @@ export interface IngressInvite {
   // Display metadata for personalized voice prompts (null for non-voice invites)
   friendName: string | null;
   guardianName: string | null;
+  // Contact binding — every invite is bound to a specific contact
+  contactId: string;
   createdAt: number;
   updatedAt: number;
 }
@@ -86,6 +88,7 @@ function rowToInvite(
     inviteCodeHash: row.inviteCodeHash,
     friendName: row.friendName,
     guardianName: row.guardianName,
+    contactId: row.contactId,
     createdAt: row.createdAt,
     updatedAt: row.updatedAt,
   };
@@ -97,6 +100,7 @@ function rowToInvite(
 
 export function createInvite(params: {
   sourceChannel: string;
+  contactId: string;
   createdBySessionId?: string;
   note?: string;
   maxUses?: number;
@@ -135,6 +139,7 @@ export function createInvite(params: {
     inviteCodeHash: params.inviteCodeHash ?? null,
     friendName: params.friendName ?? null,
     guardianName: params.guardianName ?? null,
+    contactId: params.contactId,
     createdAt: now,
     updatedAt: now,
   };
@@ -312,6 +317,20 @@ export function findByTokenHash(tokenHash: string): IngressInvite | null {
   return row ? rowToInvite(row) : null;
 }
 
+// ---------------------------------------------------------------------------
+// findById
+// ---------------------------------------------------------------------------
+
+export function findById(inviteId: string): IngressInvite | null {
+  const db = getDb();
+  const row = db
+    .select()
+    .from(assistantIngressInvites)
+    .where(eq(assistantIngressInvites.id, inviteId))
+    .get();
+  return row ? rowToInvite(row) : null;
+}
+
 // ---------------------------------------------------------------------------
 // findActiveVoiceInvites
 // ---------------------------------------------------------------------------