@vellumai/assistant 0.4.49 → 0.4.51

package/src/memory/job-handlers/cleanup.ts

@@ -2,26 +2,15 @@ import { and, asc, eq, inArray, lt } from "drizzle-orm";
 
 import type { AssistantConfig } from "../../config/types.js";
 import { getLogger } from "../../util/logger.js";
-import { checkContradictions } from "../contradiction-checker.js";
 import { getDb, rawAll, rawRun } from "../db.js";
-import { asPositiveMs, asString } from "../job-utils.js";
+import { asPositiveMs } from "../job-utils.js";
 import { enqueueMemoryJob, type MemoryJob } from "../jobs-store.js";
-import {
-  memoryEmbeddings,
-  memoryItemEntities,
-  memoryItems,
-} from "../schema.js";
+import { memoryEmbeddings, memoryItems } from "../schema.js";
 
 const log = getLogger("memory-jobs-worker");
 
 const CLEANUP_BATCH_LIMIT = 250;
 
-export async function checkContradictionsJob(job: MemoryJob): Promise<void> {
-  const itemId = asString(job.payload.itemId);
-  if (!itemId) return;
-  await checkContradictions(itemId);
-}
-
 export function cleanupStaleSupersededItemsJob(
   job: MemoryJob,
   config: AssistantConfig,
@@ -46,9 +35,6 @@ export function cleanupStaleSupersededItemsJob(
   if (stale.length === 0) return;
 
   const ids = stale.map((row) => row.id);
-  db.delete(memoryItemEntities)
-    .where(inArray(memoryItemEntities.memoryItemId, ids))
-    .run();
   db.delete(memoryEmbeddings)
     .where(
       and(

package/src/memory/job-handlers/extraction.ts

@@ -1,21 +1,11 @@
 import { eq } from "drizzle-orm";
 
-import { getConfig } from "../../config/loader.js";
-import type { AssistantConfig } from "../../config/types.js";
 import { getLogger } from "../../util/logger.js";
 import { getDb } from "../db.js";
-import {
-  extractEntitiesWithLLM,
-  linkMemoryItemToEntity,
-  resolveEntityName,
-  upsertEntity,
-  upsertEntityRelation,
-} from "../entity-extractor.js";
 import { extractAndUpsertMemoryItemsForMessage } from "../items-extractor.js";
 import { asString } from "../job-utils.js";
-import { enqueueMemoryJob, type MemoryJob } from "../jobs-store.js";
-import { extractTextFromStoredMessageContent } from "../message-content.js";
-import { memoryItemSources, messages } from "../schema.js";
+import type { MemoryJob } from "../jobs-store.js";
+import { messages } from "../schema.js";
 import { isConversationFailed } from "../task-memory-cleanup.js";
 
 const log = getLogger("memory-jobs-worker");
@@ -47,130 +37,4 @@ export async function extractItemsJob(job: MemoryJob): Promise<void> {
     scopeId,
     msg?.conversationId,
   );
-  // Queue entity extraction for this message after items are extracted
-  const config = getConfig();
-  if (config.memory.entity.enabled) {
-    enqueueMemoryJob("extract_entities", { messageId, scopeId });
-  }
-}
-
-export async function extractEntitiesJob(
-  job: MemoryJob,
-  config: AssistantConfig,
-): Promise<void> {
-  const messageId = asString(job.payload.messageId);
-  if (!messageId) return;
-
-  const db = getDb();
-
-  // Guard: skip entity extraction for failed conversations. Entity extraction
-  // jobs are enqueued by extractItemsJob after items are extracted; while new
-  // jobs won't be queued (extractItemsJob returns early for failed convos),
-  // any entity jobs enqueued before the failure marker was set must be caught.
-  const msgRow = db
-    .select({ conversationId: messages.conversationId })
-    .from(messages)
-    .where(eq(messages.id, messageId))
-    .get();
-  if (msgRow && isConversationFailed(msgRow.conversationId)) {
-    log.info(
-      { messageId, conversationId: msgRow.conversationId },
-      "Skipping entity extraction for failed conversation",
-    );
-    return;
-  }
-
-  const message = db
-    .select({
-      id: messages.id,
-      content: messages.content,
-    })
-    .from(messages)
-    .where(eq(messages.id, messageId))
-    .get();
-  if (!message) return;
-
-  const text = extractTextFromStoredMessageContent(message.content);
-  if (text.trim().length < 15) return;
-
-  const extracted = await extractEntitiesWithLLM(text, config.memory.entity);
-  const entities = extracted.entities;
-  const relations = extracted.relations;
-  if (entities.length === 0 && relations.length === 0) return;
-
-  // Find all memory items linked to this message via memory_item_sources
-  const linkedItems = db
-    .select({ memoryItemId: memoryItemSources.memoryItemId })
-    .from(memoryItemSources)
-    .where(eq(memoryItemSources.messageId, messageId))
-    .all();
-  const itemIds = linkedItems.map((row) => row.memoryItemId);
-  const entityNameToId = new Map<string, string>();
-
-  for (const entity of entities) {
-    const entityId = upsertEntity(entity);
-    entityNameToId.set(entity.name.toLowerCase(), entityId);
-    for (const alias of entity.aliases) {
-      entityNameToId.set(alias.toLowerCase(), entityId);
-    }
-    // Link all memory items from this message to the entity
-    for (const itemId of itemIds) {
-      linkMemoryItemToEntity(itemId, entityId);
-    }
-  }
-
-  const relationTelemetry = {
-    attempted: 0,
-    parsed: relations.length,
-    persisted: 0,
-    dropped: 0,
-  };
-
-  if (config.memory.entity.extractRelations.enabled && relations.length > 0) {
-    const seenRelationKeys = new Set<string>();
-    for (const relation of relations) {
-      relationTelemetry.attempted += 1;
-      const sourceLookup = relation.sourceEntityName.toLowerCase();
-      const targetLookup = relation.targetEntityName.toLowerCase();
-      const sourceEntityId =
-        entityNameToId.get(sourceLookup) ??
-        resolveEntityName(relation.sourceEntityName);
-      const targetEntityId =
-        entityNameToId.get(targetLookup) ??
-        resolveEntityName(relation.targetEntityName);
-      if (
-        !sourceEntityId ||
-        !targetEntityId ||
-        sourceEntityId === targetEntityId
-      ) {
-        relationTelemetry.dropped += 1;
-        continue;
-      }
-
-      const dedupeKey = `${sourceEntityId}|${targetEntityId}|${relation.relation}`;
-      if (seenRelationKeys.has(dedupeKey)) continue;
-      seenRelationKeys.add(dedupeKey);
-
-      upsertEntityRelation({
-        sourceEntityId,
-        targetEntityId,
-        relation: relation.relation,
-        evidence: relation.evidence,
-      });
-      relationTelemetry.persisted += 1;
-    }
-  }
-
-  log.debug(
-    {
-      messageId,
-      entityCount: entities.length,
-      linkedItems: itemIds.length,
-      relationAttempts: relationTelemetry.attempted,
-      relationParsed: relationTelemetry.parsed,
-      relationPersisted: relationTelemetry.persisted,
-      relationDropped: relationTelemetry.dropped,
-    },
-    "Extracted entity graph from message",
-  );
 }

package/src/memory/job-handlers/index-maintenance.ts

@@ -2,7 +2,7 @@ import { eq, like } from "drizzle-orm";
 
 import { getConfig } from "../../config/loader.js";
 import { getLogger } from "../../util/logger.js";
-import { getDb, rawExec } from "../db.js";
+import { getDb } from "../db.js";
 import { selectedBackendSupportsMultimodal } from "../embedding-backend.js";
 import { asString, BackendUnavailableError } from "../job-utils.js";
 import { enqueueMemoryJob, type MemoryJob } from "../jobs-store.js";
@@ -22,11 +22,6 @@ const log = getLogger("memory-jobs-worker");
 
 export function rebuildIndexJob(): void {
   const db = getDb();
-  rawExec(/*sql*/ `DELETE FROM memory_segment_fts`);
-  rawExec(/*sql*/ `
-    INSERT INTO memory_segment_fts(segment_id, text)
-    SELECT id, text FROM memory_segments
-  `);
   db.delete(memoryEmbeddings).run();
 
   const items = db

package/src/memory/job-handlers/summarization.ts

@@ -1,4 +1,4 @@
-import { and, desc, eq, gte, isNull, lt, sql } from "drizzle-orm";
+import { and, desc, eq, sql } from "drizzle-orm";
 import { v4 as uuid } from "uuid";
 
 import type { AssistantConfig } from "../../config/types.js";
@@ -12,14 +12,9 @@ import {
 import { getLogger } from "../../util/logger.js";
 import { getConversationMemoryScopeId } from "../conversation-crud.js";
 import { getDb } from "../db.js";
-import {
-  asString,
-  currentMonthWindow,
-  currentWeekWindow,
-  truncate,
-} from "../job-utils.js";
+import { asString, truncate } from "../job-utils.js";
 import { enqueueMemoryJob, type MemoryJob } from "../jobs-store.js";
-import { memoryItems, memorySegments, memorySummaries } from "../schema.js";
+import { memorySegments, memorySummaries } from "../schema.js";
 
 const log = getLogger("memory-jobs-worker");
 
@@ -38,18 +33,6 @@ const CONVERSATION_SUMMARY_SYSTEM_PROMPT = [
   "- If updating an existing summary with new data, merge new information and remove anything that was superseded.",
 ].join("\n");
 
-const GLOBAL_SUMMARY_SYSTEM_PROMPT = [
-  "You are a memory summarization system. Your job is to synthesize a higher-level summary from multiple conversation summaries and memory items.",
-  "",
-  "Guidelines:",
-  "- Identify recurring themes, cross-cutting decisions, and persistent user preferences.",
-  "- Highlight the most important facts, active projects, and ongoing concerns.",
-  "- De-duplicate information that appears across multiple conversations.",
-  "- Use concise sections with bullet points.",
-  "- Target 400-600 tokens. Be dense but readable.",
-  "- If updating an existing summary with new data, merge new information and remove anything that was superseded.",
-].join("\n");
-
 export async function buildConversationSummaryJob(
   job: MemoryJob,
   config: AssistantConfig,
@@ -144,134 +127,6 @@ export async function buildConversationSummaryJob(
   }
 }
 
-export async function buildGlobalSummaryJob(
-  scope: "weekly_global" | "monthly_global",
-  config: AssistantConfig,
-): Promise<void> {
-  const db = getDb();
-  const now = new Date();
-  const { startMs, endMs, scopeKey } =
-    scope === "weekly_global"
-      ? currentWeekWindow(now)
-      : currentMonthWindow(now);
-
-  const items = db
-    .select()
-    .from(memoryItems)
-    .where(
-      and(
-        eq(memoryItems.status, "active"),
-        isNull(memoryItems.invalidAt),
-        eq(memoryItems.scopeId, "default"),
-        gte(memoryItems.lastSeenAt, startMs),
-        lt(memoryItems.lastSeenAt, endMs),
-      ),
-    )
-    .orderBy(desc(memoryItems.lastSeenAt))
-    .limit(80)
-    .all();
-
-  // Gather conversation summaries from this period for higher-level synthesis
-  const convSummaries = db
-    .select()
-    .from(memorySummaries)
-    .where(
-      and(
-        eq(memorySummaries.scope, "conversation"),
-        eq(memorySummaries.scopeId, "default"),
-        gte(memorySummaries.endAt, startMs),
-        lt(memorySummaries.startAt, endMs),
-      ),
-    )
-    .orderBy(desc(memorySummaries.endAt))
-    .limit(20)
-    .all();
-
-  if (items.length === 0 && convSummaries.length === 0) return;
-
-  // Build input for LLM: conversation summaries + active items
-  const parts: string[] = [];
-  if (convSummaries.length > 0) {
-    parts.push("## Conversation Summaries");
-    for (const cs of convSummaries) {
-      parts.push(`### ${cs.scopeKey}\n${truncate(cs.summary, 600)}`);
-    }
-  }
-  if (items.length > 0) {
-    parts.push("## Active Memory Items");
-    for (const item of items.slice(0, 40)) {
-      parts.push(
-        `- [${item.kind}] ${item.subject}: ${truncate(item.statement, 180)}`,
-      );
-    }
-  }
-  const inputText = parts.join("\n\n");
-
-  const existing = db
-    .select()
-    .from(memorySummaries)
-    .where(
-      and(
-        eq(memorySummaries.scope, scope),
-        eq(memorySummaries.scopeKey, scopeKey),
-      ),
-    )
-    .get();
-
-  const label = scope === "weekly_global" ? "weekly" : "monthly";
-  const summaryText = await summarizeWithLLM(
-    config,
-    GLOBAL_SUMMARY_SYSTEM_PROMPT,
-    existing?.summary ?? null,
-    inputText,
-    label,
-  );
-
-  const ts = Date.now();
-  const summaryId = existing?.id ?? uuid();
-  db.insert(memorySummaries)
-    .values({
-      id: summaryId,
-      scope,
-      scopeKey,
-      summary: summaryText,
-      tokenEstimate: estimateTextTokens(summaryText),
-      version: (existing?.version ?? 0) + 1,
-      startAt: startMs,
-      endAt: endMs,
-      createdAt: ts,
-      updatedAt: ts,
-    })
-    .onConflictDoUpdate({
-      target: [memorySummaries.scope, memorySummaries.scopeKey],
-      set: {
-        summary: summaryText,
-        tokenEstimate: estimateTextTokens(summaryText),
-        version: sql`${memorySummaries.version} + 1`,
-        startAt: startMs,
-        endAt: endMs,
-        updatedAt: ts,
-      },
-    })
-    .run();
-
-  // Re-query to get the actual persisted row ID — during a race the ON CONFLICT
-  // path keeps the winner's ID, not the pre-generated UUID from the loser.
-  const actualRow = db
-    .select({ id: memorySummaries.id })
-    .from(memorySummaries)
-    .where(
-      and(
-        eq(memorySummaries.scope, scope),
-        eq(memorySummaries.scopeKey, scopeKey),
-      ),
-    )
-    .get();
-  if (actualRow) {
-    enqueueMemoryJob("embed_summary", { summaryId: actualRow.id });
-  }
-}
-
 async function summarizeWithLLM(
   config: AssistantConfig,
   systemPrompt: string,

package/src/memory/job-utils.ts

@@ -8,6 +8,7 @@ import { getLogger } from "../util/logger.js";
 import { getDb } from "./db.js";
 import {
   embedWithBackend,
+  generateSparseEmbedding,
   getMemoryBackendStatus,
 } from "./embedding-backend.js";
 import type { EmbeddingInput } from "./embedding-types.js";
@@ -201,6 +202,14 @@ export async function embedAndUpsert(
       ? normalized.text
       : `[${normalized.type}:${normalized.mimeType}]`;
 
+  // Generate sparse embedding from the same source text used for dense embedding.
+  // For non-text (media) inputs, sparse vectors are skipped since tokenization
+  // only applies to text content.
+  const sparseVector =
+    normalized.type === "text"
+      ? generateSparseEmbedding(normalized.text)
+      : undefined;
+
   // Persist embedding in SQLite for cross-restart cache
   const now = Date.now();
   try {
@@ -249,12 +258,18 @@ export async function embedAndUpsert(
   try {
     const modality = normalized.type;
     await withQdrantBreaker(() =>
-      qdrant.upsert(targetType, targetId, vector, {
-        text: payloadText,
-        modality,
-        created_at: (extraPayload?.created_at as number) ?? now,
-        ...(extraPayload as Record<string, unknown> | undefined),
-      }),
+      qdrant.upsert(
+        targetType,
+        targetId,
+        vector,
+        {
+          text: payloadText,
+          modality,
+          created_at: (extraPayload?.created_at as number) ?? now,
+          ...(extraPayload as Record<string, unknown> | undefined),
+        },
+        sparseVector,
+      ),
     );
   } catch (err) {
     log.warn(
@@ -265,56 +280,3 @@ export async function embedAndUpsert(
   }
 }
 
-// ── Time window utilities ──────────────────────────────────────────
-
-export function currentWeekWindow(now: Date): {
-  scopeKey: string;
-  startMs: number;
-  endMs: number;
-} {
-  const day = (now.getUTCDay() + 6) % 7;
-  const start = new Date(
-    Date.UTC(
-      now.getUTCFullYear(),
-      now.getUTCMonth(),
-      now.getUTCDate() - day,
-      0,
-      0,
-      0,
-      0,
-    ),
-  );
-  const end = new Date(start);
-  end.setUTCDate(end.getUTCDate() + 7);
-  const scopeKey = `${start.getUTCFullYear()}-W${weekNumber(start)
-    .toString()
-    .padStart(2, "0")}`;
-  return { scopeKey, startMs: start.getTime(), endMs: end.getTime() };
-}
-
-export function currentMonthWindow(now: Date): {
-  scopeKey: string;
-  startMs: number;
-  endMs: number;
-} {
-  const start = new Date(
-    Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1, 0, 0, 0, 0),
-  );
-  const end = new Date(
-    Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + 1, 1, 0, 0, 0, 0),
-  );
-  const scopeKey = `${start.getUTCFullYear()}-${String(
-    start.getUTCMonth() + 1,
-  ).padStart(2, "0")}`;
-  return { scopeKey, startMs: start.getTime(), endMs: end.getTime() };
-}
-
-function weekNumber(date: Date): number {
-  const tmp = new Date(
-    Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()),
-  );
-  const dayNum = tmp.getUTCDay() || 7;
-  tmp.setUTCDate(tmp.getUTCDate() + 4 - dayNum);
-  const yearStart = new Date(Date.UTC(tmp.getUTCFullYear(), 0, 1));
-  return Math.ceil(((tmp.getTime() - yearStart.getTime()) / 86400000 + 1) / 7);
-}

package/src/memory/jobs-store.ts

@@ -3,7 +3,7 @@ import { v4 as uuid } from "uuid";
 
 import { getLogger } from "../util/logger.js";
 import { truncate } from "../util/truncate.js";
-import { getDb, rawAll, rawChanges, rawGet } from "./db.js";
+import { getDb, rawAll, rawChanges } from "./db.js";
 import { memoryJobs } from "./schema.js";
 
 const log = getLogger("memory-jobs-store");
@@ -14,18 +14,14 @@ export type MemoryJobType =
   | "embed_summary"
   | "extract_items"
   | "extract_entities"
-  | "resolve_pending_conflicts_for_message"
-  | "cleanup_resolved_conflicts"
   | "cleanup_stale_superseded_items"
   | "prune_old_conversations"
   | "backfill_entity_relations"
-  | "check_contradictions"
   | "refresh_weekly_summary"
   | "refresh_monthly_summary"
   | "build_conversation_summary"
   | "backfill"
   | "rebuild_index"
-  | "reconcile_fts"
   | "delete_qdrant_vectors"
   | "media_processing"
   | "embed_media"
@@ -83,143 +79,6 @@ export function enqueueMemoryJob(
   return id;
 }
 
-/**
- * Ensure there is only one pending relation backfill orchestrator job.
- * If `force=true` arrives while a pending job exists, its payload is upgraded.
- */
-export function enqueueBackfillEntityRelationsJob(force = false): string {
-  const db = getDb();
-  const now = Date.now();
-  const existing = db
-    .select()
-    .from(memoryJobs)
-    .where(
-      and(
-        eq(memoryJobs.type, "backfill_entity_relations"),
-        eq(memoryJobs.status, "pending"),
-      ),
-    )
-    .orderBy(asc(memoryJobs.createdAt))
-    .get();
-
-  if (existing) {
-    if (force) {
-      let payload: Record<string, unknown> = {};
-      try {
-        payload = JSON.parse(existing.payload) as Record<string, unknown>;
-      } catch {
-        payload = {};
-      }
-      if (payload.force !== true) {
-        db.update(memoryJobs)
-          .set({
-            payload: JSON.stringify({ ...payload, force: true }),
-            updatedAt: now,
-          })
-          .where(eq(memoryJobs.id, existing.id))
-          .run();
-      }
-    }
-    return existing.id;
-  }
-
-  return enqueueMemoryJob("backfill_entity_relations", { force });
-}
-
-export function enqueueResolvePendingConflictsForMessageJob(
-  messageId: string,
-  scopeId = "default",
-  dbOverride?: Parameters<ReturnType<typeof getDb>["transaction"]>[0] extends (
-    tx: infer T,
-  ) => unknown
-    ? T
-    : never,
-): string {
-  const normalizedMessageId = messageId.trim();
-  if (!normalizedMessageId) {
-    throw new Error(
-      "enqueueResolvePendingConflictsForMessageJob requires a non-empty messageId",
-    );
-  }
-  const normalizedScopeId = scopeId.trim() || "default";
-  // Dedup check always uses root db since tx doesn't expose raw client
-  const existing = rawGet<{ id: string }>(
-    `
-    SELECT id
-    FROM memory_jobs
-    WHERE type = 'resolve_pending_conflicts_for_message'
-      AND status IN ('pending', 'running')
-      AND json_extract(payload, '$.messageId') = ?
-      AND COALESCE(json_extract(payload, '$.scopeId'), 'default') = ?
-    ORDER BY created_at ASC
-    LIMIT 1
-  `,
-    normalizedMessageId,
-    normalizedScopeId,
-  );
-  if (existing?.id) return existing.id;
-
-  return enqueueMemoryJob(
-    "resolve_pending_conflicts_for_message",
-    {
-      messageId: normalizedMessageId,
-      scopeId: normalizedScopeId,
-    },
-    Date.now(),
-    dbOverride,
-  );
-}
-
-export function enqueueCleanupResolvedConflictsJob(
-  retentionMs?: number,
-): string {
-  const db = getDb();
-  const now = Date.now();
-  const existing = db
-    .select()
-    .from(memoryJobs)
-    .where(
-      and(
-        eq(memoryJobs.type, "cleanup_resolved_conflicts"),
-        inArray(memoryJobs.status, ["pending", "running"]),
-      ),
-    )
-    .orderBy(asc(memoryJobs.createdAt))
-    .get();
-  if (existing) {
-    if (
-      existing.status === "pending" &&
-      typeof retentionMs === "number" &&
-      Number.isFinite(retentionMs) &&
-      retentionMs > 0
-    ) {
-      let payload: Record<string, unknown> = {};
-      try {
-        payload = JSON.parse(existing.payload) as Record<string, unknown>;
-      } catch {
-        payload = {};
-      }
-      if (payload.retentionMs !== retentionMs) {
-        db.update(memoryJobs)
-          .set({
-            payload: JSON.stringify({ ...payload, retentionMs }),
-            updatedAt: now,
-          })
-          .where(eq(memoryJobs.id, existing.id))
-          .run();
-      }
-    }
-    return existing.id;
-  }
-  const payload =
-    typeof retentionMs === "number" &&
-    Number.isFinite(retentionMs) &&
-    retentionMs > 0
-      ? { retentionMs }
-      : {};
-  return enqueueMemoryJob("cleanup_resolved_conflicts", payload);
-}
-
 export function enqueueCleanupStaleSupersededItemsJob(
   retentionMs?: number,
 ): string {
@@ -319,23 +178,6 @@ export function enqueuePruneOldConversationsJob(
   return enqueueMemoryJob("prune_old_conversations", payload);
 }
 
-export function enqueueReconcileFtsJob(): string {
-  const db = getDb();
-  const existing = db
-    .select()
-    .from(memoryJobs)
-    .where(
-      and(
-        eq(memoryJobs.type, "reconcile_fts"),
-        inArray(memoryJobs.status, ["pending", "running"]),
-      ),
-    )
-    .orderBy(asc(memoryJobs.createdAt))
-    .get();
-  if (existing) return existing.id;
-  return enqueueMemoryJob("reconcile_fts", {});
-}
-
 export function claimMemoryJobs(limit: number): MemoryJob[] {
   if (limit <= 0) return [];
   const db = getDb();