@vellumai/assistant 0.7.1 → 0.7.3

package/src/memory/graph/retriever.ts

@@ -14,6 +14,7 @@ import {
 } from "../../providers/provider-send-message.js";
 import type { ContentBlock, ImageContent } from "../../providers/types.js";
 import { getLogger } from "../../util/logger.js";
+import { isMemoryV2ReadActive } from "../context-search/sources/memory-v2.js";
 import { embedWithRetry } from "../embed.js";
 import {
   generateSparseEmbedding,
@@ -350,7 +351,7 @@ async function dedupCrossCategory(
 // Context load — conversation start
 // ---------------------------------------------------------------------------
 
-export interface ContextLoadOpts {
+interface ContextLoadOpts {
   /** Scope for memory isolation. */
   scopeId: string;
   /** Recent conversation summaries (used as retrieval queries). */
@@ -372,7 +373,7 @@ export interface ContextLoadOpts {
   userQuery?: string;
 }
 
-export interface ContextLoadResult {
+interface ContextLoadResult {
   nodes: ScoredNode[];
   serendipityNodes: ScoredNode[];
   triggeredNodes: TriggeredResult[];
@@ -425,6 +426,33 @@ export interface ContextLoadResult {
 export async function loadContextMemory(
   opts: ContextLoadOpts,
 ): Promise<ContextLoadResult> {
+  // v2 owns the read path when both gates are on. The v1 collection is in
+  // active retirement and querying it can OOM-crash Qdrant via a corrupted
+  // sparse segment, so we skip the embedding work and downstream searches
+  // entirely. Caller (`runContextLoad`) sees zero nodes and routes to the
+  // v2 activation pipeline.
+  if (isMemoryV2ReadActive(opts.config)) {
+    return {
+      nodes: [],
+      serendipityNodes: [],
+      triggeredNodes: [],
+      latencyMs: 0,
+      metrics: {
+        semanticHits: 0,
+        mergedCount: 0,
+        selectedCount: 0,
+        tier1Count: 0,
+        tier2Count: 0,
+        hybridSearchLatencyMs: 0,
+        sparseVectorUsed: false,
+        embeddingProvider: null,
+        embeddingModel: null,
+        queryContext: null,
+        topCandidates: [],
+      },
+    };
+  }
+
   const start = Date.now();
   const ctxLoadCfg = opts.config.memory.retrieval.injection.contextLoad;
   const maxNodes = opts.maxNodes ?? ctxLoadCfg.maxNodes;
@@ -497,7 +525,6 @@ export async function loadContextMemory(
       const results = await searchGraphNodes(
         queryVector,
         maxNodes * 3,
-        [opts.scopeId],
         sparseVector,
       );
       for (const r of results) {
@@ -520,7 +547,6 @@ export async function loadContextMemory(
       const results = await searchGraphNodes(
         userQueryVector,
         maxNodes * 3,
-        [opts.scopeId],
         undefined,
       );
       for (const r of results) {
@@ -845,7 +871,7 @@ export async function loadContextMemory(
 // Per-turn retrieval — mid-conversation injection
 // ---------------------------------------------------------------------------
 
-export interface TurnRetrievalOpts {
+interface TurnRetrievalOpts {
   /** The assistant's last message content. */
   assistantLastMessage: string;
   /** The user's last message content. */
@@ -858,7 +884,7 @@ export interface TurnRetrievalOpts {
   signal?: AbortSignal;
 }
 
-export interface TurnRetrievalResult {
+interface TurnRetrievalResult {
   /** New nodes to inject (not already in context). */
   nodes: ScoredNode[];
   /** Serendipity picks included in nodes. */
@@ -979,9 +1005,7 @@ export async function retrieveForTurn(
           }
           const imgVector = imgResult.vectors[0];
           if (imgVector) {
-            const imgResults = await searchGraphNodes(imgVector, 40, [
-              opts.scopeId,
-            ]);
+            const imgResults = await searchGraphNodes(imgVector, 40);
             for (const r of imgResults) {
               const current = allCandidateIds.get(r.nodeId) ?? 0;
               allCandidateIds.set(r.nodeId, Math.max(current, r.score));
@@ -1069,7 +1093,7 @@ export async function retrieveForTurn(
       queryEmbeddings = embedResults.vectors;
 
       const searchPromises = queryEmbeddings.map((vec) =>
-        searchGraphNodes(vec, 40, [opts.scopeId]),
+        searchGraphNodes(vec, 40),
       );
       const searchResults = await Promise.all(searchPromises);
 

package/src/memory/graph/tools.ts

@@ -76,7 +76,7 @@ export const graphRememberDefinition: ToolDefinition = {
       finish_turn: {
         type: "boolean",
         description:
-          "Set to true ONLY on the final `remember` call when you have nothing else to say and want to hand control back to the user. When true, the assistant turn ends after this tool call and no further LLM call is made. Do NOT set true on intermediate `remember` calls. Default: false.",
+          "When you have nothing else to say and want to hand control back to the user you MUST set this to true. When true, your turn ends after this tool call. It's critical that you do this in order to avoid unnecessary LLM calls.",
       },
     },
     required: ["content"],

package/src/memory/indexer.ts

@@ -9,6 +9,7 @@ import { getLogger } from "../util/logger.js";
 import { enqueueAutoAnalysisIfEnabled } from "./auto-analysis-enqueue.js";
 import { isAutoAnalysisConversation } from "./auto-analysis-guard.js";
 import { getMemoryCheckpoint, setMemoryCheckpoint } from "./checkpoints.js";
+import { isMemoryV2ReadActive } from "./context-search/sources/memory-v2.js";
 import { getDb } from "./db-connection.js";
 import { selectedBackendSupportsMultimodal } from "./embedding-backend.js";
 import { enqueueMemoryJob, upsertDebouncedJob } from "./jobs-store.js";
@@ -175,39 +176,9 @@ export async function indexMessageNow(
     // Summaries still run — they feed the graph retrieval pipeline and
     // are not recursion-prone.
     if (!isAutoAnalysisSource) {
-      // ── Graph extraction ────────────────────────────────────────────
-      const graphPendingKey = `graph_extract:${input.conversationId}:pending_count`;
-      const graphCurrentVal = getMemoryCheckpoint(graphPendingKey);
-      const graphPendingCount =
-        (graphCurrentVal ? parseInt(graphCurrentVal, 10) : 0) + 1;
-      setMemoryCheckpoint(graphPendingKey, String(graphPendingCount));
-
-      const graphBatchFired = graphPendingCount >= batchSize;
-      if (graphBatchFired) {
-        setMemoryCheckpoint(graphPendingKey, "0");
-      }
-
-      // Single pending `graph_extract` row per conversation. If the
-      // batch threshold just fired, pull `runAfter` back to now so the
-      // job runs immediately; otherwise debounce by the idle timeout.
-      // Routing both paths through `upsertDebouncedJob` ensures the
-      // row's `runAfter` reflects whichever trigger ran last, so a
-      // batch crossing always takes effect immediately.
-      const extractRunAfter = graphBatchFired
-        ? Date.now()
-        : Date.now() + idleTimeoutMs;
-      upsertDebouncedJob(
-        "graph_extract",
-        {
-          conversationId: input.conversationId,
-          scopeId: input.scopeId ?? "default",
-        },
-        extractRunAfter,
-      );
-
-      // Reading config here is best-effort: feature-gated triggers below
-      // (memory v2 sweep, auto-analyze batch) skip when it fails — the
-      // idle-debounced enqueues above are unaffected.
+      // Reading config here is best-effort: when it fails we treat v2 as
+      // inactive (failing-open to v1) so a config error never silently
+      // drops both extraction paths.
       let triggerConfig: ReturnType<typeof getConfig> | null = null;
       try {
         triggerConfig = getConfig();
@@ -218,20 +189,58 @@ export async function indexMessageNow(
         );
       }
 
-      // Memory v2 sweep mirrors graph_extract's debounce: when the v2
-      // flag + config are on AND `sweep_enabled` is set, every extraction
-      // trigger also enqueues a sweep. The sweep itself reads recent
-      // messages globally, so the `conversationId` here is just the dedup
-      // key — one pending row per active conversation. All three gates
-      // (feature flag, v2 master toggle, sweep_enabled) must be true.
+      const v2Config =
+        triggerConfig != null && isMemoryV2ReadActive(triggerConfig)
+          ? triggerConfig
+          : null;
+
+      // ── Graph extraction (v1) ───────────────────────────────────────
+      // Suppressed when v2 is active — v2 reads memory from buffer.md
+      // and concept pages, so the v1 graph would be stale data nobody
+      // consumes. Pending-count tracking is suppressed too; otherwise a
+      // flag flip back to v1 would fire an immediate batch from counts
+      // accumulated during the v2 window.
+      let extractRunAfter: number;
+      if (v2Config == null) {
+        const graphPendingKey = `graph_extract:${input.conversationId}:pending_count`;
+        const graphCurrentVal = getMemoryCheckpoint(graphPendingKey);
+        const graphPendingCount =
+          (graphCurrentVal ? parseInt(graphCurrentVal, 10) : 0) + 1;
+        setMemoryCheckpoint(graphPendingKey, String(graphPendingCount));
+
+        const graphBatchFired = graphPendingCount >= batchSize;
+        if (graphBatchFired) {
+          setMemoryCheckpoint(graphPendingKey, "0");
+        }
+
+        // Single pending `graph_extract` row per conversation. If the
+        // batch threshold just fired, pull `runAfter` back to now so the
+        // job runs immediately; otherwise debounce by the idle timeout.
+        // Routing both paths through `upsertDebouncedJob` ensures the
+        // row's `runAfter` reflects whichever trigger ran last, so a
+        // batch crossing always takes effect immediately.
+        extractRunAfter = graphBatchFired
+          ? Date.now()
+          : Date.now() + idleTimeoutMs;
+        upsertDebouncedJob(
+          "graph_extract",
+          {
+            conversationId: input.conversationId,
+            scopeId: input.scopeId ?? "default",
+          },
+          extractRunAfter,
+        );
+      } else {
+        extractRunAfter = Date.now() + idleTimeoutMs;
+      }
+
+      // Memory v2 sweep: when v2 is on AND `sweep_enabled` is set, every
+      // extraction trigger also enqueues a sweep. The sweep itself reads
+      // recent messages globally, so the `conversationId` here is just
+      // the dedup key — one pending row per active conversation.
       // `sweep_enabled` defaults to false because `remember()` is the
       // primary capture path; the sweep is opt-in.
-      if (
-        triggerConfig != null &&
-        isAssistantFeatureFlagEnabled("memory-v2-enabled", triggerConfig) &&
-        triggerConfig.memory.v2.enabled &&
-        triggerConfig.memory.v2.sweep_enabled
-      ) {
+      if (v2Config != null && v2Config.memory.v2.sweep_enabled) {
         upsertDebouncedJob(
           "memory_v2_sweep",
           { conversationId: input.conversationId },

package/src/memory/job-handlers/backfill.ts

@@ -7,10 +7,7 @@ import {
   resetMessageCursorCheckpoint,
   writeMessageCursorCheckpoint,
 } from "../checkpoints.js";
-import {
-  getConversationMemoryScopeId,
-  messageMetadataSchema,
-} from "../conversation-crud.js";
+import { messageMetadataSchema } from "../conversation-crud.js";
 import { getDb } from "../db-connection.js";
 import { indexMessageNow } from "../indexer.js";
 import { enqueueMemoryJob, type MemoryJob } from "../jobs-store.js";
@@ -72,13 +69,7 @@ export async function backfillJob(
     .all();
 
   if (batch.length > 0) {
-    const scopeCache = new Map<string, string>();
     for (const message of batch) {
-      let scopeId = scopeCache.get(message.conversationId);
-      if (scopeId === undefined) {
-        scopeId = getConversationMemoryScopeId(message.conversationId);
-        scopeCache.set(message.conversationId, scopeId);
-      }
       const { provenanceTrustClass, automated } = parseMessageMetadata(
         message.metadata ?? null,
       );
@@ -89,7 +80,7 @@ export async function backfillJob(
           role: message.role,
           content: message.content,
           createdAt: message.createdAt,
-          scopeId,
+          scopeId: "default",
           provenanceTrustClass,
           automated,
         },

package/src/memory/job-handlers/cleanup.ts

@@ -54,6 +54,49 @@ export function pruneOldLlmRequestLogsJob(
   );
 }
 
+/**
+ * Delete trace events older than the configured retention period.
+ * Processes in batches to avoid long DB locks and excessive WAL growth.
+ * Re-enqueues itself if more rows remain.
+ */
+export function pruneOldTraceEventsJob(
+  job: MemoryJob,
+  config: AssistantConfig,
+): void {
+  const rawRetention = job.payload.retentionDays;
+  const retentionDays =
+    typeof rawRetention === "number" &&
+    Number.isFinite(rawRetention) &&
+    rawRetention >= 0
+      ? rawRetention
+      : config.memory.cleanup.traceEventRetentionDays;
+
+  // 0 means disabled
+  if (retentionDays === 0) return;
+
+  const cutoffMs = Date.now() - retentionDays * 86_400_000;
+
+  rawRun(
+    `DELETE FROM trace_events WHERE rowid IN (SELECT rowid FROM trace_events WHERE created_at < ? LIMIT ?)`,
+    cutoffMs,
+    PRUNE_LOG_BATCH_LIMIT,
+  );
+  const deleted = rawChanges();
+
+  if (deleted >= PRUNE_LOG_BATCH_LIMIT) {
+    enqueueMemoryJob("prune_old_trace_events", { retentionDays });
+  }
+
+  log.info(
+    {
+      deleted,
+      retentionDays,
+      cutoffMs,
+    },
+    "Pruned old trace events",
+  );
+}
+
 /**
  * Delete conversations that have had no activity (updatedAt) for longer than
  * the configured retention period. Processes in batches so a single job doesn't

package/src/memory/job-handlers/embedding.ts

@@ -3,7 +3,6 @@ import { readFile } from "node:fs/promises";
 import { eq } from "drizzle-orm";
 
 import type { AssistantConfig } from "../../config/types.js";
-import { getConversationMemoryScopeId } from "../conversation-crud.js";
 import { getDb } from "../db-connection.js";
 import type { EmbeddingInput } from "../embedding-types.js";
 import { asString, embedAndUpsert } from "../job-utils.js";
@@ -18,7 +17,7 @@ import {
 
 export async function embedSegmentJob(
   job: MemoryJob,
-  config: AssistantConfig
+  config: AssistantConfig,
 ): Promise<void> {
   const segmentId = asString(job.payload.segmentId);
   if (!segmentId) return;
@@ -39,7 +38,7 @@ export async function embedSegmentJob(
 
 export async function embedSummaryJob(
   job: MemoryJob,
-  config: AssistantConfig
+  config: AssistantConfig,
 ): Promise<void> {
   const summaryId = asString(job.payload.summaryId);
   if (!summaryId) return;
@@ -60,13 +59,13 @@ export async function embedSummaryJob(
       created_at: summary.startAt,
       last_seen_at: summary.endAt,
       memory_scope_id: summary.scopeId,
-    }
+    },
   );
 }
 
 export async function embedMediaJob(
   job: MemoryJob,
-  config: AssistantConfig
+  config: AssistantConfig,
 ): Promise<void> {
   const assetId = asString(job.payload.assetId);
   if (!assetId) return;
@@ -99,7 +98,7 @@ export async function embedMediaJob(
 
 export async function embedAttachmentJob(
   job: MemoryJob,
-  config: AssistantConfig
+  config: AssistantConfig,
 ): Promise<void> {
   const messageId = asString(job.payload.messageId);
   const blockIndex = job.payload.blockIndex as number;
@@ -125,11 +124,10 @@ export async function embedAttachmentJob(
 
   // Use messageId + blockIndex as targetId for uniqueness
   const targetId = `${messageId}:${blockIndex}`;
-  const memoryScopeId = getConversationMemoryScopeId(message.conversationId);
   await embedAndUpsert(config, "media", targetId, input, {
     created_at: message.createdAt,
     message_id: messageId,
     conversation_id: message.conversationId,
-    memory_scope_id: memoryScopeId,
+    memory_scope_id: "default",
   });
 }

package/src/memory/job-handlers/summarization.ts

@@ -10,7 +10,6 @@ import {
   userMessage,
 } from "../../providers/provider-send-message.js";
 import { getLogger } from "../../util/logger.js";
-import { getConversationMemoryScopeId } from "../conversation-crud.js";
 import { getDb } from "../db-connection.js";
 import { asString, truncate } from "../job-utils.js";
 import { enqueueMemoryJob, type MemoryJob } from "../jobs-store.js";
@@ -91,10 +90,6 @@ export async function buildConversationSummaryJob(
     "conversation",
   );
 
-  // Inherit the conversation's memory scope so summaries stay aligned with
-  // the retrieval scope used by the source conversation.
-  const scopeId = getConversationMemoryScopeId(conversationId);
-
   const now = Date.now();
   const summaryId = existing?.id ?? uuid();
   const nextVersion = (existing?.version ?? 0) + 1;
@@ -108,7 +103,7 @@ export async function buildConversationSummaryJob(
       id: summaryId,
       scope: "conversation",
       scopeKey: conversationId,
-      scopeId,
+      scopeId: "default",
       summary: summaryText,
       tokenEstimate: estimateTextTokens(summaryText),
       version: nextVersion,
@@ -123,7 +118,7 @@ export async function buildConversationSummaryJob(
         summary: summaryText,
         tokenEstimate: estimateTextTokens(summaryText),
         version: sql`${memorySummaries.version} + 1`,
-        scopeId,
+        scopeId: "default",
         startAt: earliestCovered,
         endAt: latestCovered,
         updatedAt: now,

package/src/memory/jobs/__tests__/embed-concept-page.test.ts

@@ -103,6 +103,12 @@ mock.module("../../v2/qdrant.js", () => ({
   deleteConceptPageEmbedding: async (slug: string) => {
     deleteCalls.push(slug);
   },
+  // Other exports from the real module — stubbed so transitive imports
+  // don't crash on missing names when the mock replaces the module wholesale.
+  hybridQueryConceptPages: async () => [],
+  _resetMemoryV2QdrantForTests: () => {},
+  ensureConceptPageCollection: async () => {},
+  MEMORY_V2_COLLECTION: "memory_v2_concept_pages",
 }));
 
 // ── Workspace setup ────────────────────────────────────────────────
@@ -321,7 +327,7 @@ describe("enqueueEmbedConceptPageJob", () => {
     const id = enqueueEmbedConceptPageJob({ slug: "alice-prefers-vs-code" });
     expect(id).toBeTruthy();
 
-    const claimed = claimMemoryJobs(10);
+    const claimed = claimMemoryJobs({ slowLlm: 10, fast: 10, embed: 10 });
     expect(claimed).toHaveLength(1);
     const [job] = claimed;
     expect(job.type).toBe("embed_concept_page");
@@ -337,7 +343,7 @@ describe("enqueueEmbedConceptPageJob", () => {
 
     enqueueEmbedConceptPageJob({ slug: "round-trip-slug" });
 
-    const claimed = claimMemoryJobs(10);
+    const claimed = claimMemoryJobs({ slowLlm: 10, fast: 10, embed: 10 });
     expect(claimed).toHaveLength(1);
     const [job] = claimed;
     expect(job.type).toBe("embed_concept_page");

package/src/memory/jobs/embed-concept-page.ts

@@ -24,6 +24,7 @@ import type { AssistantConfig } from "../../config/types.js";
 import { BackendUnavailableError } from "../../util/errors.js";
 import { getLogger } from "../../util/logger.js";
 import { getWorkspaceDir } from "../../util/platform.js";
+import { applyCorrectionIfCalibrated } from "../anisotropy.js";
 import { getDb } from "../db-connection.js";
 import {
   embedWithBackend,
@@ -39,6 +40,10 @@ import {
   deleteConceptPageEmbedding,
   upsertConceptPageEmbedding,
 } from "../v2/qdrant.js";
+import {
+  generateBm25DocEmbedding,
+  getConceptPageCorpusStats,
+} from "../v2/sparse-bm25.js";
 
 const log = getLogger("memory-v2-embed-concept-page");
 
@@ -141,7 +146,17 @@ export async function embedConceptPageJob(
 
   // Sparse is cheap (in-process tokenization) and changes any time the body
   // changes, so we always recompute it rather than caching alongside dense.
-  const sparse = generateSparseEmbedding(text);
+  // BM25 weights live on the doc side; queries embed binary occurrence in
+  // sim.ts. When corpus stats aren't built yet (cold daemon, walking the
+  // corpus for the first time), fall back to the legacy TF-only encoding —
+  // the next reembed pass overwrites the page once stats are available.
+  const corpusStats = getConceptPageCorpusStats();
+  const sparse = corpusStats
+    ? generateBm25DocEmbedding(text, corpusStats, {
+        k1: config.memory.v2.bm25_k1,
+        b: config.memory.v2.bm25_b,
+      })
+    : generateSparseEmbedding(text);
 
   const now = Date.now();
   // Persist freshly embedded vectors for cross-restart reuse. On cache hit
@@ -189,9 +204,20 @@ export async function embedConceptPageJob(
     }
   }
 
+  // Apply anisotropy correction at the boundary between the (raw) cached
+  // dense vector and the Qdrant collection. Storing raw in SQLite and
+  // corrected in Qdrant means a recalibration just needs a reembed pass —
+  // the cache survives and the (cheap) correction math reruns over each
+  // cached vector. Pass-through when no calibration is fit yet.
+  const correctedDense = await applyCorrectionIfCalibrated(
+    dense,
+    provider,
+    model,
+  );
+
   await upsertConceptPageEmbedding({
     slug,
-    dense,
+    dense: correctedDense,
     sparse,
     updatedAt: now,
   });

package/src/memory/jobs/embed-pkb-file.test.ts

@@ -134,7 +134,7 @@ describe("enqueuePkbIndexJob", () => {
     });
     expect(id).toBeTruthy();
 
-    const claimed = claimMemoryJobs(10);
+    const claimed = claimMemoryJobs({ slowLlm: 10, fast: 10, embed: 10 });
     expect(claimed).toHaveLength(1);
     const [job] = claimed;
     const expectedType: MemoryJobType = "embed_pkb_file";
@@ -153,7 +153,7 @@ describe("enqueuePkbIndexJob", () => {
       memoryScopeId: "scope-rt",
     });
 
-    const claimed = claimMemoryJobs(10);
+    const claimed = claimMemoryJobs({ slowLlm: 10, fast: 10, embed: 10 });
     expect(claimed).toHaveLength(1);
     const [job] = claimed;
     expect(job.type).toBe("embed_pkb_file");