@vellumai/assistant 0.7.0 → 0.7.2

package/src/memory/jobs/embed-concept-page.ts

@@ -24,6 +24,7 @@ import type { AssistantConfig } from "../../config/types.js";
 import { BackendUnavailableError } from "../../util/errors.js";
 import { getLogger } from "../../util/logger.js";
 import { getWorkspaceDir } from "../../util/platform.js";
+import { applyCorrectionIfCalibrated } from "../anisotropy.js";
 import { getDb } from "../db-connection.js";
 import {
   embedWithBackend,
@@ -39,6 +40,10 @@ import {
   deleteConceptPageEmbedding,
   upsertConceptPageEmbedding,
 } from "../v2/qdrant.js";
+import {
+  generateBm25DocEmbedding,
+  getConceptPageCorpusStats,
+} from "../v2/sparse-bm25.js";
 
 const log = getLogger("memory-v2-embed-concept-page");
 
@@ -141,7 +146,17 @@ export async function embedConceptPageJob(
 
   // Sparse is cheap (in-process tokenization) and changes any time the body
   // changes, so we always recompute it rather than caching alongside dense.
-  const sparse = generateSparseEmbedding(text);
+  // BM25 weights live on the doc side; queries embed binary occurrence in
+  // sim.ts. When corpus stats aren't built yet (cold daemon, walking the
+  // corpus for the first time), fall back to the legacy TF-only encoding —
+  // the next reembed pass overwrites the page once stats are available.
+  const corpusStats = getConceptPageCorpusStats();
+  const sparse = corpusStats
+    ? generateBm25DocEmbedding(text, corpusStats, {
+        k1: config.memory.v2.bm25_k1,
+        b: config.memory.v2.bm25_b,
+      })
+    : generateSparseEmbedding(text);
 
   const now = Date.now();
   // Persist freshly embedded vectors for cross-restart reuse. On cache hit
@@ -189,9 +204,20 @@ export async function embedConceptPageJob(
     }
   }
 
+  // Apply anisotropy correction at the boundary between the (raw) cached
+  // dense vector and the Qdrant collection. Storing raw in SQLite and
+  // corrected in Qdrant means a recalibration just needs a reembed pass —
+  // the cache survives and the (cheap) correction math reruns over each
+  // cached vector. Pass-through when no calibration is fit yet.
+  const correctedDense = await applyCorrectionIfCalibrated(
+    dense,
+    provider,
+    model,
+  );
+
   await upsertConceptPageEmbedding({
     slug,
-    dense,
+    dense: correctedDense,
     sparse,
     updatedAt: now,
   });

package/src/memory/jobs/embed-pkb-file.test.ts

@@ -134,7 +134,7 @@ describe("enqueuePkbIndexJob", () => {
     });
     expect(id).toBeTruthy();
 
-    const claimed = claimMemoryJobs(10);
+    const claimed = claimMemoryJobs({ slowLlm: 10, fast: 10, embed: 10 });
     expect(claimed).toHaveLength(1);
     const [job] = claimed;
     const expectedType: MemoryJobType = "embed_pkb_file";
@@ -153,7 +153,7 @@ describe("enqueuePkbIndexJob", () => {
       memoryScopeId: "scope-rt",
     });
 
-    const claimed = claimMemoryJobs(10);
+    const claimed = claimMemoryJobs({ slowLlm: 10, fast: 10, embed: 10 });
     expect(claimed).toHaveLength(1);
     const [job] = claimed;
     expect(job.type).toBe("embed_pkb_file");

package/src/memory/jobs-store.ts

@@ -40,11 +40,10 @@ export type MemoryJobType =
   | "memory_v2_sweep"
   | "memory_v2_consolidate"
   | "memory_v2_migrate"
-  | "memory_v2_rebuild_edges"
   | "memory_v2_reembed"
   | "memory_v2_activation_recompute";
 
-const EMBED_JOB_TYPES: MemoryJobType[] = [
+export const EMBED_JOB_TYPES: MemoryJobType[] = [
   "embed_segment",
   "embed_summary",
   "embed_media",
@@ -54,6 +53,21 @@ const EMBED_JOB_TYPES: MemoryJobType[] = [
   "graph_trigger_embed",
 ];
 
+export const SLOW_LLM_JOB_TYPES: MemoryJobType[] = [
+  "graph_consolidate",
+  "graph_pattern_scan",
+  "graph_narrative_refine",
+  "graph_extract",
+  "conversation_analyze",
+  "build_conversation_summary",
+  "generate_conversation_starters",
+  "memory_v2_sweep",
+  "memory_v2_consolidate",
+  "memory_v2_migrate",
+  "backfill",
+  "graph_bootstrap",
+];
+
 export interface MemoryJob<T = Record<string, unknown>> {
   id: string;
   type: MemoryJobType;
@@ -352,8 +366,15 @@ export function enqueuePruneOldConversationsJob(
   return enqueueMemoryJob("prune_old_conversations", payload);
 }
 
-export function claimMemoryJobs(limit: number): MemoryJob[] {
-  if (limit <= 0) return [];
+export interface LaneBudgets {
+  slowLlm: number;
+  fast: number;
+  embed: number;
+}
+
+export function claimMemoryJobs(limits: LaneBudgets): MemoryJob[] {
+  if (limits.slowLlm <= 0 && limits.fast <= 0 && limits.embed <= 0) return [];
+
   const db = getDb();
   const now = Date.now();
   const pendingFilter = and(
@@ -361,38 +382,60 @@ export function claimMemoryJobs(limit: number): MemoryJob[] {
     lte(memoryJobs.runAfter, now),
   );
 
-  // Claim non-embed jobs first, then fill remaining slots with embed jobs.
-  // This prevents embed retries from starving other job types during a backend outage.
-  const nonEmbedCandidates = db
-    .select()
-    .from(memoryJobs)
-    .where(and(pendingFilter, notInArray(memoryJobs.type, EMBED_JOB_TYPES)))
-    .orderBy(asc(memoryJobs.runAfter), asc(memoryJobs.createdAt))
-    .limit(limit)
-    .all();
+  // Slow lane: long-running LLM jobs (graph extract/consolidate, analysis, etc.).
+  const slowCandidates =
+    limits.slowLlm > 0
+      ? db
+          .select()
+          .from(memoryJobs)
+          .where(
+            and(pendingFilter, inArray(memoryJobs.type, SLOW_LLM_JOB_TYPES)),
+          )
+          .orderBy(asc(memoryJobs.runAfter), asc(memoryJobs.createdAt))
+          .limit(limits.slowLlm)
+          .all()
+      : [];
 
-  const remainingSlots = limit - nonEmbedCandidates.length;
+  // Fast lane: everything that is neither slow-LLM nor embed.
+  const fastCandidates =
+    limits.fast > 0
+      ? db
+          .select()
+          .from(memoryJobs)
+          .where(
+            and(
+              pendingFilter,
+              notInArray(memoryJobs.type, SLOW_LLM_JOB_TYPES),
+              notInArray(memoryJobs.type, EMBED_JOB_TYPES),
+            ),
+          )
+          .orderBy(asc(memoryJobs.runAfter), asc(memoryJobs.createdAt))
+          .limit(limits.fast)
+          .all()
+      : [];
 
-  // When the Qdrant circuit breaker is open, skip embed jobs entirely —
-  // they would just be claimed → fail → deferred, wasting CPU cycles.
-  // Exception: if the cooldown has elapsed (breaker ready for half-open probe),
-  // allow exactly 1 embed job through so the breaker can self-heal.
+  // Embed lane: gated by the Qdrant circuit breaker. When the breaker is open,
+  // skip embed jobs entirely — they would just be claimed → fail → deferred,
+  // wasting CPU cycles. Exception: if the cooldown has elapsed (breaker ready
+  // for half-open probe), allow exactly 1 embed job through so the breaker
+  // can self-heal. Note: this gate applies ONLY to the embed lane; slow and
+  // fast lanes run unimpeded.
   const breakerOpen = isQdrantBreakerOpen();
   const probeAllowed = breakerOpen && shouldAllowQdrantProbe();
   const skipEmbedJobs = breakerOpen && !probeAllowed;
-  const embedLimit = probeAllowed ? 1 : remainingSlots;
+  const embedLimit = probeAllowed ? Math.min(1, limits.embed) : limits.embed;
 
-  if (skipEmbedJobs && remainingSlots > 0) {
+  if (skipEmbedJobs && limits.embed > 0) {
     log.debug("Skipping embed job claims — Qdrant circuit breaker is open");
   }
-  if (probeAllowed && remainingSlots > 0) {
+  if (probeAllowed && limits.embed > 0) {
     log.debug(
       "Allowing 1 embed probe job — Qdrant circuit breaker cooldown elapsed",
     );
   }
 
   const embedCandidates =
-    remainingSlots > 0 && !skipEmbedJobs
+    embedLimit > 0 && !skipEmbedJobs
       ? db
           .select()
           .from(memoryJobs)
@@ -402,7 +445,7 @@ export function claimMemoryJobs(limit: number): MemoryJob[] {
           .all()
       : [];
 
-  const candidates = [...nonEmbedCandidates, ...embedCandidates];
+  const candidates = [...slowCandidates, ...fastCandidates, ...embedCandidates];
 
   const claimed: MemoryJob[] = [];
   for (const row of candidates) {

package/src/memory/jobs-worker.ts

@@ -6,7 +6,6 @@ import { getMemoryCheckpoint, setMemoryCheckpoint } from "./checkpoints.js";
 import {
   getLastScheduledCleanupEnqueueMs,
   markScheduledCleanupEnqueued,
-  resetCleanupScheduleThrottle as resetCleanupScheduleThrottleImpl,
 } from "./cleanup-schedule-state.js";
 import { conversationAnalyzeJob } from "./conversation-analyze-job.js";
 import { maybeRunDbMaintenance } from "./db-maintenance.js";
@@ -51,6 +50,7 @@ import {
   claimMemoryJobs,
   completeMemoryJob,
   deferMemoryJob,
+  EMBED_JOB_TYPES,
   enqueueMemoryJob,
   enqueuePruneOldConversationsJob,
   enqueuePruneOldLlmRequestLogsJob,
@@ -59,12 +59,12 @@ import {
   type MemoryJob,
   type MemoryJobType,
   resetRunningJobsToPending,
+  SLOW_LLM_JOB_TYPES,
 } from "./jobs-store.js";
 import { QdrantCircuitOpenError } from "./qdrant-circuit-breaker.js";
 import {
   memoryV2ActivationRecomputeJob,
   memoryV2MigrateJob,
-  memoryV2RebuildEdgesJob,
   memoryV2ReembedJob,
 } from "./v2/backfill-jobs.js";
 import { memoryV2ConsolidateJob } from "./v2/consolidation-job.js";
@@ -88,6 +88,7 @@ const LEGACY_JOB_TYPES = new Set([
   "journal_carry_forward",
   "generate_capability_cards",
   "generate_thread_starters",
+  "memory_v2_rebuild_edges",
 ]);
 
 export const POLL_INTERVAL_MIN_MS = 1_500;
@@ -157,7 +158,9 @@ export function startMemoryJobsWorker(): MemoryJobsWorker {
   };
 }
 
-async function runMemoryJobsOnce(
+type ProcessGroup = (group: MemoryJob[]) => Promise<number>;
+
+export async function runMemoryJobsOnce(
   options: { enableScheduledCleanup?: boolean } = {},
 ): Promise<number> {
   const config = getConfig();
@@ -170,10 +173,20 @@ async function runMemoryJobsOnce(
     log.warn({ timedOut }, "Timed out stalled memory jobs");
   }
 
-  const batchSize = Math.max(1, config.memory.jobs.batchSize);
-  const concurrency = Math.max(1, config.memory.jobs.workerConcurrency);
-  const jobs = claimMemoryJobs(batchSize);
-  if (jobs.length === 0) {
+  const cfgSlow = Math.max(1, config.memory.jobs.slowLlmConcurrency);
+  const cfgFast = Math.max(1, config.memory.jobs.fastConcurrency);
+  const cfgEmbed = Math.max(1, config.memory.jobs.embedConcurrency);
+
+  // Claim per-lane budgets so a backlog of slow LLM jobs cannot starve fast
+  // jobs (and vice versa). The Qdrant circuit breaker still gates only the
+  // embed lane inside `claimMemoryJobs`.
+  const claimed = claimMemoryJobs({
+    slowLlm: cfgSlow,
+    fast: cfgFast,
+    embed: cfgEmbed,
+  });
+
+  if (claimed.length === 0) {
     if (enableScheduledCleanup) {
       maybeEnqueueScheduledCleanupJobs(config);
     }
@@ -182,34 +195,22 @@ async function runMemoryJobsOnce(
     return 0;
   }
 
-  // Group jobs so they can run concurrently across independent work units.
-  // Jobs targeting different conversations (via payload.conversationId) are
-  // placed in separate groups and can run in parallel. Jobs targeting the
-  // same conversation, or global jobs without a conversationId (backfill,
-  // cleanup, rebuild_index), are grouped together and run sequentially to
-  // prevent checkpoint races.
-  const jobGroups = new Map<string, MemoryJob[]>();
-  for (const job of jobs) {
-    const convId =
-      typeof job.payload.conversationId === "string"
-        ? job.payload.conversationId
-        : null;
-    const groupKey = convId ? `${job.type}:${convId}` : job.type;
-    let group = jobGroups.get(groupKey);
-    if (!group) {
-      group = [];
-      jobGroups.set(groupKey, group);
+  const slowSet = new Set<MemoryJobType>(SLOW_LLM_JOB_TYPES);
+  const embedSet = new Set<MemoryJobType>(EMBED_JOB_TYPES);
+  const slowJobs: MemoryJob[] = [];
+  const fastJobs: MemoryJob[] = [];
+  const embedJobs: MemoryJob[] = [];
+  for (const job of claimed) {
+    if (slowSet.has(job.type)) {
+      slowJobs.push(job);
+    } else if (embedSet.has(job.type)) {
+      embedJobs.push(job);
+    } else {
+      fastJobs.push(job);
     }
-    group.push(job);
   }
 
-  let processed = 0;
-  const typeGroups = [...jobGroups.values()];
-
-  // Run type groups concurrently using a task pool (up to workerConcurrency
-  // active at a time). Unlike the old wave approach, a new group starts as
-  // soon as any slot frees up — no waiting for an entire wave to finish.
-  const processGroup = async (group: MemoryJob[]): Promise<number> => {
+  const processGroup: ProcessGroup = async (group) => {
     let groupProcessed = 0;
     for (const job of group) {
       try {
@@ -230,8 +231,59 @@ async function runMemoryJobsOnce(
     return groupProcessed;
   };
 
+  // Run all three lanes in parallel. Each lane runs its own bounded task pool
+  // so a slow `graph_consolidate` cannot block embed or fast jobs from making
+  // progress, and per-`(type, conversationId)` grouping inside each lane keeps
+  // same-conversation jobs serialized.
+  const [slowProcessed, fastProcessed, embedProcessed] = await Promise.all([
+    runLanePool(slowJobs, cfgSlow, processGroup),
+    runLanePool(fastJobs, cfgFast, processGroup),
+    runLanePool(embedJobs, cfgEmbed, processGroup),
+  ]);
+
+  if (enableScheduledCleanup) {
+    maybeEnqueueScheduledCleanupJobs(config);
+  }
+  maybeEnqueueGraphMaintenanceJobs(config);
+  maybeRunDbMaintenance();
+  return slowProcessed + fastProcessed + embedProcessed;
+}
+
+/**
+ * Run a single lane's jobs through a bounded task pool of size `concurrency`.
+ *
+ * Jobs targeting different conversations (via payload.conversationId) are
+ * placed in separate groups and run in parallel up to the lane's concurrency
+ * cap. Jobs targeting the same conversation — or global jobs without a
+ * conversationId — share a group and run sequentially to avoid checkpoint
+ * races.
+ */
+async function runLanePool(
+  jobs: MemoryJob[],
+  concurrency: number,
+  processGroup: ProcessGroup,
+): Promise<number> {
+  if (jobs.length === 0) return 0;
+
+  const groups = new Map<string, MemoryJob[]>();
+  for (const job of jobs) {
+    const convId =
+      typeof job.payload.conversationId === "string"
+        ? job.payload.conversationId
+        : null;
+    const groupKey = convId ? `${job.type}:${convId}` : job.type;
+    let group = groups.get(groupKey);
+    if (!group) {
+      group = [];
+      groups.set(groupKey, group);
+    }
+    group.push(job);
+  }
+
+  let processed = 0;
+  const typeGroups = [...groups.values()];
+
   if (typeGroups.length <= concurrency) {
-    // Fast path: all groups fit within the concurrency limit
     const results = await Promise.allSettled(typeGroups.map(processGroup));
     for (const result of results) {
       if (result.status === "fulfilled") {
@@ -243,39 +295,35 @@ async function runMemoryJobsOnce(
         );
       }
     }
-  } else {
-    // Task pool: maintain `concurrency` in-flight groups at all times
-    let nextIdx = 0;
-
-    const startNext = (): Promise<void> | undefined => {
-      if (nextIdx >= typeGroups.length) return undefined;
-      const group = typeGroups[nextIdx++]!;
-      return processGroup(group)
-        .then(
-          (count) => {
-            processed += count;
-          },
-          (err) => {
-            log.error(
-              { err },
-              "Memory job group rejected unexpectedly — jobs in this batch may have been dropped",
-            );
-          },
-        )
-        .then(() => startNext());
-    };
-
-    const workers = Array.from(
-      { length: Math.min(concurrency, typeGroups.length) },
-      () => startNext()!,
-    );
-    await Promise.all(workers);
+    return processed;
   }
-  if (enableScheduledCleanup) {
-    maybeEnqueueScheduledCleanupJobs(config);
-  }
-  maybeEnqueueGraphMaintenanceJobs(config);
-  maybeRunDbMaintenance();
+
+  // Task pool: keep `concurrency` groups in flight at all times so a new group
+  // starts the instant any slot frees up.
+  let nextIdx = 0;
+  const startNext = (): Promise<void> | undefined => {
+    if (nextIdx >= typeGroups.length) return undefined;
+    const group = typeGroups[nextIdx++]!;
+    return processGroup(group)
+      .then(
+        (count) => {
+          processed += count;
+        },
+        (err) => {
+          log.error(
+            { err },
+            "Memory job group rejected unexpectedly — jobs in this batch may have been dropped",
+          );
+        },
+      )
+      .then(() => startNext());
+  };
+
+  const workers = Array.from(
+    { length: Math.min(concurrency, typeGroups.length) },
+    () => startNext()!,
+  );
+  await Promise.all(workers);
   return processed;
 }
 
@@ -473,9 +521,6 @@ async function processJob(
     case "memory_v2_migrate":
       await memoryV2MigrateJob(job, config);
       return;
-    case "memory_v2_rebuild_edges":
-      await memoryV2RebuildEdgesJob(job, config);
-      return;
     case "memory_v2_reembed":
       await memoryV2ReembedJob(job, config);
       return;
@@ -494,16 +539,6 @@ async function processJob(
   }
 }
 
-// ── Cleanup scheduling ─────────────────────────────────────────────
-
-/**
- * Re-export of the shared throttle-reset helper. The underlying state lives
- * in cleanup-schedule-state.ts so that lighter-weight callers (e.g.
- * ConfigWatcher) can reset it without pulling in jobs-worker's transitive
- * imports.
- */
-export const resetCleanupScheduleThrottle = resetCleanupScheduleThrottleImpl;
-
 /**
  * Enqueue periodic cleanup jobs using config-driven retention windows.
  * Enqueue is deduped in jobs-store, so repeated calls remain safe.
@@ -546,7 +581,7 @@ const GRAPH_CONSOLIDATE_INTERVAL_MS = 4 * 60 * 60 * 1000; // 4 hours
 const GRAPH_PATTERN_SCAN_INTERVAL_MS = 24 * 60 * 60 * 1000; // 1 day
 const GRAPH_NARRATIVE_INTERVAL_MS = 7 * 24 * 60 * 60 * 1000; // 1 week
 
-const GRAPH_MAINTENANCE_CHECKPOINTS = {
+export const GRAPH_MAINTENANCE_CHECKPOINTS = {
   decay: "graph_maintenance:decay:last_run",
   consolidate: "graph_maintenance:consolidate:last_run",
   patternScan: "graph_maintenance:pattern_scan:last_run",

package/src/memory/llm-request-log-store.ts

@@ -43,32 +43,6 @@ export function recordRequestLog(
   return id;
 }
 
-export function queryRequestLogs(
-  conversationId: string,
-  startTime: number,
-  endTime: number,
-): Array<{
-  id: string;
-  conversationId: string;
-  requestPayload: string;
-  responsePayload: string;
-  createdAt: number;
-}> {
-  const db = getDb();
-  return db
-    .select()
-    .from(llmRequestLogs)
-    .where(
-      and(
-        eq(llmRequestLogs.conversationId, conversationId),
-        gte(llmRequestLogs.createdAt, startTime),
-        lte(llmRequestLogs.createdAt, endTime),
-      ),
-    )
-    .orderBy(llmRequestLogs.createdAt)
-    .all();
-}
-
 export function backfillMessageIdOnLogs(
   conversationId: string,
   messageId: string,
@@ -85,21 +59,6 @@ export function backfillMessageIdOnLogs(
     .run();
 }
 
-/**
- * Set `messageId` on specific log rows identified by their IDs.
- * Unlike `backfillMessageIdOnLogs` (which updates all null-messageId rows
- * for a conversation), this targets only the given log rows — safe for
- * concurrent watch/assistant turns.
- */
-export function setMessageIdOnLogs(logIds: string[], messageId: string): void {
-  if (logIds.length === 0) return;
-  const db = getDb();
-  db.update(llmRequestLogs)
-    .set({ messageId })
-    .where(inArray(llmRequestLogs.id, logIds))
-    .run();
-}
-
 /**
  * Re-link LLM request logs from a set of source message IDs to a target
  * message. Used during message consolidation so logs from deleted