@vellumai/assistant 0.7.1 → 0.7.3

package/src/memory/jobs-store.ts

@@ -18,6 +18,7 @@ export type MemoryJobType =
   | "embed_summary"
   | "prune_old_conversations"
   | "prune_old_llm_request_logs"
+  | "prune_old_trace_events"
   | "build_conversation_summary"
   | "conversation_analyze"
   | "backfill"
@@ -43,7 +44,7 @@ export type MemoryJobType =
   | "memory_v2_reembed"
   | "memory_v2_activation_recompute";
 
-const EMBED_JOB_TYPES: MemoryJobType[] = [
+export const EMBED_JOB_TYPES: MemoryJobType[] = [
   "embed_segment",
   "embed_summary",
   "embed_media",
@@ -53,6 +54,21 @@ const EMBED_JOB_TYPES: MemoryJobType[] = [
   "graph_trigger_embed",
 ];
 
+export const SLOW_LLM_JOB_TYPES: MemoryJobType[] = [
+  "graph_consolidate",
+  "graph_pattern_scan",
+  "graph_narrative_refine",
+  "graph_extract",
+  "conversation_analyze",
+  "build_conversation_summary",
+  "generate_conversation_starters",
+  "memory_v2_sweep",
+  "memory_v2_consolidate",
+  "memory_v2_migrate",
+  "backfill",
+  "graph_bootstrap",
+];
+
 export interface MemoryJob<T = Record<string, unknown>> {
   id: string;
   type: MemoryJobType;
@@ -351,8 +367,62 @@ export function enqueuePruneOldConversationsJob(
   return enqueueMemoryJob("prune_old_conversations", payload);
 }
 
-export function claimMemoryJobs(limit: number): MemoryJob[] {
-  if (limit <= 0) return [];
+export function enqueuePruneOldTraceEventsJob(retentionDays?: number): string {
+  const db = getDb();
+  const existing = db
+    .select()
+    .from(memoryJobs)
+    .where(
+      and(
+        eq(memoryJobs.type, "prune_old_trace_events"),
+        inArray(memoryJobs.status, ["pending", "running"]),
+      ),
+    )
+    .orderBy(asc(memoryJobs.createdAt))
+    .get();
+  if (existing) {
+    if (
+      existing.status === "pending" &&
+      typeof retentionDays === "number" &&
+      Number.isFinite(retentionDays) &&
+      retentionDays >= 0
+    ) {
+      let payload: Record<string, unknown> = {};
+      try {
+        payload = JSON.parse(existing.payload) as Record<string, unknown>;
+      } catch {
+        payload = {};
+      }
+      if (payload.retentionDays !== retentionDays) {
+        db.update(memoryJobs)
+          .set({
+            payload: JSON.stringify({ ...payload, retentionDays }),
+            updatedAt: Date.now(),
+          })
+          .where(eq(memoryJobs.id, existing.id))
+          .run();
+      }
+    }
+    return existing.id;
+  }
+  const payload =
+    typeof retentionDays === "number" &&
+    Number.isFinite(retentionDays) &&
+    retentionDays >= 0
+      ? { retentionDays }
+      : {};
+  return enqueueMemoryJob("prune_old_trace_events", payload);
+}
+
+export interface LaneBudgets {
+  slowLlm: number;
+  fast: number;
+  embed: number;
+}
+
+export function claimMemoryJobs(limits: LaneBudgets): MemoryJob[] {
+  if (limits.slowLlm <= 0 && limits.fast <= 0 && limits.embed <= 0) return [];
+
   const db = getDb();
   const now = Date.now();
   const pendingFilter = and(
@@ -360,38 +430,60 @@ export function claimMemoryJobs(limit: number): MemoryJob[] {
     lte(memoryJobs.runAfter, now),
   );
 
-  // Claim non-embed jobs first, then fill remaining slots with embed jobs.
-  // This prevents embed retries from starving other job types during a backend outage.
-  const nonEmbedCandidates = db
-    .select()
-    .from(memoryJobs)
-    .where(and(pendingFilter, notInArray(memoryJobs.type, EMBED_JOB_TYPES)))
-    .orderBy(asc(memoryJobs.runAfter), asc(memoryJobs.createdAt))
-    .limit(limit)
-    .all();
+  // Slow lane: long-running LLM jobs (graph extract/consolidate, analysis, etc.).
+  const slowCandidates =
+    limits.slowLlm > 0
+      ? db
+          .select()
+          .from(memoryJobs)
+          .where(
+            and(pendingFilter, inArray(memoryJobs.type, SLOW_LLM_JOB_TYPES)),
+          )
+          .orderBy(asc(memoryJobs.runAfter), asc(memoryJobs.createdAt))
+          .limit(limits.slowLlm)
+          .all()
+      : [];
 
-  const remainingSlots = limit - nonEmbedCandidates.length;
+  // Fast lane: everything that is neither slow-LLM nor embed.
+  const fastCandidates =
+    limits.fast > 0
+      ? db
+          .select()
+          .from(memoryJobs)
+          .where(
+            and(
+              pendingFilter,
+              notInArray(memoryJobs.type, SLOW_LLM_JOB_TYPES),
+              notInArray(memoryJobs.type, EMBED_JOB_TYPES),
+            ),
+          )
+          .orderBy(asc(memoryJobs.runAfter), asc(memoryJobs.createdAt))
+          .limit(limits.fast)
+          .all()
+      : [];
 
-  // When the Qdrant circuit breaker is open, skip embed jobs entirely —
-  // they would just be claimed → fail → deferred, wasting CPU cycles.
-  // Exception: if the cooldown has elapsed (breaker ready for half-open probe),
-  // allow exactly 1 embed job through so the breaker can self-heal.
+  // Embed lane: gated by the Qdrant circuit breaker. When the breaker is open,
+  // skip embed jobs entirely — they would just be claimed → fail → deferred,
+  // wasting CPU cycles. Exception: if the cooldown has elapsed (breaker ready
+  // for half-open probe), allow exactly 1 embed job through so the breaker
+  // can self-heal. Note: this gate applies ONLY to the embed lane; slow and
+  // fast lanes run unimpeded.
   const breakerOpen = isQdrantBreakerOpen();
   const probeAllowed = breakerOpen && shouldAllowQdrantProbe();
   const skipEmbedJobs = breakerOpen && !probeAllowed;
-  const embedLimit = probeAllowed ? 1 : remainingSlots;
+  const embedLimit = probeAllowed ? Math.min(1, limits.embed) : limits.embed;
 
-  if (skipEmbedJobs && remainingSlots > 0) {
+  if (skipEmbedJobs && limits.embed > 0) {
     log.debug("Skipping embed job claims — Qdrant circuit breaker is open");
   }
-  if (probeAllowed && remainingSlots > 0) {
+  if (probeAllowed && limits.embed > 0) {
     log.debug(
       "Allowing 1 embed probe job — Qdrant circuit breaker cooldown elapsed",
     );
   }
 
   const embedCandidates =
-    remainingSlots > 0 && !skipEmbedJobs
+    embedLimit > 0 && !skipEmbedJobs
       ? db
           .select()
           .from(memoryJobs)
@@ -401,7 +493,7 @@ export function claimMemoryJobs(limit: number): MemoryJob[] {
           .all()
       : [];
 
-  const candidates = [...nonEmbedCandidates, ...embedCandidates];
+  const candidates = [...slowCandidates, ...fastCandidates, ...embedCandidates];
 
   const claimed: MemoryJob[] = [];
   for (const row of candidates) {

package/src/memory/jobs-worker.ts

@@ -1,12 +1,17 @@
-import { isAssistantFeatureFlagEnabled } from "../config/assistant-feature-flags.js";
 import { getConfig } from "../config/loader.js";
 import type { AssistantConfig } from "../config/types.js";
+import {
+  checkDiskPressureBackgroundGate,
+  diskPressureBackgroundSkipLogFields,
+  shouldLogDiskPressureBackgroundSkip,
+} from "../daemon/disk-pressure-background-gate.js";
 import { getLogger } from "../util/logger.js";
 import { getMemoryCheckpoint, setMemoryCheckpoint } from "./checkpoints.js";
 import {
   getLastScheduledCleanupEnqueueMs,
   markScheduledCleanupEnqueued,
 } from "./cleanup-schedule-state.js";
+import { isMemoryV2ReadActive } from "./context-search/sources/memory-v2.js";
 import { conversationAnalyzeJob } from "./conversation-analyze-job.js";
 import { maybeRunDbMaintenance } from "./db-maintenance.js";
 import { bootstrapFromHistory } from "./graph/bootstrap.js";
@@ -23,6 +28,7 @@ import { backfillJob } from "./job-handlers/backfill.js";
 import {
   pruneOldConversationsJob,
   pruneOldLlmRequestLogsJob,
+  pruneOldTraceEventsJob,
 } from "./job-handlers/cleanup.js";
 import { generateConversationStartersJob } from "./job-handlers/conversation-starters.js";
 // ── Per-job-type handlers ──────────────────────────────────────────
@@ -50,14 +56,17 @@ import {
   claimMemoryJobs,
   completeMemoryJob,
   deferMemoryJob,
+  EMBED_JOB_TYPES,
   enqueueMemoryJob,
   enqueuePruneOldConversationsJob,
   enqueuePruneOldLlmRequestLogsJob,
+  enqueuePruneOldTraceEventsJob,
   failMemoryJob,
   failStalledJobs,
   type MemoryJob,
   type MemoryJobType,
   resetRunningJobsToPending,
+  SLOW_LLM_JOB_TYPES,
 } from "./jobs-store.js";
 import { QdrantCircuitOpenError } from "./qdrant-circuit-breaker.js";
 import {
@@ -156,23 +165,49 @@ export function startMemoryJobsWorker(): MemoryJobsWorker {
   };
 }
 
-async function runMemoryJobsOnce(
+type ProcessGroup = (group: MemoryJob[]) => Promise<number>;
+
+export async function runMemoryJobsOnce(
   options: { enableScheduledCleanup?: boolean } = {},
 ): Promise<number> {
   const config = getConfig();
   if (!config.memory.enabled) return 0;
   const enableScheduledCleanup = options.enableScheduledCleanup === true;
 
+  const diskPressureGate = checkDiskPressureBackgroundGate("background-work");
+  if (diskPressureGate.action === "skip") {
+    if (shouldLogDiskPressureBackgroundSkip("memory-jobs-worker")) {
+      log.warn(
+        {
+          source: "memory",
+          ...diskPressureBackgroundSkipLogFields(diskPressureGate),
+        },
+        "Memory jobs worker skipped during disk pressure cleanup mode",
+      );
+    }
+    return 0;
+  }
+
   // Fail jobs that have been running longer than the configured timeout
   const timedOut = failStalledJobs(config.memory.jobs.stalledJobTimeoutMs);
   if (timedOut > 0) {
     log.warn({ timedOut }, "Timed out stalled memory jobs");
   }
 
-  const batchSize = Math.max(1, config.memory.jobs.batchSize);
-  const concurrency = Math.max(1, config.memory.jobs.workerConcurrency);
-  const jobs = claimMemoryJobs(batchSize);
-  if (jobs.length === 0) {
+  const cfgSlow = Math.max(1, config.memory.jobs.slowLlmConcurrency);
+  const cfgFast = Math.max(1, config.memory.jobs.fastConcurrency);
+  const cfgEmbed = Math.max(1, config.memory.jobs.embedConcurrency);
+
+  // Claim per-lane budgets so a backlog of slow LLM jobs cannot starve fast
+  // jobs (and vice versa). The Qdrant circuit breaker still gates only the
+  // embed lane inside `claimMemoryJobs`.
+  const claimed = claimMemoryJobs({
+    slowLlm: cfgSlow,
+    fast: cfgFast,
+    embed: cfgEmbed,
+  });
+
+  if (claimed.length === 0) {
     if (enableScheduledCleanup) {
       maybeEnqueueScheduledCleanupJobs(config);
     }
@@ -181,34 +216,22 @@ async function runMemoryJobsOnce(
     return 0;
   }
 
-  // Group jobs so they can run concurrently across independent work units.
-  // Jobs targeting different conversations (via payload.conversationId) are
-  // placed in separate groups and can run in parallel. Jobs targeting the
-  // same conversation, or global jobs without a conversationId (backfill,
-  // cleanup, rebuild_index), are grouped together and run sequentially to
-  // prevent checkpoint races.
-  const jobGroups = new Map<string, MemoryJob[]>();
-  for (const job of jobs) {
-    const convId =
-      typeof job.payload.conversationId === "string"
-        ? job.payload.conversationId
-        : null;
-    const groupKey = convId ? `${job.type}:${convId}` : job.type;
-    let group = jobGroups.get(groupKey);
-    if (!group) {
-      group = [];
-      jobGroups.set(groupKey, group);
+  const slowSet = new Set<MemoryJobType>(SLOW_LLM_JOB_TYPES);
+  const embedSet = new Set<MemoryJobType>(EMBED_JOB_TYPES);
+  const slowJobs: MemoryJob[] = [];
+  const fastJobs: MemoryJob[] = [];
+  const embedJobs: MemoryJob[] = [];
+  for (const job of claimed) {
+    if (slowSet.has(job.type)) {
+      slowJobs.push(job);
+    } else if (embedSet.has(job.type)) {
+      embedJobs.push(job);
+    } else {
+      fastJobs.push(job);
     }
-    group.push(job);
   }
 
-  let processed = 0;
-  const typeGroups = [...jobGroups.values()];
-
-  // Run type groups concurrently using a task pool (up to workerConcurrency
-  // active at a time). Unlike the old wave approach, a new group starts as
-  // soon as any slot frees up — no waiting for an entire wave to finish.
-  const processGroup = async (group: MemoryJob[]): Promise<number> => {
+  const processGroup: ProcessGroup = async (group) => {
     let groupProcessed = 0;
     for (const job of group) {
       try {
@@ -229,8 +252,59 @@ async function runMemoryJobsOnce(
     return groupProcessed;
   };
 
+  // Run all three lanes in parallel. Each lane runs its own bounded task pool
+  // so a slow `graph_consolidate` cannot block embed or fast jobs from making
+  // progress, and per-`(type, conversationId)` grouping inside each lane keeps
+  // same-conversation jobs serialized.
+  const [slowProcessed, fastProcessed, embedProcessed] = await Promise.all([
+    runLanePool(slowJobs, cfgSlow, processGroup),
+    runLanePool(fastJobs, cfgFast, processGroup),
+    runLanePool(embedJobs, cfgEmbed, processGroup),
+  ]);
+
+  if (enableScheduledCleanup) {
+    maybeEnqueueScheduledCleanupJobs(config);
+  }
+  maybeEnqueueGraphMaintenanceJobs(config);
+  maybeRunDbMaintenance();
+  return slowProcessed + fastProcessed + embedProcessed;
+}
+
+/**
+ * Run a single lane's jobs through a bounded task pool of size `concurrency`.
+ *
+ * Jobs targeting different conversations (via payload.conversationId) are
+ * placed in separate groups and run in parallel up to the lane's concurrency
+ * cap. Jobs targeting the same conversation — or global jobs without a
+ * conversationId — share a group and run sequentially to avoid checkpoint
+ * races.
+ */
+async function runLanePool(
+  jobs: MemoryJob[],
+  concurrency: number,
+  processGroup: ProcessGroup,
+): Promise<number> {
+  if (jobs.length === 0) return 0;
+
+  const groups = new Map<string, MemoryJob[]>();
+  for (const job of jobs) {
+    const convId =
+      typeof job.payload.conversationId === "string"
+        ? job.payload.conversationId
+        : null;
+    const groupKey = convId ? `${job.type}:${convId}` : job.type;
+    let group = groups.get(groupKey);
+    if (!group) {
+      group = [];
+      groups.set(groupKey, group);
+    }
+    group.push(job);
+  }
+
+  let processed = 0;
+  const typeGroups = [...groups.values()];
+
   if (typeGroups.length <= concurrency) {
-    // Fast path: all groups fit within the concurrency limit
     const results = await Promise.allSettled(typeGroups.map(processGroup));
     for (const result of results) {
       if (result.status === "fulfilled") {
@@ -242,39 +316,35 @@ async function runMemoryJobsOnce(
         );
       }
     }
-  } else {
-    // Task pool: maintain `concurrency` in-flight groups at all times
-    let nextIdx = 0;
-
-    const startNext = (): Promise<void> | undefined => {
-      if (nextIdx >= typeGroups.length) return undefined;
-      const group = typeGroups[nextIdx++]!;
-      return processGroup(group)
-        .then(
-          (count) => {
-            processed += count;
-          },
-          (err) => {
-            log.error(
-              { err },
-              "Memory job group rejected unexpectedly — jobs in this batch may have been dropped",
-            );
-          },
-        )
-        .then(() => startNext());
-    };
-
-    const workers = Array.from(
-      { length: Math.min(concurrency, typeGroups.length) },
-      () => startNext()!,
-    );
-    await Promise.all(workers);
-  }
-  if (enableScheduledCleanup) {
-    maybeEnqueueScheduledCleanupJobs(config);
+    return processed;
   }
-  maybeEnqueueGraphMaintenanceJobs(config);
-  maybeRunDbMaintenance();
+
+  // Task pool: keep `concurrency` groups in flight at all times so a new group
+  // starts the instant any slot frees up.
+  let nextIdx = 0;
+  const startNext = (): Promise<void> | undefined => {
+    if (nextIdx >= typeGroups.length) return undefined;
+    const group = typeGroups[nextIdx++]!;
+    return processGroup(group)
+      .then(
+        (count) => {
+          processed += count;
+        },
+        (err) => {
+          log.error(
+            { err },
+            "Memory job group rejected unexpectedly — jobs in this batch may have been dropped",
+          );
+        },
+      )
+      .then(() => startNext());
+  };
+
+  const workers = Array.from(
+    { length: Math.min(concurrency, typeGroups.length) },
+    () => startNext()!,
+  );
+  await Promise.all(workers);
   return processed;
 }
 
@@ -406,6 +476,9 @@ async function processJob(
     case "prune_old_llm_request_logs":
       pruneOldLlmRequestLogsJob(job, config);
       return;
+    case "prune_old_trace_events":
+      pruneOldTraceEventsJob(job, config);
+      return;
     case "build_conversation_summary":
       await buildConversationSummaryJob(job, config);
       return;
@@ -511,14 +584,20 @@ function maybeEnqueueScheduledCleanupJobs(
     cleanup.llmRequestLogRetentionMs !== null
       ? enqueuePruneOldLlmRequestLogsJob(cleanup.llmRequestLogRetentionMs)
       : null;
+  const pruneTraceEventsJobId =
+    cleanup.traceEventRetentionDays > 0
+      ? enqueuePruneOldTraceEventsJob(cleanup.traceEventRetentionDays)
+      : null;
   markScheduledCleanupEnqueued(nowMs);
   log.debug(
     {
       pruneConversationsJobId,
       pruneLlmRequestLogsJobId,
+      pruneTraceEventsJobId,
       enqueueIntervalMs: cleanup.enqueueIntervalMs,
       conversationRetentionDays: cleanup.conversationRetentionDays,
       llmRequestLogRetentionMs: cleanup.llmRequestLogRetentionMs,
+      traceEventRetentionDays: cleanup.traceEventRetentionDays,
     },
     "Enqueued scheduled memory cleanup jobs",
   );
@@ -541,58 +620,66 @@ export const GRAPH_MAINTENANCE_CHECKPOINTS = {
 } as const;
 
 /**
- * Enqueue periodic graph maintenance jobs (decay, consolidation, pattern scan, narrative).
- * Uses durable checkpoints so intervals survive daemon restarts — jobs only fire
- * when the actual elapsed time since last run exceeds the interval.
+ * Enqueue periodic graph maintenance jobs.
+ *
+ * Mutually exclusive between v1 and v2:
+ *   - v2 active (both `memory-v2-enabled` flag and `memory.v2.enabled`
+ *     config on) → only `memory_v2_consolidate` is scheduled.
+ *   - v2 inactive → the four v1 entries (decay, consolidate, pattern_scan,
+ *     narrative) are scheduled instead.
  *
- * The v2 consolidation entry is gated on both the `memory-v2-enabled` feature
- * flag and the `memory.v2.enabled` config — both must be true for the cron
- * to fire. Sweep is intentionally not on this schedule: it is debounced from
- * the live `graph_extract` trigger path (see `indexMessageNow` in
- * `indexer.ts`) so it runs on the same idle/message-count cadence.
+ * Read/write paths route to v2 when the flag is on, so v1 graph data goes
+ * unread; running v1 maintenance alongside v2 is wasted compute and LLM
+ * spend. The v1 code path remains live so flipping the flag back to off
+ * fully re-engages v1.
+ *
+ * Uses durable checkpoints so intervals survive daemon restarts — jobs only
+ * fire when the actual elapsed time since last run exceeds the interval.
+ * Sweep is intentionally not on this schedule: it is debounced from the
+ * live `graph_extract` trigger path (see `indexMessageNow` in `indexer.ts`)
+ * so it runs on the same idle/message-count cadence.
  */
 export function maybeEnqueueGraphMaintenanceJobs(
   config: AssistantConfig,
   nowMs = Date.now(),
 ): void {
+  const v2Active = isMemoryV2ReadActive(config);
+
   const schedule: Array<{
     key: string;
     intervalMs: number;
     jobType: MemoryJobType;
-  }> = [
-    {
-      key: GRAPH_MAINTENANCE_CHECKPOINTS.decay,
-      intervalMs: GRAPH_DECAY_INTERVAL_MS,
-      jobType: "graph_decay",
-    },
-    {
-      key: GRAPH_MAINTENANCE_CHECKPOINTS.consolidate,
-      intervalMs: GRAPH_CONSOLIDATE_INTERVAL_MS,
-      jobType: "graph_consolidate",
-    },
-    {
-      key: GRAPH_MAINTENANCE_CHECKPOINTS.patternScan,
-      intervalMs: GRAPH_PATTERN_SCAN_INTERVAL_MS,
-      jobType: "graph_pattern_scan",
-    },
-    {
-      key: GRAPH_MAINTENANCE_CHECKPOINTS.narrative,
-      intervalMs: GRAPH_NARRATIVE_INTERVAL_MS,
-      jobType: "graph_narrative_refine",
-    },
-  ];
-
-  if (
-    isAssistantFeatureFlagEnabled("memory-v2-enabled", config) &&
-    config.memory.v2.enabled
-  ) {
-    schedule.push({
-      key: GRAPH_MAINTENANCE_CHECKPOINTS.memoryV2Consolidate,
-      intervalMs:
-        config.memory.v2.consolidation_interval_hours * 60 * 60 * 1000,
-      jobType: "memory_v2_consolidate",
-    });
-  }
+  }> = v2Active
+    ? [
+        {
+          key: GRAPH_MAINTENANCE_CHECKPOINTS.memoryV2Consolidate,
+          intervalMs:
+            config.memory.v2.consolidation_interval_hours * 60 * 60 * 1000,
+          jobType: "memory_v2_consolidate",
+        },
+      ]
+    : [
+        {
+          key: GRAPH_MAINTENANCE_CHECKPOINTS.decay,
+          intervalMs: GRAPH_DECAY_INTERVAL_MS,
+          jobType: "graph_decay",
+        },
+        {
+          key: GRAPH_MAINTENANCE_CHECKPOINTS.consolidate,
+          intervalMs: GRAPH_CONSOLIDATE_INTERVAL_MS,
+          jobType: "graph_consolidate",
+        },
+        {
+          key: GRAPH_MAINTENANCE_CHECKPOINTS.patternScan,
+          intervalMs: GRAPH_PATTERN_SCAN_INTERVAL_MS,
+          jobType: "graph_pattern_scan",
+        },
+        {
+          key: GRAPH_MAINTENANCE_CHECKPOINTS.narrative,
+          intervalMs: GRAPH_NARRATIVE_INTERVAL_MS,
+          jobType: "graph_narrative_refine",
+        },
+      ];
 
   for (const { key, intervalMs, jobType } of schedule) {
     const lastRun = parseInt(getMemoryCheckpoint(key) ?? "0", 10);