@vellumai/assistant 0.7.1 → 0.7.2

package/src/memory/jobs-store.ts

@@ -43,7 +43,7 @@ export type MemoryJobType =
   | "memory_v2_reembed"
   | "memory_v2_activation_recompute";
 
-const EMBED_JOB_TYPES: MemoryJobType[] = [
+export const EMBED_JOB_TYPES: MemoryJobType[] = [
   "embed_segment",
   "embed_summary",
   "embed_media",
@@ -53,6 +53,21 @@ const EMBED_JOB_TYPES: MemoryJobType[] = [
   "graph_trigger_embed",
 ];
 
+export const SLOW_LLM_JOB_TYPES: MemoryJobType[] = [
+  "graph_consolidate",
+  "graph_pattern_scan",
+  "graph_narrative_refine",
+  "graph_extract",
+  "conversation_analyze",
+  "build_conversation_summary",
+  "generate_conversation_starters",
+  "memory_v2_sweep",
+  "memory_v2_consolidate",
+  "memory_v2_migrate",
+  "backfill",
+  "graph_bootstrap",
+];
+
 export interface MemoryJob<T = Record<string, unknown>> {
   id: string;
   type: MemoryJobType;
@@ -351,8 +366,15 @@ export function enqueuePruneOldConversationsJob(
   return enqueueMemoryJob("prune_old_conversations", payload);
 }
 
-export function claimMemoryJobs(limit: number): MemoryJob[] {
-  if (limit <= 0) return [];
+export interface LaneBudgets {
+  slowLlm: number;
+  fast: number;
+  embed: number;
+}
+
+export function claimMemoryJobs(limits: LaneBudgets): MemoryJob[] {
+  if (limits.slowLlm <= 0 && limits.fast <= 0 && limits.embed <= 0) return [];
+
   const db = getDb();
   const now = Date.now();
   const pendingFilter = and(
@@ -360,38 +382,60 @@ export function claimMemoryJobs(limit: number): MemoryJob[] {
     lte(memoryJobs.runAfter, now),
   );
 
-  // Claim non-embed jobs first, then fill remaining slots with embed jobs.
-  // This prevents embed retries from starving other job types during a backend outage.
-  const nonEmbedCandidates = db
-    .select()
-    .from(memoryJobs)
-    .where(and(pendingFilter, notInArray(memoryJobs.type, EMBED_JOB_TYPES)))
-    .orderBy(asc(memoryJobs.runAfter), asc(memoryJobs.createdAt))
-    .limit(limit)
-    .all();
+  // Slow lane: long-running LLM jobs (graph extract/consolidate, analysis, etc.).
+  const slowCandidates =
+    limits.slowLlm > 0
+      ? db
+          .select()
+          .from(memoryJobs)
+          .where(
+            and(pendingFilter, inArray(memoryJobs.type, SLOW_LLM_JOB_TYPES)),
+          )
+          .orderBy(asc(memoryJobs.runAfter), asc(memoryJobs.createdAt))
+          .limit(limits.slowLlm)
+          .all()
+      : [];
 
-  const remainingSlots = limit - nonEmbedCandidates.length;
+  // Fast lane: everything that is neither slow-LLM nor embed.
+  const fastCandidates =
+    limits.fast > 0
+      ? db
+          .select()
+          .from(memoryJobs)
+          .where(
+            and(
+              pendingFilter,
+              notInArray(memoryJobs.type, SLOW_LLM_JOB_TYPES),
+              notInArray(memoryJobs.type, EMBED_JOB_TYPES),
+            ),
+          )
+          .orderBy(asc(memoryJobs.runAfter), asc(memoryJobs.createdAt))
+          .limit(limits.fast)
+          .all()
+      : [];
 
-  // When the Qdrant circuit breaker is open, skip embed jobs entirely —
-  // they would just be claimed → fail → deferred, wasting CPU cycles.
-  // Exception: if the cooldown has elapsed (breaker ready for half-open probe),
-  // allow exactly 1 embed job through so the breaker can self-heal.
+  // Embed lane: gated by the Qdrant circuit breaker. When the breaker is open,
+  // skip embed jobs entirely — they would just be claimed → fail → deferred,
+  // wasting CPU cycles. Exception: if the cooldown has elapsed (breaker ready
+  // for half-open probe), allow exactly 1 embed job through so the breaker
+  // can self-heal. Note: this gate applies ONLY to the embed lane; slow and
+  // fast lanes run unimpeded.
   const breakerOpen = isQdrantBreakerOpen();
   const probeAllowed = breakerOpen && shouldAllowQdrantProbe();
   const skipEmbedJobs = breakerOpen && !probeAllowed;
-  const embedLimit = probeAllowed ? 1 : remainingSlots;
+  const embedLimit = probeAllowed ? Math.min(1, limits.embed) : limits.embed;
 
-  if (skipEmbedJobs && remainingSlots > 0) {
+  if (skipEmbedJobs && limits.embed > 0) {
     log.debug("Skipping embed job claims — Qdrant circuit breaker is open");
   }
-  if (probeAllowed && remainingSlots > 0) {
+  if (probeAllowed && limits.embed > 0) {
     log.debug(
       "Allowing 1 embed probe job — Qdrant circuit breaker cooldown elapsed",
     );
   }
 
   const embedCandidates =
-    remainingSlots > 0 && !skipEmbedJobs
+    embedLimit > 0 && !skipEmbedJobs
       ? db
           .select()
           .from(memoryJobs)
@@ -401,7 +445,7 @@ export function claimMemoryJobs(limit: number): MemoryJob[] {
           .all()
       : [];
 
-  const candidates = [...nonEmbedCandidates, ...embedCandidates];
+  const candidates = [...slowCandidates, ...fastCandidates, ...embedCandidates];
 
   const claimed: MemoryJob[] = [];
   for (const row of candidates) {

package/src/memory/jobs-worker.ts

@@ -50,6 +50,7 @@ import {
   claimMemoryJobs,
   completeMemoryJob,
   deferMemoryJob,
+  EMBED_JOB_TYPES,
   enqueueMemoryJob,
   enqueuePruneOldConversationsJob,
   enqueuePruneOldLlmRequestLogsJob,
@@ -58,6 +59,7 @@ import {
   type MemoryJob,
   type MemoryJobType,
   resetRunningJobsToPending,
+  SLOW_LLM_JOB_TYPES,
 } from "./jobs-store.js";
 import { QdrantCircuitOpenError } from "./qdrant-circuit-breaker.js";
 import {
@@ -156,7 +158,9 @@ export function startMemoryJobsWorker(): MemoryJobsWorker {
   };
 }
 
-async function runMemoryJobsOnce(
+type ProcessGroup = (group: MemoryJob[]) => Promise<number>;
+
+export async function runMemoryJobsOnce(
   options: { enableScheduledCleanup?: boolean } = {},
 ): Promise<number> {
   const config = getConfig();
@@ -169,10 +173,20 @@ async function runMemoryJobsOnce(
     log.warn({ timedOut }, "Timed out stalled memory jobs");
   }
 
-  const batchSize = Math.max(1, config.memory.jobs.batchSize);
-  const concurrency = Math.max(1, config.memory.jobs.workerConcurrency);
-  const jobs = claimMemoryJobs(batchSize);
-  if (jobs.length === 0) {
+  const cfgSlow = Math.max(1, config.memory.jobs.slowLlmConcurrency);
+  const cfgFast = Math.max(1, config.memory.jobs.fastConcurrency);
+  const cfgEmbed = Math.max(1, config.memory.jobs.embedConcurrency);
+
+  // Claim per-lane budgets so a backlog of slow LLM jobs cannot starve fast
+  // jobs (and vice versa). The Qdrant circuit breaker still gates only the
+  // embed lane inside `claimMemoryJobs`.
+  const claimed = claimMemoryJobs({
+    slowLlm: cfgSlow,
+    fast: cfgFast,
+    embed: cfgEmbed,
+  });
+
+  if (claimed.length === 0) {
     if (enableScheduledCleanup) {
       maybeEnqueueScheduledCleanupJobs(config);
     }
@@ -181,34 +195,22 @@ async function runMemoryJobsOnce(
     return 0;
   }
 
-  // Group jobs so they can run concurrently across independent work units.
-  // Jobs targeting different conversations (via payload.conversationId) are
-  // placed in separate groups and can run in parallel. Jobs targeting the
-  // same conversation, or global jobs without a conversationId (backfill,
-  // cleanup, rebuild_index), are grouped together and run sequentially to
-  // prevent checkpoint races.
-  const jobGroups = new Map<string, MemoryJob[]>();
-  for (const job of jobs) {
-    const convId =
-      typeof job.payload.conversationId === "string"
-        ? job.payload.conversationId
-        : null;
-    const groupKey = convId ? `${job.type}:${convId}` : job.type;
-    let group = jobGroups.get(groupKey);
-    if (!group) {
-      group = [];
-      jobGroups.set(groupKey, group);
+  const slowSet = new Set<MemoryJobType>(SLOW_LLM_JOB_TYPES);
+  const embedSet = new Set<MemoryJobType>(EMBED_JOB_TYPES);
+  const slowJobs: MemoryJob[] = [];
+  const fastJobs: MemoryJob[] = [];
+  const embedJobs: MemoryJob[] = [];
+  for (const job of claimed) {
+    if (slowSet.has(job.type)) {
+      slowJobs.push(job);
+    } else if (embedSet.has(job.type)) {
+      embedJobs.push(job);
+    } else {
+      fastJobs.push(job);
     }
-    group.push(job);
   }
 
-  let processed = 0;
-  const typeGroups = [...jobGroups.values()];
-
-  // Run type groups concurrently using a task pool (up to workerConcurrency
-  // active at a time). Unlike the old wave approach, a new group starts as
-  // soon as any slot frees up — no waiting for an entire wave to finish.
-  const processGroup = async (group: MemoryJob[]): Promise<number> => {
+  const processGroup: ProcessGroup = async (group) => {
     let groupProcessed = 0;
     for (const job of group) {
       try {
@@ -229,8 +231,59 @@ async function runMemoryJobsOnce(
     return groupProcessed;
   };
 
+  // Run all three lanes in parallel. Each lane runs its own bounded task pool
+  // so a slow `graph_consolidate` cannot block embed or fast jobs from making
+  // progress, and per-`(type, conversationId)` grouping inside each lane keeps
+  // same-conversation jobs serialized.
+  const [slowProcessed, fastProcessed, embedProcessed] = await Promise.all([
+    runLanePool(slowJobs, cfgSlow, processGroup),
+    runLanePool(fastJobs, cfgFast, processGroup),
+    runLanePool(embedJobs, cfgEmbed, processGroup),
+  ]);
+
+  if (enableScheduledCleanup) {
+    maybeEnqueueScheduledCleanupJobs(config);
+  }
+  maybeEnqueueGraphMaintenanceJobs(config);
+  maybeRunDbMaintenance();
+  return slowProcessed + fastProcessed + embedProcessed;
+}
+
+/**
+ * Run a single lane's jobs through a bounded task pool of size `concurrency`.
+ *
+ * Jobs targeting different conversations (via payload.conversationId) are
+ * placed in separate groups and run in parallel up to the lane's concurrency
+ * cap. Jobs targeting the same conversation — or global jobs without a
+ * conversationId — share a group and run sequentially to avoid checkpoint
+ * races.
+ */
+async function runLanePool(
+  jobs: MemoryJob[],
+  concurrency: number,
+  processGroup: ProcessGroup,
+): Promise<number> {
+  if (jobs.length === 0) return 0;
+
+  const groups = new Map<string, MemoryJob[]>();
+  for (const job of jobs) {
+    const convId =
+      typeof job.payload.conversationId === "string"
+        ? job.payload.conversationId
+        : null;
+    const groupKey = convId ? `${job.type}:${convId}` : job.type;
+    let group = groups.get(groupKey);
+    if (!group) {
+      group = [];
+      groups.set(groupKey, group);
+    }
+    group.push(job);
+  }
+
+  let processed = 0;
+  const typeGroups = [...groups.values()];
+
   if (typeGroups.length <= concurrency) {
-    // Fast path: all groups fit within the concurrency limit
     const results = await Promise.allSettled(typeGroups.map(processGroup));
     for (const result of results) {
       if (result.status === "fulfilled") {
@@ -242,39 +295,35 @@ async function runMemoryJobsOnce(
         );
       }
     }
-  } else {
-    // Task pool: maintain `concurrency` in-flight groups at all times
-    let nextIdx = 0;
-
-    const startNext = (): Promise<void> | undefined => {
-      if (nextIdx >= typeGroups.length) return undefined;
-      const group = typeGroups[nextIdx++]!;
-      return processGroup(group)
-        .then(
-          (count) => {
-            processed += count;
-          },
-          (err) => {
-            log.error(
-              { err },
-              "Memory job group rejected unexpectedly — jobs in this batch may have been dropped",
-            );
-          },
-        )
-        .then(() => startNext());
-    };
-
-    const workers = Array.from(
-      { length: Math.min(concurrency, typeGroups.length) },
-      () => startNext()!,
-    );
-    await Promise.all(workers);
+    return processed;
   }
-  if (enableScheduledCleanup) {
-    maybeEnqueueScheduledCleanupJobs(config);
-  }
-  maybeEnqueueGraphMaintenanceJobs(config);
-  maybeRunDbMaintenance();
+
+  // Task pool: keep `concurrency` groups in flight at all times so a new group
+  // starts the instant any slot frees up.
+  let nextIdx = 0;
+  const startNext = (): Promise<void> | undefined => {
+    if (nextIdx >= typeGroups.length) return undefined;
+    const group = typeGroups[nextIdx++]!;
+    return processGroup(group)
+      .then(
+        (count) => {
+          processed += count;
+        },
+        (err) => {
+          log.error(
+            { err },
+            "Memory job group rejected unexpectedly — jobs in this batch may have been dropped",
+          );
+        },
+      )
+      .then(() => startNext());
+  };
+
+  const workers = Array.from(
+    { length: Math.min(concurrency, typeGroups.length) },
+    () => startNext()!,
+  );
+  await Promise.all(workers);
   return processed;
 }
 

package/src/memory/memory-v2-activation-log-store.ts

@@ -14,7 +14,7 @@ export interface MemoryV2ConceptRowRecord {
   simNow: number;
   spreadContribution: number;
   source: "prior_state" | "ann_top50" | "both";
-  status: "in_context" | "injected" | "not_injected";
+  status: "in_context" | "injected" | "not_injected" | "page_missing";
 }
 
 export interface MemoryV2SkillRowRecord {

package/src/memory/migrations/237-heartbeat-runs.ts

@@ -0,0 +1,45 @@
+import type { DrizzleDb } from "../db-connection.js";
+import { getSqliteFrom } from "../db-connection.js";
+import { tableHasColumn } from "./schema-introspection.js";
+import { withCrashRecovery } from "./validate-migration-state.js";
+
+const CHECKPOINT_KEY = "migration_heartbeat_runs_v1";
+
+/**
+ * Create the heartbeat_runs table for tracking heartbeat execution lifecycle.
+ *
+ * Each row represents one scheduled heartbeat tick, tracking its progression
+ * through the status lifecycle: pending -> running -> ok/error/timeout, or
+ * pending -> skipped/missed/superseded.
+ */
+export function migrateHeartbeatRuns(database: DrizzleDb): void {
+  withCrashRecovery(database, CHECKPOINT_KEY, () => {
+    if (tableHasColumn(database, "heartbeat_runs", "id")) {
+      return;
+    }
+    const raw = getSqliteFrom(database);
+    raw.exec(/*sql*/ `
+      CREATE TABLE IF NOT EXISTS heartbeat_runs (
+        id TEXT PRIMARY KEY,
+        scheduled_for INTEGER NOT NULL,
+        started_at INTEGER,
+        finished_at INTEGER,
+        duration_ms INTEGER,
+        status TEXT NOT NULL,
+        skip_reason TEXT,
+        error TEXT,
+        conversation_id TEXT,
+        created_at INTEGER NOT NULL
+      )
+    `);
+    raw.exec(/*sql*/ `
+      CREATE INDEX IF NOT EXISTS idx_heartbeat_runs_scheduled_for
+        ON heartbeat_runs (scheduled_for)
+    `);
+  });
+}
+
+export function downHeartbeatRuns(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+  raw.exec(/*sql*/ `DROP TABLE IF EXISTS heartbeat_runs`);
+}

package/src/memory/migrations/238-schedule-retry-policy.ts

@@ -0,0 +1,20 @@
+import type { DrizzleDb } from "../db-connection.js";
+import { getSqliteFrom } from "../db-connection.js";
+
+export function migrateScheduleRetryPolicy(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+  try {
+    raw.exec(
+      `ALTER TABLE cron_jobs ADD COLUMN max_retries INTEGER NOT NULL DEFAULT 3`,
+    );
+  } catch {
+    /* Column already exists */
+  }
+  try {
+    raw.exec(
+      `ALTER TABLE cron_jobs ADD COLUMN retry_backoff_ms INTEGER NOT NULL DEFAULT 60000`,
+    );
+  } catch {
+    /* Column already exists */
+  }
+}

package/src/memory/migrations/index.ts

@@ -195,6 +195,11 @@ export {
   downToolInvocationsMatchedRuleId,
   migrateToolInvocationsMatchedRuleId,
 } from "./236-tool-invocations-matched-rule-id.js";
+export {
+  downHeartbeatRuns,
+  migrateHeartbeatRuns,
+} from "./237-heartbeat-runs.js";
+export { migrateScheduleRetryPolicy } from "./238-schedule-retry-policy.js";
 export {
   MIGRATION_REGISTRY,
   type MigrationRegistryEntry,

package/src/memory/migrations/registry.ts

@@ -47,6 +47,7 @@ import { downActivationState } from "./232-activation-state.js";
 import { downMemoryV2ActivationLogs } from "./234-memory-v2-activation-logs.js";
 import { downSlackCompactionWatermark } from "./235-slack-compaction-watermark.js";
 import { downToolInvocationsMatchedRuleId } from "./236-tool-invocations-matched-rule-id.js";
+import { downHeartbeatRuns } from "./237-heartbeat-runs.js";
 
 export interface MigrationRegistryEntry {
   /** The checkpoint key written to memory_checkpoints on completion. */
@@ -404,6 +405,13 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
       "Add matched_trust_rule_id column to tool_invocations for trust rule audit and rule editor UI",
     down: downToolInvocationsMatchedRuleId,
   },
+  {
+    key: "migration_heartbeat_runs_v1",
+    version: 47,
+    description:
+      "Create heartbeat_runs table for tracking heartbeat execution lifecycle with CAS state transitions",
+    down: downHeartbeatRuns,
+  },
 ];
 
 export function getMaxMigrationVersion(): number {

package/src/memory/pkb/pkb-search.ts

@@ -2,7 +2,9 @@
 // PKB — Qdrant hybrid search for indexed PKB markdown files
 // ---------------------------------------------------------------------------
 
+import { getConfig } from "../../config/loader.js";
 import { getLogger } from "../../util/logger.js";
+import { isMemoryV2ReadActive } from "../context-search/sources/memory-v2.js";
 import {
   isQdrantBreakerOpen,
   withQdrantBreaker,
@@ -40,6 +42,11 @@ export async function searchPkbFiles(
   limit: number,
   scopeIds?: string[],
 ): Promise<PkbSearchResult[]> {
+  // v2 owns the read path when both gates are on; v2 absorbs PKB as a read
+  // source, so PKB hint search short-circuits to keep traffic off the v1
+  // collection (avoiding OOM-crash risk from a corrupted sparse segment).
+  if (isMemoryV2ReadActive(getConfig())) return [];
+
   if (isQdrantBreakerOpen()) {
     log.warn("Qdrant circuit breaker open, skipping PKB search");
     return [];