@vellumai/assistant 0.7.2 → 0.8.0

package/src/memory/jobs/embed-concept-page.ts

@@ -98,52 +98,93 @@ export async function embedConceptPageJob(
     );
   }
 
-  const contentHash = embeddingInputContentHash({ type: "text", text });
   const expectedDim = config.memory.qdrant.vectorSize;
-  let provider = status.provider;
-  let model = status.model!;
+  // The status provider is the cache lookup key for any prior row; the
+  // *actual* provider/model come back on the embedded result. They usually
+  // match, but a backend swap mid-run would surface here — body and summary
+  // are then re-embedded together so both rows write under the same identity.
+  const cacheProvider = status.provider;
+  const cacheModel = status.model!;
+
+  const db = getDb();
 
   // Cache lookup: same (targetType, targetId, provider, model) row gets
   // reused across runs as long as `contentHash` matches. The dim mismatch
   // check guards against a config change (vectorSize bumped) since the last
-  // write — in that case we treat the row as stale and re-embed.
-  const db = getDb();
-  let cachedRow = db
-    .select({
-      vectorBlob: memoryEmbeddings.vectorBlob,
-      vectorJson: memoryEmbeddings.vectorJson,
-      dimensions: memoryEmbeddings.dimensions,
-      contentHash: memoryEmbeddings.contentHash,
-    })
-    .from(memoryEmbeddings)
-    .where(
-      and(
-        eq(memoryEmbeddings.targetType, CONCEPT_PAGE_TARGET_TYPE),
-        eq(memoryEmbeddings.targetId, slug),
-        eq(memoryEmbeddings.provider, provider),
-        eq(memoryEmbeddings.model, model),
-      ),
-    )
-    .get();
-  if (cachedRow && cachedRow.dimensions !== expectedDim) cachedRow = undefined;
-  if (cachedRow && cachedRow.contentHash !== contentHash) cachedRow = undefined;
-
-  let dense: number[];
-  let cacheHit = false;
-  if (cachedRow) {
-    dense = cachedRow.vectorBlob
-      ? blobToVector(cachedRow.vectorBlob as Buffer)
-      : (JSON.parse(cachedRow.vectorJson!) as number[]);
-    cacheHit = true;
-  } else {
-    const embedded = await embedWithBackend(config, [{ type: "text", text }]);
-    const vector = embedded.vectors[0];
-    if (!vector) return;
-    dense = vector;
-    provider = embedded.provider;
-    model = embedded.model;
+  // write — in that case we treat the row as stale and re-embed. The body
+  // and (optional) summary share the same provider/model — but each gets
+  // its own cache row keyed by a distinct targetId so summary edits don't
+  // invalidate the body cache and vice versa.
+  const bodyContentHash = embeddingInputContentHash({ type: "text", text });
+  const bodyCache = readEmbeddingCache(
+    db,
+    slug,
+    cacheProvider,
+    cacheModel,
+    expectedDim,
+  );
+  const bodyCacheHit = bodyCache?.contentHash === bodyContentHash;
+
+  // Optional summary embedding — only when the page has a `summary` in its
+  // frontmatter. Pages without one fall back to body-only retrieval at
+  // query time (the activation pipeline reads the summary score as
+  // undefined and uses the body score directly).
+  const summaryText = page.frontmatter.summary?.trim() ?? "";
+  const hasSummary = summaryText.length > 0;
+  const summaryCacheId = `${slug}#summary`;
+  const summaryContentHash = hasSummary
+    ? embeddingInputContentHash({ type: "text", text: summaryText })
+    : undefined;
+  const summaryCache = hasSummary
+    ? readEmbeddingCache(
+        db,
+        summaryCacheId,
+        cacheProvider,
+        cacheModel,
+        expectedDim,
+      )
+    : null;
+  const summaryCacheHit =
+    hasSummary && summaryCache?.contentHash === summaryContentHash;
+
+  // Batch all cache misses into one `embedWithBackend` call. Each backend
+  // round-trip is the dominant cost — fresh body + fresh summary in a
+  // single batch saves a round-trip vs serial calls and gives both vectors
+  // the same provider/model regardless of any backend rotation mid-run.
+  type Slot = "body" | "summary";
+  const toEmbed: Array<{ type: "text"; text: string }> = [];
+  const slots: Slot[] = [];
+  if (!bodyCacheHit) {
+    toEmbed.push({ type: "text", text });
+    slots.push("body");
+  }
+  if (hasSummary && !summaryCacheHit) {
+    toEmbed.push({ type: "text", text: summaryText });
+    slots.push("summary");
   }
 
+  let bodyDense: number[] | undefined = bodyCacheHit ? bodyCache!.dense : undefined;
+  let summaryDense: number[] | undefined = summaryCacheHit
+    ? summaryCache!.dense
+    : undefined;
+  let writeProvider = cacheProvider;
+  let writeModel = cacheModel;
+  if (toEmbed.length > 0) {
+    const embedded = await embedWithBackend(config, toEmbed);
+    writeProvider = embedded.provider;
+    writeModel = embedded.model;
+    for (let i = 0; i < slots.length; i++) {
+      const vector = embedded.vectors[i];
+      if (!vector) continue;
+      if (slots[i] === "body") bodyDense = vector;
+      else summaryDense = vector;
+    }
+  }
+  // Body embedding is the ground truth — without it the page can't surface.
+  // (Cache hit paths populate `bodyDense` above; a fresh embed that returned
+  // no vectors short-circuits here too.)
+  if (!bodyDense) return;
+
   // Sparse is cheap (in-process tokenization) and changes any time the body
   // changes, so we always recompute it rather than caching alongside dense.
   // BM25 weights live on the doc side; queries embed binary occurrence in
@@ -151,57 +192,42 @@ export async function embedConceptPageJob(
   // corpus for the first time), fall back to the legacy TF-only encoding —
   // the next reembed pass overwrites the page once stats are available.
   const corpusStats = getConceptPageCorpusStats();
-  const sparse = corpusStats
-    ? generateBm25DocEmbedding(text, corpusStats, {
-        k1: config.memory.v2.bm25_k1,
-        b: config.memory.v2.bm25_b,
-      })
-    : generateSparseEmbedding(text);
+  const encodeSparse = (input: string) =>
+    corpusStats
+      ? generateBm25DocEmbedding(input, corpusStats, {
+          k1: config.memory.v2.bm25_k1,
+          b: config.memory.v2.bm25_b,
+        })
+      : generateSparseEmbedding(input);
+  const sparse = encodeSparse(text);
+  const summarySparse = hasSummary ? encodeSparse(summaryText) : undefined;
 
   const now = Date.now();
   // Persist freshly embedded vectors for cross-restart reuse. On cache hit
   // the existing row already has identical content + hash, so the write
   // would be a no-op — skip it. Best-effort: write failure is not fatal,
   // we still want the Qdrant upsert below to fire.
-  if (!cacheHit) {
-    try {
-      const blobValue = vectorToBlob(dense);
-      db.insert(memoryEmbeddings)
-        .values({
-          id: randomUUID(),
-          targetType: CONCEPT_PAGE_TARGET_TYPE,
-          targetId: slug,
-          provider,
-          model,
-          dimensions: dense.length,
-          vectorBlob: blobValue,
-          vectorJson: null,
-          contentHash,
-          createdAt: now,
-          updatedAt: now,
-        })
-        .onConflictDoUpdate({
-          target: [
-            memoryEmbeddings.targetType,
-            memoryEmbeddings.targetId,
-            memoryEmbeddings.provider,
-            memoryEmbeddings.model,
-          ],
-          set: {
-            vectorBlob: blobValue,
-            vectorJson: null,
-            dimensions: dense.length,
-            contentHash,
-            updatedAt: now,
-          },
-        })
-        .run();
-    } catch (err) {
-      log.warn(
-        { err, slug },
-        "Failed to write concept-page embedding cache row",
-      );
-    }
+  if (!bodyCacheHit) {
+    writeEmbeddingCache(db, {
+      slug,
+      cacheId: slug,
+      dense: bodyDense,
+      contentHash: bodyContentHash,
+      provider: writeProvider,
+      model: writeModel,
+      now,
+    });
+  }
+  if (hasSummary && !summaryCacheHit && summaryDense && summaryContentHash) {
+    writeEmbeddingCache(db, {
+      slug,
+      cacheId: summaryCacheId,
+      dense: summaryDense,
+      contentHash: summaryContentHash,
+      provider: writeProvider,
+      model: writeModel,
+      now,
+    });
   }
 
   // Apply anisotropy correction at the boundary between the (raw) cached
@@ -210,19 +236,129 @@ export async function embedConceptPageJob(
   // the cache survives and the (cheap) correction math reruns over each
   // cached vector. Pass-through when no calibration is fit yet.
   const correctedDense = await applyCorrectionIfCalibrated(
-    dense,
-    provider,
-    model,
+    bodyDense,
+    writeProvider,
+    writeModel,
   );
+  const correctedSummaryDense = summaryDense
+    ? await applyCorrectionIfCalibrated(summaryDense, writeProvider, writeModel)
+    : undefined;
 
   await upsertConceptPageEmbedding({
     slug,
     dense: correctedDense,
     sparse,
+    summary:
+      correctedSummaryDense && summarySparse
+        ? { dense: correctedSummaryDense, sparse: summarySparse }
+        : undefined,
     updatedAt: now,
   });
 }
 
+/** SQLite cache row shape returned by `readEmbeddingCache`. */
+interface EmbeddingCacheEntry {
+  dense: number[];
+  contentHash: string;
+}
+
+/**
+ * Look up a cached dense vector keyed on `(targetType, targetId, provider,
+ * model)`. Returns the row only when the persisted dimensions match the
+ * configured expectation — a stale row from a previous `vectorSize` is
+ * treated as a cache miss so the caller re-embeds.
+ */
+function readEmbeddingCache(
+  db: ReturnType<typeof getDb>,
+  cacheId: string,
+  provider: string,
+  model: string,
+  expectedDim: number,
+): EmbeddingCacheEntry | null {
+  const row = db
+    .select({
+      vectorBlob: memoryEmbeddings.vectorBlob,
+      vectorJson: memoryEmbeddings.vectorJson,
+      dimensions: memoryEmbeddings.dimensions,
+      contentHash: memoryEmbeddings.contentHash,
+    })
+    .from(memoryEmbeddings)
+    .where(
+      and(
+        eq(memoryEmbeddings.targetType, CONCEPT_PAGE_TARGET_TYPE),
+        eq(memoryEmbeddings.targetId, cacheId),
+        eq(memoryEmbeddings.provider, provider),
+        eq(memoryEmbeddings.model, model),
+      ),
+    )
+    .get();
+  if (!row || row.dimensions !== expectedDim) return null;
+  // A row without a contentHash is a legacy/corrupt entry — treat as a miss
+  // and force a re-embed rather than misalign the cache key.
+  if (row.contentHash === null) return null;
+  const dense = row.vectorBlob
+    ? blobToVector(row.vectorBlob as Buffer)
+    : (JSON.parse(row.vectorJson!) as number[]);
+  return { dense, contentHash: row.contentHash };
+}
+
+/**
+ * Persist a freshly embedded dense vector in the SQLite cache. Best-effort:
+ * a write failure is logged and swallowed so the Qdrant upsert still runs.
+ */
+function writeEmbeddingCache(
+  db: ReturnType<typeof getDb>,
+  params: {
+    slug: string;
+    cacheId: string;
+    dense: number[];
+    contentHash: string;
+    provider: string;
+    model: string;
+    now: number;
+  },
+): void {
+  const { slug, cacheId, dense, contentHash, provider, model, now } = params;
+  try {
+    const blobValue = vectorToBlob(dense);
+    db.insert(memoryEmbeddings)
+      .values({
+        id: randomUUID(),
+        targetType: CONCEPT_PAGE_TARGET_TYPE,
+        targetId: cacheId,
+        provider,
+        model,
+        dimensions: dense.length,
+        vectorBlob: blobValue,
+        vectorJson: null,
+        contentHash,
+        createdAt: now,
+        updatedAt: now,
+      })
+      .onConflictDoUpdate({
+        target: [
+          memoryEmbeddings.targetType,
+          memoryEmbeddings.targetId,
+          memoryEmbeddings.provider,
+          memoryEmbeddings.model,
+        ],
+        set: {
+          vectorBlob: blobValue,
+          vectorJson: null,
+          dimensions: dense.length,
+          contentHash,
+          updatedAt: now,
+        },
+      })
+      .run();
+  } catch (err) {
+    log.warn(
+      { err, slug, cacheId },
+      "Failed to write concept-page embedding cache row",
+    );
+  }
+}
+
 /**
  * Enqueue an `embed_concept_page` job (async, fire-and-forget). Modeled on
  * `enqueuePkbIndexJob` — callers that want a slug re-embedded after a write

package/src/memory/jobs-store.ts

@@ -18,6 +18,7 @@ export type MemoryJobType =
   | "embed_summary"
   | "prune_old_conversations"
   | "prune_old_llm_request_logs"
+  | "prune_old_trace_events"
   | "build_conversation_summary"
   | "conversation_analyze"
   | "backfill"
@@ -366,6 +367,53 @@ export function enqueuePruneOldConversationsJob(
   return enqueueMemoryJob("prune_old_conversations", payload);
 }
 
+export function enqueuePruneOldTraceEventsJob(retentionDays?: number): string {
+  const db = getDb();
+  const existing = db
+    .select()
+    .from(memoryJobs)
+    .where(
+      and(
+        eq(memoryJobs.type, "prune_old_trace_events"),
+        inArray(memoryJobs.status, ["pending", "running"]),
+      ),
+    )
+    .orderBy(asc(memoryJobs.createdAt))
+    .get();
+  if (existing) {
+    if (
+      existing.status === "pending" &&
+      typeof retentionDays === "number" &&
+      Number.isFinite(retentionDays) &&
+      retentionDays >= 0
+    ) {
+      let payload: Record<string, unknown> = {};
+      try {
+        payload = JSON.parse(existing.payload) as Record<string, unknown>;
+      } catch {
+        payload = {};
+      }
+      if (payload.retentionDays !== retentionDays) {
+        db.update(memoryJobs)
+          .set({
+            payload: JSON.stringify({ ...payload, retentionDays }),
+            updatedAt: Date.now(),
+          })
+          .where(eq(memoryJobs.id, existing.id))
+          .run();
+      }
+    }
+    return existing.id;
+  }
+  const payload =
+    typeof retentionDays === "number" &&
+    Number.isFinite(retentionDays) &&
+    retentionDays >= 0
+      ? { retentionDays }
+      : {};
+  return enqueueMemoryJob("prune_old_trace_events", payload);
+}
+
 export interface LaneBudgets {
   slowLlm: number;
   fast: number;

package/src/memory/jobs-worker.ts

@@ -1,6 +1,10 @@
-import { isAssistantFeatureFlagEnabled } from "../config/assistant-feature-flags.js";
 import { getConfig } from "../config/loader.js";
 import type { AssistantConfig } from "../config/types.js";
+import {
+  checkDiskPressureBackgroundGate,
+  diskPressureBackgroundSkipLogFields,
+  shouldLogDiskPressureBackgroundSkip,
+} from "../daemon/disk-pressure-background-gate.js";
 import { getLogger } from "../util/logger.js";
 import { getMemoryCheckpoint, setMemoryCheckpoint } from "./checkpoints.js";
 import {
@@ -23,6 +27,7 @@ import { backfillJob } from "./job-handlers/backfill.js";
 import {
   pruneOldConversationsJob,
   pruneOldLlmRequestLogsJob,
+  pruneOldTraceEventsJob,
 } from "./job-handlers/cleanup.js";
 import { generateConversationStartersJob } from "./job-handlers/conversation-starters.js";
 // ── Per-job-type handlers ──────────────────────────────────────────
@@ -54,6 +59,7 @@ import {
   enqueueMemoryJob,
   enqueuePruneOldConversationsJob,
   enqueuePruneOldLlmRequestLogsJob,
+  enqueuePruneOldTraceEventsJob,
   failMemoryJob,
   failStalledJobs,
   type MemoryJob,
@@ -167,6 +173,20 @@ export async function runMemoryJobsOnce(
   if (!config.memory.enabled) return 0;
   const enableScheduledCleanup = options.enableScheduledCleanup === true;
 
+  const diskPressureGate = checkDiskPressureBackgroundGate("background-work");
+  if (diskPressureGate.action === "skip") {
+    if (shouldLogDiskPressureBackgroundSkip("memory-jobs-worker")) {
+      log.warn(
+        {
+          source: "memory",
+          ...diskPressureBackgroundSkipLogFields(diskPressureGate),
+        },
+        "Memory jobs worker skipped during disk pressure cleanup mode",
+      );
+    }
+    return 0;
+  }
+
   // Fail jobs that have been running longer than the configured timeout
   const timedOut = failStalledJobs(config.memory.jobs.stalledJobTimeoutMs);
   if (timedOut > 0) {
@@ -455,6 +475,9 @@ async function processJob(
     case "prune_old_llm_request_logs":
       pruneOldLlmRequestLogsJob(job, config);
       return;
+    case "prune_old_trace_events":
+      pruneOldTraceEventsJob(job, config);
+      return;
     case "build_conversation_summary":
       await buildConversationSummaryJob(job, config);
       return;
@@ -486,6 +509,11 @@ async function processJob(
       await embedGraphTriggerJob(job, config);
       return;
     case "graph_extract":
+      // Stale rows enqueued before v2 was enabled (or by any unguarded v1
+      // path) must not consume embedding/extraction budget when v2 is on.
+      if (config.memory.v2.enabled) {
+        return;
+      }
       await graphExtractJob(job, config);
       return;
     case "conversation_analyze":
@@ -560,14 +588,20 @@ function maybeEnqueueScheduledCleanupJobs(
     cleanup.llmRequestLogRetentionMs !== null
       ? enqueuePruneOldLlmRequestLogsJob(cleanup.llmRequestLogRetentionMs)
       : null;
+  const pruneTraceEventsJobId =
+    cleanup.traceEventRetentionDays > 0
+      ? enqueuePruneOldTraceEventsJob(cleanup.traceEventRetentionDays)
+      : null;
   markScheduledCleanupEnqueued(nowMs);
   log.debug(
     {
       pruneConversationsJobId,
       pruneLlmRequestLogsJobId,
+      pruneTraceEventsJobId,
       enqueueIntervalMs: cleanup.enqueueIntervalMs,
       conversationRetentionDays: cleanup.conversationRetentionDays,
       llmRequestLogRetentionMs: cleanup.llmRequestLogRetentionMs,
+      traceEventRetentionDays: cleanup.traceEventRetentionDays,
     },
     "Enqueued scheduled memory cleanup jobs",
   );
@@ -590,58 +624,66 @@ export const GRAPH_MAINTENANCE_CHECKPOINTS = {
 } as const;
 
 /**
- * Enqueue periodic graph maintenance jobs (decay, consolidation, pattern scan, narrative).
- * Uses durable checkpoints so intervals survive daemon restarts — jobs only fire
- * when the actual elapsed time since last run exceeds the interval.
+ * Enqueue periodic graph maintenance jobs.
  *
- * The v2 consolidation entry is gated on both the `memory-v2-enabled` feature
- * flag and the `memory.v2.enabled` config — both must be true for the cron
- * to fire. Sweep is intentionally not on this schedule: it is debounced from
- * the live `graph_extract` trigger path (see `indexMessageNow` in
- * `indexer.ts`) so it runs on the same idle/message-count cadence.
+ * Mutually exclusive between v1 and v2:
+ *   - v2 active (`memory.v2.enabled` on) → only `memory_v2_consolidate` is
+ *     scheduled.
+ *   - v2 inactive → the four v1 entries (decay, consolidate, pattern_scan,
+ *     narrative) are scheduled instead.
+ *
+ * Read/write paths route to v2 when the flag is on, so v1 graph data goes
+ * unread; running v1 maintenance alongside v2 is wasted compute and LLM
+ * spend. The v1 code path remains live so flipping the flag back to off
+ * fully re-engages v1.
+ *
+ * Uses durable checkpoints so intervals survive daemon restarts — jobs only
+ * fire when the actual elapsed time since last run exceeds the interval.
+ * Sweep is intentionally not on this schedule: it is debounced from the
+ * live `graph_extract` trigger path (see `indexMessageNow` in `indexer.ts`)
+ * so it runs on the same idle/message-count cadence.
  */
 export function maybeEnqueueGraphMaintenanceJobs(
   config: AssistantConfig,
   nowMs = Date.now(),
 ): void {
+  const v2Active = config.memory.v2.enabled;
+
   const schedule: Array<{
     key: string;
     intervalMs: number;
     jobType: MemoryJobType;
-  }> = [
-    {
-      key: GRAPH_MAINTENANCE_CHECKPOINTS.decay,
-      intervalMs: GRAPH_DECAY_INTERVAL_MS,
-      jobType: "graph_decay",
-    },
-    {
-      key: GRAPH_MAINTENANCE_CHECKPOINTS.consolidate,
-      intervalMs: GRAPH_CONSOLIDATE_INTERVAL_MS,
-      jobType: "graph_consolidate",
-    },
-    {
-      key: GRAPH_MAINTENANCE_CHECKPOINTS.patternScan,
-      intervalMs: GRAPH_PATTERN_SCAN_INTERVAL_MS,
-      jobType: "graph_pattern_scan",
-    },
-    {
-      key: GRAPH_MAINTENANCE_CHECKPOINTS.narrative,
-      intervalMs: GRAPH_NARRATIVE_INTERVAL_MS,
-      jobType: "graph_narrative_refine",
-    },
-  ];
-
-  if (
-    isAssistantFeatureFlagEnabled("memory-v2-enabled", config) &&
-    config.memory.v2.enabled
-  ) {
-    schedule.push({
-      key: GRAPH_MAINTENANCE_CHECKPOINTS.memoryV2Consolidate,
-      intervalMs:
-        config.memory.v2.consolidation_interval_hours * 60 * 60 * 1000,
-      jobType: "memory_v2_consolidate",
-    });
-  }
+  }> = v2Active
+    ? [
+        {
+          key: GRAPH_MAINTENANCE_CHECKPOINTS.memoryV2Consolidate,
+          intervalMs:
+            config.memory.v2.consolidation_interval_hours * 60 * 60 * 1000,
+          jobType: "memory_v2_consolidate",
+        },
+      ]
+    : [
+        {
+          key: GRAPH_MAINTENANCE_CHECKPOINTS.decay,
+          intervalMs: GRAPH_DECAY_INTERVAL_MS,
+          jobType: "graph_decay",
+        },
+        {
+          key: GRAPH_MAINTENANCE_CHECKPOINTS.consolidate,
+          intervalMs: GRAPH_CONSOLIDATE_INTERVAL_MS,
+          jobType: "graph_consolidate",
+        },
+        {
+          key: GRAPH_MAINTENANCE_CHECKPOINTS.patternScan,
+          intervalMs: GRAPH_PATTERN_SCAN_INTERVAL_MS,
+          jobType: "graph_pattern_scan",
+        },
+        {
+          key: GRAPH_MAINTENANCE_CHECKPOINTS.narrative,
+          intervalMs: GRAPH_NARRATIVE_INTERVAL_MS,
+          jobType: "graph_narrative_refine",
+        },
+      ];
 
   for (const { key, intervalMs, jobType } of schedule) {
     const lastRun = parseInt(getMemoryCheckpoint(key) ?? "0", 10);