@vellumai/assistant 0.7.3 → 0.8.1

package/src/memory/jobs/embed-concept-page.ts

@@ -98,51 +98,125 @@ export async function embedConceptPageJob(
     );
   }
 
-  const contentHash = embeddingInputContentHash({ type: "text", text });
   const expectedDim = config.memory.qdrant.vectorSize;
-  let provider = status.provider;
-  let model = status.model!;
+  // The status provider is the cache lookup key for any prior row; the
+  // *actual* provider/model come back on the embedded result. They usually
+  // match, but a backend swap mid-run would surface here — body and summary
+  // are then re-embedded together so both rows write under the same identity.
+  const cacheProvider = status.provider;
+  const cacheModel = status.model!;
+
+  const db = getDb();
 
   // Cache lookup: same (targetType, targetId, provider, model) row gets
   // reused across runs as long as `contentHash` matches. The dim mismatch
   // check guards against a config change (vectorSize bumped) since the last
-  // write — in that case we treat the row as stale and re-embed.
-  const db = getDb();
-  let cachedRow = db
-    .select({
-      vectorBlob: memoryEmbeddings.vectorBlob,
-      vectorJson: memoryEmbeddings.vectorJson,
-      dimensions: memoryEmbeddings.dimensions,
-      contentHash: memoryEmbeddings.contentHash,
-    })
-    .from(memoryEmbeddings)
-    .where(
-      and(
-        eq(memoryEmbeddings.targetType, CONCEPT_PAGE_TARGET_TYPE),
-        eq(memoryEmbeddings.targetId, slug),
-        eq(memoryEmbeddings.provider, provider),
-        eq(memoryEmbeddings.model, model),
-      ),
-    )
-    .get();
-  if (cachedRow && cachedRow.dimensions !== expectedDim) cachedRow = undefined;
-  if (cachedRow && cachedRow.contentHash !== contentHash) cachedRow = undefined;
-
-  let dense: number[];
-  let cacheHit = false;
-  if (cachedRow) {
-    dense = cachedRow.vectorBlob
-      ? blobToVector(cachedRow.vectorBlob as Buffer)
-      : (JSON.parse(cachedRow.vectorJson!) as number[]);
-    cacheHit = true;
-  } else {
-    const embedded = await embedWithBackend(config, [{ type: "text", text }]);
-    const vector = embedded.vectors[0];
-    if (!vector) return;
-    dense = vector;
-    provider = embedded.provider;
-    model = embedded.model;
+  // write — in that case we treat the row as stale and re-embed. The body
+  // and (optional) summary share the same provider/model — but each gets
+  // its own cache row keyed by a distinct targetId so summary edits don't
+  // invalidate the body cache and vice versa.
+  const bodyContentHash = embeddingInputContentHash({ type: "text", text });
+  const bodyCache = readEmbeddingCache(
+    db,
+    slug,
+    cacheProvider,
+    cacheModel,
+    expectedDim,
+  );
+  const bodyCacheHit = bodyCache?.contentHash === bodyContentHash;
+
+  // Optional summary embedding — only when the page has a `summary` in its
+  // frontmatter. Pages without one fall back to body-only retrieval at
+  // query time (the activation pipeline reads the summary score as
+  // undefined and uses the body score directly).
+  const summaryText = page.frontmatter.summary?.trim() ?? "";
+  const hasSummary = summaryText.length > 0;
+  const summaryCacheId = `${slug}#summary`;
+  const summaryContentHash = hasSummary
+    ? embeddingInputContentHash({ type: "text", text: summaryText })
+    : undefined;
+  const summaryCache = hasSummary
+    ? readEmbeddingCache(
+        db,
+        summaryCacheId,
+        cacheProvider,
+        cacheModel,
+        expectedDim,
+      )
+    : null;
+  const summaryCacheHit =
+    hasSummary && summaryCache?.contentHash === summaryContentHash;
+
+  // Batch all cache misses into one `embedWithBackend` call. Each backend
+  // round-trip is the dominant cost — fresh body + fresh summary in a
+  // single batch saves a round-trip vs serial calls and gives both vectors
+  // the same provider/model regardless of any backend rotation mid-run.
+  type Slot = "body" | "summary";
+  const toEmbed: Array<{ type: "text"; text: string }> = [];
+  const slots: Slot[] = [];
+  if (!bodyCacheHit) {
+    toEmbed.push({ type: "text", text });
+    slots.push("body");
+  }
+  if (hasSummary && !summaryCacheHit) {
+    toEmbed.push({ type: "text", text: summaryText });
+    slots.push("summary");
+  }
+
+  let bodyDense: number[] | undefined = bodyCacheHit
+    ? bodyCache!.dense
+    : undefined;
+  let summaryDense: number[] | undefined = summaryCacheHit
+    ? summaryCache!.dense
+    : undefined;
+  let writeProvider = cacheProvider;
+  let writeModel = cacheModel;
+  let bodyFresh = false;
+  let summaryFresh = false;
+  if (toEmbed.length > 0) {
+    let embedded = await embedWithBackend(config, toEmbed);
+    let appliedSlots = slots;
+    // Backend rotation between `getMemoryBackendStatus()` and
+    // `embedWithBackend()` would tag the cached half with the old
+    // provider/model and the fresh half with the new — writing both into
+    // one Qdrant point mixes embedding spaces. Re-embed every slot fresh
+    // when we detect the rotation so the point's named vectors share one
+    // identity.
+    const rotated =
+      (bodyCacheHit || summaryCacheHit) &&
+      (embedded.provider !== cacheProvider || embedded.model !== cacheModel);
+    if (rotated) {
+      const allTexts: Array<{ type: "text"; text: string }> = [
+        { type: "text", text },
+      ];
+      const allSlots: Slot[] = ["body"];
+      if (hasSummary) {
+        allTexts.push({ type: "text", text: summaryText });
+        allSlots.push("summary");
+      }
+      embedded = await embedWithBackend(config, allTexts);
+      appliedSlots = allSlots;
+      bodyDense = undefined;
+      summaryDense = undefined;
+    }
+    writeProvider = embedded.provider;
+    writeModel = embedded.model;
+    for (let i = 0; i < appliedSlots.length; i++) {
+      const vector = embedded.vectors[i];
+      if (!vector) continue;
+      if (appliedSlots[i] === "body") {
+        bodyDense = vector;
+        bodyFresh = true;
+      } else {
+        summaryDense = vector;
+        summaryFresh = true;
+      }
+    }
   }
+  // Body embedding is the ground truth — without it the page can't surface.
+  // (Cache hit paths populate `bodyDense` above; a fresh embed that returned
+  // no vectors short-circuits here too.)
+  if (!bodyDense) return;
 
   // Sparse is cheap (in-process tokenization) and changes any time the body
   // changes, so we always recompute it rather than caching alongside dense.
@@ -151,57 +225,45 @@ export async function embedConceptPageJob(
   // corpus for the first time), fall back to the legacy TF-only encoding —
   // the next reembed pass overwrites the page once stats are available.
   const corpusStats = getConceptPageCorpusStats();
-  const sparse = corpusStats
-    ? generateBm25DocEmbedding(text, corpusStats, {
-        k1: config.memory.v2.bm25_k1,
-        b: config.memory.v2.bm25_b,
-      })
-    : generateSparseEmbedding(text);
+  const encodeSparse = (input: string) =>
+    corpusStats
+      ? generateBm25DocEmbedding(input, corpusStats, {
+          k1: config.memory.v2.bm25_k1,
+          b: config.memory.v2.bm25_b,
+        })
+      : generateSparseEmbedding(input);
+  const sparse = encodeSparse(text);
+  const summarySparse = hasSummary ? encodeSparse(summaryText) : undefined;
 
   const now = Date.now();
   // Persist freshly embedded vectors for cross-restart reuse. On cache hit
   // the existing row already has identical content + hash, so the write
-  // would be a no-op — skip it. Best-effort: write failure is not fatal,
-  // we still want the Qdrant upsert below to fire.
-  if (!cacheHit) {
-    try {
-      const blobValue = vectorToBlob(dense);
-      db.insert(memoryEmbeddings)
-        .values({
-          id: randomUUID(),
-          targetType: CONCEPT_PAGE_TARGET_TYPE,
-          targetId: slug,
-          provider,
-          model,
-          dimensions: dense.length,
-          vectorBlob: blobValue,
-          vectorJson: null,
-          contentHash,
-          createdAt: now,
-          updatedAt: now,
-        })
-        .onConflictDoUpdate({
-          target: [
-            memoryEmbeddings.targetType,
-            memoryEmbeddings.targetId,
-            memoryEmbeddings.provider,
-            memoryEmbeddings.model,
-          ],
-          set: {
-            vectorBlob: blobValue,
-            vectorJson: null,
-            dimensions: dense.length,
-            contentHash,
-            updatedAt: now,
-          },
-        })
-        .run();
-    } catch (err) {
-      log.warn(
-        { err, slug },
-        "Failed to write concept-page embedding cache row",
-      );
-    }
+  // would be a no-op — skip it. Backend rotation flips a cache hit into a
+  // fresh embed (see `rotated` above); the `*Fresh` flags capture that so
+  // the new vector overwrites the now-stale cache row under the new
+  // provider/model identity. Best-effort: write failure is not fatal, we
+  // still want the Qdrant upsert below to fire.
+  if (bodyFresh) {
+    writeEmbeddingCache(db, {
+      slug,
+      cacheId: slug,
+      dense: bodyDense,
+      contentHash: bodyContentHash,
+      provider: writeProvider,
+      model: writeModel,
+      now,
+    });
+  }
+  if (hasSummary && summaryFresh && summaryDense && summaryContentHash) {
+    writeEmbeddingCache(db, {
+      slug,
+      cacheId: summaryCacheId,
+      dense: summaryDense,
+      contentHash: summaryContentHash,
+      provider: writeProvider,
+      model: writeModel,
+      now,
+    });
   }
 
   // Apply anisotropy correction at the boundary between the (raw) cached
@@ -210,19 +272,129 @@ export async function embedConceptPageJob(
   // the cache survives and the (cheap) correction math reruns over each
   // cached vector. Pass-through when no calibration is fit yet.
   const correctedDense = await applyCorrectionIfCalibrated(
-    dense,
-    provider,
-    model,
+    bodyDense,
+    writeProvider,
+    writeModel,
   );
+  const correctedSummaryDense = summaryDense
+    ? await applyCorrectionIfCalibrated(summaryDense, writeProvider, writeModel)
+    : undefined;
 
   await upsertConceptPageEmbedding({
     slug,
     dense: correctedDense,
     sparse,
+    summary:
+      correctedSummaryDense && summarySparse
+        ? { dense: correctedSummaryDense, sparse: summarySparse }
+        : undefined,
     updatedAt: now,
   });
 }
 
+/** SQLite cache row shape returned by `readEmbeddingCache`. */
+interface EmbeddingCacheEntry {
+  dense: number[];
+  contentHash: string;
+}
+
+/**
+ * Look up a cached dense vector keyed on `(targetType, targetId, provider,
+ * model)`. Returns the row only when the persisted dimensions match the
+ * configured expectation — a stale row from a previous `vectorSize` is
+ * treated as a cache miss so the caller re-embeds.
+ */
+function readEmbeddingCache(
+  db: ReturnType<typeof getDb>,
+  cacheId: string,
+  provider: string,
+  model: string,
+  expectedDim: number,
+): EmbeddingCacheEntry | null {
+  const row = db
+    .select({
+      vectorBlob: memoryEmbeddings.vectorBlob,
+      vectorJson: memoryEmbeddings.vectorJson,
+      dimensions: memoryEmbeddings.dimensions,
+      contentHash: memoryEmbeddings.contentHash,
+    })
+    .from(memoryEmbeddings)
+    .where(
+      and(
+        eq(memoryEmbeddings.targetType, CONCEPT_PAGE_TARGET_TYPE),
+        eq(memoryEmbeddings.targetId, cacheId),
+        eq(memoryEmbeddings.provider, provider),
+        eq(memoryEmbeddings.model, model),
+      ),
+    )
+    .get();
+  if (!row || row.dimensions !== expectedDim) return null;
+  // A row without a contentHash is a legacy/corrupt entry — treat as a miss
+  // and force a re-embed rather than misalign the cache key.
+  if (row.contentHash === null) return null;
+  const dense = row.vectorBlob
+    ? blobToVector(row.vectorBlob as Buffer)
+    : (JSON.parse(row.vectorJson!) as number[]);
+  return { dense, contentHash: row.contentHash };
+}
+
+/**
+ * Persist a freshly embedded dense vector in the SQLite cache. Best-effort:
+ * a write failure is logged and swallowed so the Qdrant upsert still runs.
+ */
+function writeEmbeddingCache(
+  db: ReturnType<typeof getDb>,
+  params: {
+    slug: string;
+    cacheId: string;
+    dense: number[];
+    contentHash: string;
+    provider: string;
+    model: string;
+    now: number;
+  },
+): void {
+  const { slug, cacheId, dense, contentHash, provider, model, now } = params;
+  try {
+    const blobValue = vectorToBlob(dense);
+    db.insert(memoryEmbeddings)
+      .values({
+        id: randomUUID(),
+        targetType: CONCEPT_PAGE_TARGET_TYPE,
+        targetId: cacheId,
+        provider,
+        model,
+        dimensions: dense.length,
+        vectorBlob: blobValue,
+        vectorJson: null,
+        contentHash,
+        createdAt: now,
+        updatedAt: now,
+      })
+      .onConflictDoUpdate({
+        target: [
+          memoryEmbeddings.targetType,
+          memoryEmbeddings.targetId,
+          memoryEmbeddings.provider,
+          memoryEmbeddings.model,
+        ],
+        set: {
+          vectorBlob: blobValue,
+          vectorJson: null,
+          dimensions: dense.length,
+          contentHash,
+          updatedAt: now,
+        },
+      })
+      .run();
+  } catch (err) {
+    log.warn(
+      { err, slug, cacheId },
+      "Failed to write concept-page embedding cache row",
+    );
+  }
+}
+
 /**
  * Enqueue an `embed_concept_page` job (async, fire-and-forget). Modeled on
  * `enqueuePkbIndexJob` — callers that want a slug re-embedded after a write

package/src/memory/jobs-store.ts

@@ -42,7 +42,8 @@ export type MemoryJobType =
   | "memory_v2_consolidate"
   | "memory_v2_migrate"
   | "memory_v2_reembed"
-  | "memory_v2_activation_recompute";
+  | "memory_v2_activation_recompute"
+  | "memory_retrospective";
 
 export const EMBED_JOB_TYPES: MemoryJobType[] = [
   "embed_segment",
@@ -52,6 +53,7 @@ export const EMBED_JOB_TYPES: MemoryJobType[] = [
   "embed_graph_node",
   "embed_pkb_file",
   "graph_trigger_embed",
+  "embed_concept_page",
 ];
 
 export const SLOW_LLM_JOB_TYPES: MemoryJobType[] = [
@@ -65,6 +67,7 @@ export const SLOW_LLM_JOB_TYPES: MemoryJobType[] = [
   "memory_v2_sweep",
   "memory_v2_consolidate",
   "memory_v2_migrate",
+  "memory_retrospective",
   "backfill",
   "graph_bootstrap",
 ];
@@ -251,6 +254,53 @@ export function upsertAutoAnalysisJob(
   }
 }
 
+/**
+ * Upsert a pending `memory_retrospective` job keyed by `conversationId`. All
+ * four retrospective triggers (interval, message_count, compaction,
+ * lifecycle) collapse into a single pending row per conversation — rapid
+ * triggers coalesce instead of double-firing. The `runAfter` parameter on a
+ * follow-up enqueue overwrites the existing row's `runAfter` so a sooner
+ * trigger can pull a later-scheduled job earlier; a later-scheduled trigger
+ * does NOT push a sooner-scheduled row further out (consumer takes the
+ * minimum). The trigger metadata is intentionally not retained — it is only
+ * useful for observability at enqueue time.
+ */
+export function upsertMemoryRetrospectiveJob(
+  payload: { conversationId: string },
+  runAfter: number = Date.now(),
+  dbOverride?: Parameters<ReturnType<typeof getDb>["transaction"]>[0] extends (
+    tx: infer T,
+  ) => unknown
+    ? T
+    : never,
+): void {
+  const db = dbOverride ?? getDb();
+  const existing = db
+    .select()
+    .from(memoryJobs)
+    .where(
+      and(
+        eq(memoryJobs.type, "memory_retrospective"),
+        eq(memoryJobs.status, "pending"),
+        sql`json_extract(${memoryJobs.payload}, '$.conversationId') = ${payload.conversationId}`,
+      ),
+    )
+    .get();
+  if (existing) {
+    // Take the minimum of the existing and incoming runAfter so the earliest
+    // trigger always wins. A later trigger never pushes work further out.
+    const nextRunAfter = Math.min(existing.runAfter, runAfter);
+    if (nextRunAfter !== existing.runAfter) {
+      db.update(memoryJobs)
+        .set({ runAfter: nextRunAfter, updatedAt: Date.now() })
+        .where(eq(memoryJobs.id, existing.id))
+        .run();
+    }
+    return;
+  }
+  enqueueMemoryJob("memory_retrospective", payload, runAfter, dbOverride);
+}
+
 /**
  * Check whether a pending or running job of the given type already exists.
  * Used to prevent duplicate enqueues for long-running maintenance jobs.

package/src/memory/jobs-worker.ts

@@ -11,7 +11,6 @@ import {
   getLastScheduledCleanupEnqueueMs,
   markScheduledCleanupEnqueued,
 } from "./cleanup-schedule-state.js";
-import { isMemoryV2ReadActive } from "./context-search/sources/memory-v2.js";
 import { conversationAnalyzeJob } from "./conversation-analyze-job.js";
 import { maybeRunDbMaintenance } from "./db-maintenance.js";
 import { bootstrapFromHistory } from "./graph/bootstrap.js";
@@ -68,6 +67,8 @@ import {
   resetRunningJobsToPending,
   SLOW_LLM_JOB_TYPES,
 } from "./jobs-store.js";
+import { memoryRetrospectiveJob } from "./memory-retrospective-job.js";
+import { sweepOrphanMemoryRetrospectiveConversations } from "./memory-retrospective-startup-cleanup.js";
 import { QdrantCircuitOpenError } from "./qdrant-circuit-breaker.js";
 import {
   memoryV2ActivationRecomputeJob,
@@ -79,6 +80,26 @@ import { memoryV2SweepJob } from "./v2/sweep-job.js";
 
 const log = getLogger("memory-jobs-worker");
 
+/**
+ * V1 job types that read or write the v1 Qdrant collection via
+ * `getQdrantClient()`. When `memory.v2.enabled` is true, the v1 client is
+ * intentionally left uninitialized in `lifecycle.ts`, so these handlers would
+ * throw `BackendUnavailableError` and accumulate as a deferred backlog. Stale
+ * rows from indexer.ts and other unguarded enqueue sites must short-circuit
+ * here for the same reason `graph_extract` does below.
+ */
+const V1_QDRANT_JOB_TYPES = new Set<MemoryJobType>([
+  "embed_segment",
+  "embed_summary",
+  "embed_media",
+  "embed_attachment",
+  "embed_graph_node",
+  "embed_pkb_file",
+  "graph_trigger_embed",
+  "rebuild_index",
+  "delete_qdrant_vectors",
+]);
+
 /**
  * Job types whose handlers have been removed. Existing rows may still sit in
  * the database — the worker completes them silently instead of throwing.
@@ -112,6 +133,19 @@ export function startMemoryJobsWorker(): MemoryJobsWorker {
     log.info({ recovered }, "Recovered stale running memory jobs");
   }
 
+  // After running-job recovery (so legitimate in-flight retries aren't
+  // swept), clean up orphan memory-retrospective background conversations
+  // left behind by daemon crashes mid-job. Best-effort — never block worker
+  // startup on cleanup failures.
+  try {
+    sweepOrphanMemoryRetrospectiveConversations();
+  } catch (err) {
+    log.warn(
+      { err },
+      "Memory-retrospective startup cleanup failed; continuing worker startup",
+    );
+  }
+
   let stopped = false;
   let tickRunning = false;
   let timer: ReturnType<typeof setTimeout>;
@@ -125,16 +159,24 @@ export function startMemoryJobsWorker(): MemoryJobsWorker {
         enableScheduledCleanup: true,
       });
       if (processed > 0) {
-        currentIntervalMs = POLL_INTERVAL_MIN_MS;
+        // Per-tick claim budget equals the lane caps, so when a tick
+        // processed work the next tick must run immediately to drain any
+        // remaining backlog. Holding the 1.5s floor between ticks would cap
+        // sustained throughput at lane-cap jobs per 1.5s and starve large
+        // backlogs of short jobs.
+        currentIntervalMs = 0;
       } else {
         currentIntervalMs = Math.min(
-          currentIntervalMs * 2,
+          Math.max(currentIntervalMs * 2, POLL_INTERVAL_MIN_MS),
           POLL_INTERVAL_MAX_MS,
         );
       }
     } catch (err) {
       log.error({ err }, "Memory worker tick failed");
-      currentIntervalMs = Math.min(currentIntervalMs * 2, POLL_INTERVAL_MAX_MS);
+      currentIntervalMs = Math.min(
+        Math.max(currentIntervalMs * 2, POLL_INTERVAL_MIN_MS),
+        POLL_INTERVAL_MAX_MS,
+      );
     } finally {
       tickRunning = false;
     }
@@ -463,6 +505,9 @@ async function processJob(
   job: MemoryJob,
   config: AssistantConfig,
 ): Promise<void> {
+  if (config.memory.v2.enabled && V1_QDRANT_JOB_TYPES.has(job.type)) {
+    return;
+  }
   switch (job.type) {
     case "embed_segment":
       await embedSegmentJob(job, config);
@@ -510,6 +555,11 @@ async function processJob(
       await embedGraphTriggerJob(job, config);
       return;
     case "graph_extract":
+      // Stale rows enqueued before v2 was enabled (or by any unguarded v1
+      // path) must not consume embedding/extraction budget when v2 is on.
+      if (config.memory.v2.enabled) {
+        return;
+      }
       await graphExtractJob(job, config);
       return;
     case "conversation_analyze":
@@ -551,6 +601,9 @@ async function processJob(
     case "memory_v2_activation_recompute":
       await memoryV2ActivationRecomputeJob(job, config);
       return;
+    case "memory_retrospective":
+      await memoryRetrospectiveJob(job, config);
+      return;
 
     default: {
       const rawType = (job as { type: string }).type;
@@ -623,8 +676,8 @@ export const GRAPH_MAINTENANCE_CHECKPOINTS = {
  * Enqueue periodic graph maintenance jobs.
  *
  * Mutually exclusive between v1 and v2:
- *   - v2 active (both `memory-v2-enabled` flag and `memory.v2.enabled`
- *     config on) → only `memory_v2_consolidate` is scheduled.
+ *   - v2 active (`memory.v2.enabled` on) → only `memory_v2_consolidate` is
+ *     scheduled.
  *   - v2 inactive → the four v1 entries (decay, consolidate, pattern_scan,
  *     narrative) are scheduled instead.
  *
@@ -643,7 +696,7 @@ export function maybeEnqueueGraphMaintenanceJobs(
   config: AssistantConfig,
   nowMs = Date.now(),
 ): void {
-  const v2Active = isMemoryV2ReadActive(config);
+  const v2Active = config.memory.v2.enabled;
 
   const schedule: Array<{
     key: string;