@vellumai/assistant 0.6.0 → 0.6.1

package/src/memory/graph/retriever.ts

@@ -35,7 +35,12 @@ import {
   evaluateTemporalTriggers,
   type TriggeredResult,
 } from "./triggers.js";
-import type { MemoryEdge, MemoryNode, ScoredNode } from "./types.js";
+import type {
+  MemoryEdge,
+  MemoryNode,
+  RetrievalMetrics,
+  ScoredNode,
+} from "./types.js";
 import { isCapabilityNode } from "./types.js";
 
 const log = getLogger("graph-retriever");
@@ -78,7 +83,7 @@ async function rerankAndDedup(
     const provider = await getConfiguredProvider();
     if (!provider) return candidates.slice(0, maxNodes);
 
-    // Compact listing for the LLM: numbered index + age + first 100 chars
+    // Numbered listing for the LLM: index + age + full content
     const now = Date.now();
     const listing = candidates
       .map((s, i) => {
@@ -87,11 +92,7 @@ async function rerankAndDedup(
           ageDays < 1
             ? `${Math.floor(ageDays * 24)}h`
             : `${Math.floor(ageDays)}d`;
-        const preview =
-          s.node.content.length > 100
-            ? s.node.content.slice(0, 100) + "…"
-            : s.node.content;
-        return `${i + 1}. (${age}) ${preview}`;
+        return `${i + 1}. (${age}) ${s.node.content}`;
       })
       .join("\n");
 
@@ -176,10 +177,11 @@ async function dedupForTurn(
   candidates: ScoredNode[],
   maxNodes: number,
   query: string,
-): Promise<ScoredNode[]> {
+): Promise<{ nodes: ScoredNode[]; llmApplied: boolean }> {
   try {
     const provider = await getConfiguredProvider();
-    if (!provider) return candidates.slice(0, maxNodes);
+    if (!provider)
+      return { nodes: candidates.slice(0, maxNodes), llmApplied: false };
 
     const now = Date.now();
     const listing = candidates
@@ -189,11 +191,7 @@ async function dedupForTurn(
           ageDays < 1
             ? `${Math.floor(ageDays * 24)}h`
             : `${Math.floor(ageDays)}d`;
-        const preview =
-          s.node.content.length > 150
-            ? s.node.content.slice(0, 150) + "…"
-            : s.node.content;
-        return `${i + 1}. (${age}) ${preview}`;
+        return `${i + 1}. (${age}) ${s.node.content}`;
       })
       .join("\n");
 
@@ -211,6 +209,98 @@ async function dedupForTurn(
       },
     );
 
+    const toolBlock = extractToolUse(response);
+    if (!toolBlock)
+      return { nodes: candidates.slice(0, maxNodes), llmApplied: false };
+
+    const input = toolBlock.input as { items?: number[] };
+    if (!input.items?.length)
+      return { nodes: candidates.slice(0, maxNodes), llmApplied: false };
+
+    const reranked: ScoredNode[] = [];
+    const seen = new Set<number>();
+    for (const num of input.items) {
+      const idx = num - 1;
+      if (idx >= 0 && idx < candidates.length && !seen.has(idx)) {
+        reranked.push(candidates[idx]);
+        seen.add(idx);
+      }
+    }
+
+    return reranked.length > 0
+      ? { nodes: reranked.slice(0, maxNodes), llmApplied: true }
+      : { nodes: candidates.slice(0, maxNodes), llmApplied: false };
+  } catch (err) {
+    log.warn(
+      { err: err instanceof Error ? err.message : String(err) },
+      "Per-turn dedup+rerank failed, using scored order",
+    );
+    return { nodes: candidates.slice(0, maxNodes), llmApplied: false };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Cross-category dedup — dedup-only (no relevance filtering)
+// ---------------------------------------------------------------------------
+
+const DEDUP_ITEMS_TOOL = {
+  name: "select_items",
+  description:
+    "Select ALL items that survive deduplication. When multiple items describe the same event/fact, keep only the richest version. Do not filter by relevance — keep everything that is not a duplicate.",
+  input_schema: {
+    type: "object" as const,
+    properties: {
+      items: {
+        type: "array" as const,
+        description:
+          "Item numbers to keep (1-indexed). Remove duplicates — when multiple entries describe the same event/fact, keep ONLY the richest version. Keep all non-duplicate items.",
+        items: { type: "number" as const },
+      },
+    },
+    required: ["items"] as const,
+  },
+};
+
+/**
+ * Dedup-only pass for cross-category duplicate removal. Unlike `dedupForTurn`,
+ * this does NOT filter by relevance to a query — it ONLY removes duplicates
+ * and keeps everything else. Used after context load to catch topic-level
+ * duplicates across reserved categories and serendipity.
+ */
+async function dedupCrossCategory(
+  candidates: ScoredNode[],
+  maxNodes: number,
+): Promise<ScoredNode[]> {
+  try {
+    const provider = await getConfiguredProvider();
+    if (!provider) return candidates.slice(0, maxNodes);
+
+    const now = Date.now();
+    const listing = candidates
+      .map((s, i) => {
+        const ageDays = (now - s.node.created) / (1000 * 60 * 60 * 24);
+        const age =
+          ageDays < 1
+            ? `${Math.floor(ageDays * 24)}h`
+            : `${Math.floor(ageDays)}d`;
+        return `${i + 1}. (${age}) ${s.node.content}`;
+      })
+      .join("\n");
+
+    const response = await provider.sendMessage(
+      [userMessage(listing)],
+      [DEDUP_ITEMS_TOOL],
+      `Deduplicate the following numbered items. When multiple items describe the same event, fact, or status, keep ONLY the richest version. Keep ALL items that are not duplicates — do not filter by relevance or topic. Call the select_items tool with every item that survives dedup.`,
+      {
+        config: {
+          modelIntent: "latency-optimized" as const,
+          tool_choice: { type: "tool" as const, name: "select_items" },
+          thinking: { type: "disabled" },
+          temperature: 0,
+        },
+      },
+    );
+
     const toolBlock = extractToolUse(response);
     if (!toolBlock) return candidates.slice(0, maxNodes);
 
@@ -233,7 +323,7 @@ async function dedupForTurn(
   } catch (err) {
     log.warn(
       { err: err instanceof Error ? err.message : String(err) },
-      "Per-turn dedup+rerank failed, using scored order",
+      "Cross-category dedup failed, using original order",
     );
     return candidates.slice(0, maxNodes);
   }
@@ -263,6 +353,7 @@ export interface ContextLoadResult {
   serendipityNodes: ScoredNode[];
   triggeredNodes: TriggeredResult[];
   latencyMs: number;
+  metrics: RetrievalMetrics;
 }
 
 /**
@@ -287,15 +378,21 @@ export async function loadContextMemory(
 
   // 1. Embed recent conversation summaries as retrieval queries
   let queryVector: number[] | null = null;
+  let embeddingProvider: string | null = null;
+  let embeddingModel: string | null = null;
+  let contextQueryText: string | null = null;
   if (opts.recentSummaries.length > 0) {
     try {
       const queryText = opts.recentSummaries.join("\n\n");
       const truncated =
         queryText.length > 3000 ? queryText.slice(0, 3000) : queryText;
+      contextQueryText = truncated;
       const result = await embedWithRetry(opts.config, [truncated], {
         signal: opts.signal,
       });
       queryVector = result.vectors[0] ?? null;
+      embeddingProvider = result.provider;
+      embeddingModel = result.model;
     } catch (err) {
       log.warn({ err }, "Failed to embed summaries for context load");
     }
@@ -303,7 +400,9 @@ export async function loadContextMemory(
 
   // 2. Hybrid retrieval from Qdrant (dense search on graph_node points)
   const semanticCandidateIds = new Map<string, number>(); // nodeId → score
+  let hybridSearchLatencyMs = 0;
   if (queryVector) {
+    const searchStart = Date.now();
     try {
       const results = await searchGraphNodes(queryVector, maxNodes * 3, [
         opts.scopeId,
@@ -313,8 +412,11 @@ export async function loadContextMemory(
       }
     } catch (err) {
       log.warn({ err }, "Qdrant search failed for context load");
+    } finally {
+      hybridSearchLatencyMs = Date.now() - searchStart;
     }
   }
+  const pureSemanticHits = semanticCandidateIds.size;
 
   // Also include top-significance nodes as a fallback
   const topSignificance = queryNodes({
@@ -498,105 +600,15 @@ export async function loadContextMemory(
     },
   );
 
-  // 6. Reserve slots for recent prospective nodes (commitments, tasks, plans).
-  //    These MUST surface at conversation start regardless of score — if the user
-  //    said "I have a doctor appointment tomorrow," Velissa must remember it.
-  const PROSPECTIVE_RESERVE = 10;
-  const recentProspective = queryNodes({
-    scopeId: opts.scopeId,
-    types: ["prospective"],
-    fidelityNot: ["gone"],
-    createdAfter: nowMs - 3 * 24 * 60 * 60 * 1000, // last 3 days
-    limit: PROSPECTIVE_RESERVE,
-  });
-
-  // Filter out prospective nodes that have been superseded or resolved.
-  // A "supersedes" or "resolved-by" edge targeting a node means its
-  // content has been replaced by a newer memory — stop force-surfacing it.
-  const unresolvedProspective = recentProspective.filter((node) => {
-    const incoming = getEdgesForNode(node.id, "incoming");
-    return !incoming.some(
-      (e) =>
-        e.relationship === "supersedes" || e.relationship === "resolved-by",
-    );
-  });
-
-  // Score them so they have breakdowns, but they're guaranteed inclusion
-  const prospectiveIds = new Set(unresolvedProspective.map((n) => n.id));
-  const reservedNodes: ScoredNode[] = unresolvedProspective.map((node) => {
-    const existing = scored.find((s) => s.node.id === node.id);
-    if (existing) return existing;
-    return scoreCandidate(node, {
-      semanticSimilarity: 0,
-      effectiveSignificance: computeEffectiveSignificance(node, nowMs),
-      emotionalIntensity: node.emotionalCharge.intensity,
-      temporalBoost: (computeTemporalBoost(node, now) + 1) / 2,
-      recencyBoost: computeRecencyBoost(node, nowMs),
-      triggerBoost: 0,
-      activationBoost: 0,
-    });
-  });
-
-  // Reserve slots for upcoming events (nodes with event dates in the future).
-  // Like prospective reservation, these MUST surface — if the user said
-  // "I have a flight Tuesday," the assistant must remember it regardless of score.
-  const UPCOMING_RESERVE = 5;
-  const upcomingEvents = queryNodes({
-    scopeId: opts.scopeId,
-    fidelityNot: ["gone"],
-    hasEventDate: true,
-    eventDateAfter: nowMs,
-    eventDateBefore: nowMs + 30 * 24 * 60 * 60 * 1000, // next 30 days
-    limit: 20, // Fetch extra candidates — post-sort by proximity below
-  });
-
-  // Sort by event date ascending so soonest events get reserved first
-  // (queryNodes sorts by significance, which would drop a tomorrow-event
-  // with low significance in favor of a 3-weeks-away high-significance one)
-  upcomingEvents.sort((a, b) => (a.eventDate ?? 0) - (b.eventDate ?? 0));
-
-  const unresolvedUpcoming = upcomingEvents
-    .filter((node) => {
-      if (prospectiveIds.has(node.id)) return false; // already reserved as prospective
-      const incoming = getEdgesForNode(node.id, "incoming");
-      return !incoming.some(
-        (e) =>
-          e.relationship === "supersedes" || e.relationship === "resolved-by",
-      );
-    })
-    .slice(0, UPCOMING_RESERVE);
-
-  const upcomingIds = new Set(unresolvedUpcoming.map((n) => n.id));
-  const reservedUpcoming: ScoredNode[] = unresolvedUpcoming.map((node) => {
-    const existing = scored.find((s) => s.node.id === node.id);
-    if (existing) return existing;
-    return scoreCandidate(node, {
-      semanticSimilarity: 0,
-      effectiveSignificance: computeEffectiveSignificance(node, nowMs),
-      emotionalIntensity: node.emotionalCharge.intensity,
-      temporalBoost: (computeTemporalBoost(node, now) + 1) / 2,
-      recencyBoost: computeRecencyBoost(node, nowMs),
-      triggerBoost: 0,
-      activationBoost: 0,
-    });
-  });
-
-  // Remove reserved nodes and all procedural nodes from the main pool.
-  // Procedural nodes have dedicated reserved slots — any that didn't make
-  // the cut shouldn't compete with organic memories for general slots.
-  const mainPool = scored.filter(
-    (s) =>
-      !isCapabilityNode(s.node) &&
-      !prospectiveIds.has(s.node.id) &&
-      !upcomingIds.has(s.node.id),
-  );
+  // 6. Remove procedural nodes from the main pool — they have dedicated
+  //    reserved slots and shouldn't compete with organic memories.
+  //    Prospective/upcoming reserves were removed in favor of the PKB
+  //    (personal knowledge base) which handles commitments and schedule
+  //    via always-loaded flat files.
+  const mainPool = scored.filter((s) => !isCapabilityNode(s.node));
   const mainSlots = Math.max(
     0,
-    maxNodes -
-      serendipitySlots -
-      reservedNodes.length -
-      reservedUpcoming.length -
-      reservedCapabilities.length,
+    maxNodes - serendipitySlots - reservedCapabilities.length,
   );
 
   // 7. LLM re-ranking on the main pool: dedup + select
@@ -606,18 +618,14 @@ export async function loadContextMemory(
     opts.config,
   );
 
-  // 8. Combine: reserved prospective + reserved upcoming + reserved capabilities + reranked main pool
-  const deterministic = [
-    ...reservedNodes,
-    ...reservedUpcoming,
-    ...reservedCapabilities,
-    ...reranked,
-  ].slice(0, maxNodes - serendipitySlots);
+  // 8. Combine: reserved capabilities + reranked main pool
+  const deterministic = [...reservedCapabilities, ...reranked].slice(
+    0,
+    maxNodes - serendipitySlots,
+  );
   // Exclude procedural nodes from serendipity — they have reserved slots
   // and shouldn't appear as random wildcard picks.
-  const serendipityPool = scored.filter(
-    (s) => !isCapabilityNode(s.node),
-  );
+  const serendipityPool = scored.filter((s) => !isCapabilityNode(s.node));
   const serendipityPicks = sampleSerendipity(serendipityPool, serendipitySlots);
 
   // Deduplicate serendipity against deterministic
@@ -626,11 +634,56 @@ export async function loadContextMemory(
     (s) => !deterministicIds.has(s.node.id),
   );
 
+  // 9. Cross-category dedup: catch topic-level duplicates across reserved
+  //    categories (prospective, upcoming, capabilities) and serendipity.
+  //    Only runs when the combined set is large enough to warrant an LLM call.
+  const CROSS_DEDUP_THRESHOLD = 15;
+  const combined = [...deterministic, ...uniqueSerendipity];
+  let dedupedDeterministic = deterministic;
+  let dedupedSerendipity = uniqueSerendipity;
+
+  if (combined.length > CROSS_DEDUP_THRESHOLD) {
+    const deduped = await dedupCrossCategory(
+      combined,
+      combined.length, // preserve all non-duplicate nodes
+    );
+
+    // Re-split into deterministic vs serendipity by checking original membership
+    dedupedDeterministic = deduped.filter((s) =>
+      deterministicIds.has(s.node.id),
+    );
+    dedupedSerendipity = deduped.filter(
+      (s) => !deterministicIds.has(s.node.id),
+    );
+  }
+
+  const TOP_N = 20;
+  const topCandidates = scored.slice(0, TOP_N).map((s) => ({
+    nodeId: s.node.id,
+    type: s.node.type,
+    score: s.score,
+    semanticSimilarity: s.scoreBreakdown.semanticSimilarity,
+    recencyBoost: s.scoreBreakdown.recencyBoost,
+  }));
+
   return {
-    nodes: deterministic,
-    serendipityNodes: uniqueSerendipity,
+    nodes: dedupedDeterministic,
+    serendipityNodes: dedupedSerendipity,
     triggeredNodes: allTriggered,
     latencyMs: Date.now() - start,
+    metrics: {
+      semanticHits: pureSemanticHits,
+      mergedCount: scored.length,
+      selectedCount: dedupedDeterministic.length + dedupedSerendipity.length,
+      tier1Count: 0,
+      tier2Count: reservedCapabilities.length,
+      hybridSearchLatencyMs,
+      sparseVectorUsed: false,
+      embeddingProvider,
+      embeddingModel,
+      queryContext: contextQueryText,
+      topCandidates,
+    },
   };
 }
 
@@ -654,9 +707,12 @@ export interface TurnRetrievalOpts {
 export interface TurnRetrievalResult {
   /** New nodes to inject (not already in context). */
   nodes: ScoredNode[];
+  /** Serendipity picks included in nodes. */
+  serendipityNodes: ScoredNode[];
   /** Triggers that fired this turn. */
   triggeredNodes: TriggeredResult[];
   latencyMs: number;
+  metrics: RetrievalMetrics;
 }
 
 /**
@@ -674,6 +730,24 @@ export async function retrieveForTurn(
   const now = new Date();
   const nowMs = now.getTime();
 
+  let embeddingProvider: string | null = null;
+  let embeddingModel: string | null = null;
+  let hybridSearchLatencyMs = 0;
+
+  const ZERO_METRICS: RetrievalMetrics = {
+    semanticHits: 0,
+    mergedCount: 0,
+    selectedCount: 0,
+    tier1Count: 0,
+    tier2Count: 0,
+    hybridSearchLatencyMs: 0,
+    sparseVectorUsed: false,
+    embeddingProvider: null,
+    embeddingModel: null,
+    queryContext: null,
+    topCandidates: [],
+  };
+
   // 1. Build query from last exchange
   const queryText = [opts.assistantLastMessage, opts.userLastMessage]
     .filter((m) => m.length > 0)
@@ -685,6 +759,7 @@ export async function retrieveForTurn(
     (b): b is ImageContent => b.type === "image",
   );
   const allCandidateIds = new Map<string, number>(); // nodeId → best score
+  const searchStart = Date.now();
 
   if (imageBlocks.length > 0) {
     try {
@@ -705,6 +780,10 @@ export async function retrieveForTurn(
           const imgResult = await embedWithRetry(opts.config, [imageInput], {
             signal: opts.signal,
           });
+          if (!embeddingProvider) {
+            embeddingProvider = imgResult.provider;
+            embeddingModel = imgResult.model;
+          }
           const imgVector = imgResult.vectors[0];
           if (imgVector) {
             const imgResults = await searchGraphNodes(imgVector, 40, [
@@ -723,7 +802,20 @@ export async function retrieveForTurn(
   }
 
   if (queryText.trim().length === 0 && allCandidateIds.size === 0) {
-    return { nodes: [], triggeredNodes: [], latencyMs: Date.now() - start };
+    return {
+      nodes: [],
+      serendipityNodes: [],
+      triggeredNodes: [],
+      latencyMs: Date.now() - start,
+      metrics: {
+        ...ZERO_METRICS,
+        hybridSearchLatencyMs:
+          imageBlocks.length > 0 ? Date.now() - searchStart : 0,
+        embeddingProvider,
+        embeddingModel,
+        queryContext: queryText || null,
+      },
+    };
   }
 
   // Chunk if too large (8k token ≈ 32k chars conservative estimate)
@@ -764,6 +856,8 @@ export async function retrieveForTurn(
       const embedResults = await embedWithRetry(opts.config, chunks, {
         signal: opts.signal,
       });
+      embeddingProvider = embedResults.provider;
+      embeddingModel = embedResults.model;
       queryEmbeddings = embedResults.vectors;
 
       const searchPromises = queryEmbeddings.map((vec) =>
@@ -777,14 +871,35 @@ export async function retrieveForTurn(
           allCandidateIds.set(r.nodeId, Math.max(current, r.score));
         }
       }
+      hybridSearchLatencyMs = Date.now() - searchStart;
     } catch (err) {
       log.warn({ err }, "Embedding/search failed for turn retrieval");
       if (allCandidateIds.size === 0) {
-        return { nodes: [], triggeredNodes: [], latencyMs: Date.now() - start };
+        return {
+          nodes: [],
+          serendipityNodes: [],
+          triggeredNodes: [],
+          latencyMs: Date.now() - start,
+          metrics: {
+            ...ZERO_METRICS,
+            hybridSearchLatencyMs: Date.now() - searchStart,
+            embeddingProvider,
+            embeddingModel,
+            queryContext: queryText || null,
+          },
+        };
       }
     }
   }
 
+  // Capture search latency for image-only searches (text path sets it inside its try block)
+  if (hybridSearchLatencyMs === 0 && allCandidateIds.size > 0) {
+    hybridSearchLatencyMs = Date.now() - searchStart;
+  }
+
+  // Snapshot pure vector-search results before triggers inflate the set
+  const pureSemanticHits = allCandidateIds.size;
+
   // 3. Evaluate semantic triggers
   const semanticTriggers = getActiveTriggersByType("semantic", opts.scopeId);
   const triggeredSemantic =
@@ -813,20 +928,36 @@ export async function retrieveForTurn(
   if (newCandidateIds.length === 0) {
     return {
       nodes: [],
+      serendipityNodes: [],
       triggeredNodes: triggeredSemantic,
       latencyMs: Date.now() - start,
+      metrics: {
+        ...ZERO_METRICS,
+        semanticHits: pureSemanticHits,
+        hybridSearchLatencyMs,
+        embeddingProvider,
+        embeddingModel,
+        queryContext: queryText || null,
+      },
     };
   }
 
   // 5. Hydrate and score
   const nodes = getNodesByIds(newCandidateIds);
   const scored: ScoredNode[] = [];
+  const capabilityCandidates: { node: MemoryNode; sim: number }[] = [];
 
   for (const node of nodes) {
     if (node.fidelity === "gone") continue;
-    // Procedural nodes (capabilities) have reserved slots at context-load
-    // and shouldn't compete with organic memories in per-turn injection.
-    if (isCapabilityNode(node)) continue;
+    // Capability nodes (auto-seeded skills/CLI) are excluded from the general
+    // scoring pool — they compete in the dedicated procedural reserve below.
+    if (isCapabilityNode(node)) {
+      capabilityCandidates.push({
+        node,
+        sim: allCandidateIds.get(node.id) ?? 0,
+      });
+      continue;
+    }
 
     const semanticSim = allCandidateIds.get(node.id) ?? 0;
     const effectiveSig = computeEffectiveSignificance(node, nowMs);
@@ -853,156 +984,128 @@ export async function retrieveForTurn(
     );
   }
 
+  // 5b. Reserve slots for capability nodes (skills/CLI).
+  // Sourced from vector search candidates — only semantically relevant
+  // capabilities compete for reserved slots.
+  const PROCEDURAL_RESERVE = 3;
+
+  const proceduralCandidates = capabilityCandidates
+    .filter(({ node }) => !opts.tracker.isInContext(node.id))
+    .sort((a, b) => b.sim - a.sim);
+
+  const seenProcCapIds = new Set<string>();
+  const rankedProcedural = proceduralCandidates
+    .filter(({ node }) => {
+      const match = node.content.match(
+        /^skill:(\S+)\n|^cli:(\S+)\n|^\s*The ".*?" skill \(([^)]+)\)|^\s*The "assistant (\S+)" CLI command/,
+      );
+      const capId = match?.[1] ?? match?.[2] ?? match?.[3] ?? match?.[4];
+      if (capId) {
+        if (seenProcCapIds.has(capId)) return false;
+        seenProcCapIds.add(capId);
+      }
+      return true;
+    })
+    .slice(0, PROCEDURAL_RESERVE);
+
+  const proceduralScored: ScoredNode[] = rankedProcedural.map(({ node, sim }) =>
+    scoreCandidate(
+      node,
+      {
+        semanticSimilarity: sim,
+        effectiveSignificance: computeEffectiveSignificance(node, nowMs),
+        emotionalIntensity: node.emotionalCharge.intensity,
+        temporalBoost: (computeTemporalBoost(node, now) + 1) / 2,
+        recencyBoost: computeRecencyBoost(node, nowMs),
+        triggerBoost: triggerBoostMap.get(node.id) ?? 0,
+        activationBoost: 0,
+      },
+      PER_TURN_WEIGHTS,
+    ),
+  );
+
+  const PROCEDURAL_SIM_FLOOR = 0.15;
+  const proceduralInjected = proceduralScored.filter(
+    (s) => s.scoreBreakdown.semanticSimilarity >= PROCEDURAL_SIM_FLOOR,
+  );
+  const proceduralIds = new Set(proceduralInjected.map((s) => s.node.id));
+
   // Sort and apply threshold — pull a wider pool for dedup, then trim
   scored.sort((a, b) => b.score - a.score);
   const INJECTION_THRESHOLD = 0.3;
-  const PRE_DEDUP_POOL = 40;
-  const MAX_INJECTED = 8;
+  const PRE_DEDUP_POOL = 20;
+  const MAX_INJECTED = 4;
   const pool = scored
     .filter((s) => s.score >= INJECTION_THRESHOLD)
     .slice(0, PRE_DEDUP_POOL);
 
   // Dedup + rerank with a fast model when the pool is large enough to warrant it
-  const injected =
-    pool.length > MAX_INJECTED
-      ? await dedupForTurn(pool, MAX_INJECTED, opts.userLastMessage)
-      : pool;
-
-  return {
-    nodes: injected,
-    triggeredNodes: triggeredSemantic,
-    latencyMs: Date.now() - start,
-  };
-}
-
-// ---------------------------------------------------------------------------
-// Periodic refresh — every N turns, replenish memory context
-// ---------------------------------------------------------------------------
-
-export interface RefreshOpts {
-  /** Recent turns (last 5-6) concatenated as text. */
-  recentTurnsText: string;
-  scopeId: string;
-  config: AssistantConfig;
-  tracker: InContextTracker;
-  signal?: AbortSignal;
-  /** Max new nodes to inject (default 10). */
-  maxNodes?: number;
-}
-
-export interface RefreshResult {
-  nodes: ScoredNode[];
-  latencyMs: number;
-}
-
-/** Default interval between refresh cycles. */
-export const REFRESH_INTERVAL_TURNS = 5;
-
-/**
- * Periodic context refresh. Runs every N turns to catch memories that
- * the per-turn injection missed due to its high threshold.
- *
- * Uses a wider window (recent 5-6 turns) as the query to capture the
- * evolved conversational vibe. No LLM re-ranking — pure embedding +
- * scoring for speed (~200ms).
- *
- * Also runs after compaction to replenish lost memory context.
- */
-export async function refreshContextMemory(
-  opts: RefreshOpts,
-): Promise<RefreshResult> {
-  const start = Date.now();
-  const now = new Date();
-  const nowMs = now.getTime();
-  const maxNodes = opts.maxNodes ?? 10;
-
-  if (opts.recentTurnsText.trim().length === 0) {
-    return { nodes: [], latencyMs: Date.now() - start };
-  }
-
-  // 1. Embed recent turns window
-  const queryText =
-    opts.recentTurnsText.length > 6000
-      ? opts.recentTurnsText.slice(-6000)
-      : opts.recentTurnsText;
-
-  let queryVector: number[] | null = null;
-  try {
-    const result = await embedWithRetry(opts.config, [queryText], {
-      signal: opts.signal,
-    });
-    queryVector = result.vectors[0] ?? null;
-  } catch (err) {
-    log.warn({ err }, "Embedding failed for context refresh");
-    return { nodes: [], latencyMs: Date.now() - start };
-  }
-
-  if (!queryVector) {
-    return { nodes: [], latencyMs: Date.now() - start };
+  let injected: ScoredNode[];
+  let llmDedupApplied = false;
+  if (pool.length > MAX_INJECTED) {
+    const result = await dedupForTurn(pool, MAX_INJECTED, opts.userLastMessage);
+    injected = result.nodes;
+    llmDedupApplied = result.llmApplied;
+  } else {
+    injected = pool;
   }
 
-  // 2. Search — cast a wider net than per-turn
-  let candidates: Array<{ nodeId: string; score: number }>;
-  try {
-    candidates = await searchGraphNodes(queryVector, maxNodes * 3, [
-      opts.scopeId,
+  // Remove procedural-reserved nodes from general set to avoid double-counting
+  const generalInjected = injected.filter((s) => !proceduralIds.has(s.node.id));
+
+  // Backfill vacated general slots from the remaining pool so we always
+  // return up to MAX_INJECTED general memories when eligible candidates exist.
+  // Only skip backfill when LLM dedup genuinely ran — it intentionally rejected
+  // items as duplicates/irrelevant. When dedupForTurn fell back to a plain
+  // top-N slice (no provider, tool call failure), backfill is still appropriate.
+  if (generalInjected.length < MAX_INJECTED && !llmDedupApplied) {
+    const usedIds = new Set([
+      ...generalInjected.map((s) => s.node.id),
+      ...proceduralIds,
     ]);
-  } catch (err) {
-    log.warn({ err }, "Qdrant search failed for context refresh");
-    return { nodes: [], latencyMs: Date.now() - start };
+    const backfillCandidates = pool.filter((s) => !usedIds.has(s.node.id));
+    const needed = MAX_INJECTED - generalInjected.length;
+    for (let i = 0; i < Math.min(needed, backfillCandidates.length); i++) {
+      generalInjected.push(backfillCandidates[i]);
+    }
   }
 
-  // 3. Filter to nodes NOT already in context
-  const newCandidates = candidates.filter(
-    (c) => !opts.tracker.isInContext(c.nodeId),
-  );
+  const allDeterministic = [...generalInjected, ...proceduralInjected];
+  const deterministicIds = new Set(allDeterministic.map((n) => n.node.id));
 
-  if (newCandidates.length === 0) {
-    return { nodes: [], latencyMs: Date.now() - start };
-  }
-
-  // 4. Hydrate and score
-  const nodes = getNodesByIds(newCandidates.map((c) => c.nodeId));
-  const candidateScoreMap = new Map(
-    newCandidates.map((c) => [c.nodeId, c.score]),
+  // Reserve 1 serendipity slot from scored candidates not in the deterministic set
+  const serendipityPool = scored.filter(
+    (s) => s.score >= INJECTION_THRESHOLD && !deterministicIds.has(s.node.id),
   );
-
-  const scored: ScoredNode[] = [];
-  for (const node of nodes) {
-    if (node.fidelity === "gone") continue;
-    if (isCapabilityNode(node)) continue;
-
-    const semanticSim = candidateScoreMap.get(node.id) ?? 0;
-    const effectiveSig = computeEffectiveSignificance(node, nowMs);
-    const temporal = computeTemporalBoost(node, now);
-    const recency = computeRecencyBoost(node, nowMs);
-
-    scored.push(
-      scoreCandidate(
-        node,
-        {
-          semanticSimilarity: semanticSim,
-          effectiveSignificance: effectiveSig,
-          emotionalIntensity: node.emotionalCharge.intensity,
-          temporalBoost: (temporal + 1) / 2,
-          recencyBoost: recency,
-          triggerBoost: 0,
-          activationBoost: 0,
-        },
-        PER_TURN_WEIGHTS,
-      ),
-    );
-  }
-
-  // 5. Return top N — lower threshold than per-turn since this is a periodic refresh
-  scored.sort((a, b) => b.score - a.score);
-  const REFRESH_THRESHOLD = 0.15;
-  const refreshed = scored
-    .filter((s) => s.score >= REFRESH_THRESHOLD)
-    .slice(0, maxNodes);
+  const serendipityPicks = sampleSerendipity(serendipityPool, 1);
+  const allInjected = [...allDeterministic, ...serendipityPicks];
+
+  const TOP_N = 20;
+  const topCandidates = scored.slice(0, TOP_N).map((s) => ({
+    nodeId: s.node.id,
+    type: s.node.type,
+    score: s.score,
+    semanticSimilarity: s.scoreBreakdown.semanticSimilarity,
+    recencyBoost: s.scoreBreakdown.recencyBoost,
+  }));
 
   return {
-    nodes: refreshed,
+    nodes: allInjected,
+    serendipityNodes: serendipityPicks,
+    triggeredNodes: triggeredSemantic,
     latencyMs: Date.now() - start,
+    metrics: {
+      semanticHits: pureSemanticHits,
+      mergedCount: scored.length,
+      selectedCount: allInjected.length,
+      tier1Count: 0,
+      tier2Count: 0,
+      hybridSearchLatencyMs,
+      sparseVectorUsed: false,
+      embeddingProvider,
+      embeddingModel,
+      queryContext: queryText || null,
+      topCandidates,
+    },
   };
 }