@vellumai/assistant 0.3.3 → 0.3.5

package/src/memory/recall-cache.ts

@@ -0,0 +1,107 @@
+import { createHash } from 'crypto';
+import type { MemoryRecallResult, MemoryRecallOptions } from './search/types.js';
+
+/**
+ * In-memory cache for memory recall results.
+ *
+ * The full retrieval pipeline (FTS5 + Qdrant + entity graph + RRF merge) is
+ * expensive. When the same query is issued multiple turns in a row (common
+ * when the conversation context hasn't changed), we can serve the cached
+ * result instantly.
+ *
+ * Invalidation: a monotonic version counter is bumped whenever new memory
+ * is indexed (segments, items, embeddings). Cache entries are only valid
+ * when their version matches the current global version.
+ */
+
+interface CacheEntry {
+  version: number;
+  createdAt: number;
+  result: MemoryRecallResult;
+}
+
+const MAX_ENTRIES = 32;
+const TTL_MS = 60_000; // 60 seconds
+
+let _version = 0;
+const _cache = new Map<string, CacheEntry>();
+
+/** Bump the global memory version, invalidating all cached recall results. */
+export function bumpMemoryVersion(): void {
+  _version++;
+}
+
+/** Return the current memory version (for snapshot-based staleness checks). */
+export function getMemoryVersion(): number {
+  return _version;
+}
+
+/** Build a deterministic cache key from the recall inputs. */
+function buildCacheKey(
+  query: string,
+  conversationId: string,
+  options?: MemoryRecallOptions,
+): string {
+  const parts = [
+    query,
+    conversationId,
+    options?.scopeId ?? '',
+    options?.scopePolicyOverride
+      ? `${options.scopePolicyOverride.scopeId}:${options.scopePolicyOverride.fallbackToDefault}`
+      : '',
+    options?.excludeMessageIds ? [...options.excludeMessageIds].sort().join(',') : '',
+    options?.maxInjectTokensOverride != null ? String(options.maxInjectTokensOverride) : '',
+  ];
+  return createHash('sha256').update(parts.join('\0')).digest('hex');
+}
+
+/** Look up a cached recall result. Returns undefined on miss or stale entry. */
+export function getCachedRecall(
+  query: string,
+  conversationId: string,
+  options?: MemoryRecallOptions,
+): MemoryRecallResult | undefined {
+  const key = buildCacheKey(query, conversationId, options);
+  const entry = _cache.get(key);
+  if (!entry) return undefined;
+  if (entry.version !== _version || Date.now() - entry.createdAt > TTL_MS) {
+    _cache.delete(key);
+    return undefined;
+  }
+  return entry.result;
+}
+
+/**
+ * Store a recall result in the cache. Evicts oldest entries when full.
+ *
+ * When `snapshotVersion` is provided, the entry is only stored if the
+ * snapshot still matches the current global version — this prevents a
+ * stale result from being cached under a version that was bumped while
+ * the retrieval pipeline was in flight.
+ */
+export function setCachedRecall(
+  query: string,
+  conversationId: string,
+  options: MemoryRecallOptions | undefined,
+  result: MemoryRecallResult,
+  snapshotVersion?: number,
+): void {
+  // If a snapshot version was provided, only cache when it still matches
+  // the current version — otherwise the result may be stale.
+  if (snapshotVersion !== undefined && snapshotVersion !== _version) return;
+
+  const key = buildCacheKey(query, conversationId, options);
+
+  // Evict oldest entries if at capacity
+  if (_cache.size >= MAX_ENTRIES && !_cache.has(key)) {
+    const oldest = _cache.keys().next().value;
+    if (oldest !== undefined) _cache.delete(oldest);
+  }
+
+  _cache.set(key, { version: _version, createdAt: Date.now(), result });
+}
+
+/** Clear the entire cache (useful for testing). */
+export function clearRecallCache(): void {
+  _cache.clear();
+}

package/src/memory/retriever.ts

@@ -19,6 +19,7 @@ import { semanticSearch, isQdrantConnectionError } from './search/semantic.js';
 import { entitySearch } from './search/entity.js';
 import { mergeCandidates, applySourceCaps, rerankWithLLM, trimToTokenBudget, markItemUsage } from './search/ranking.js';
 import { buildInjectedText, MEMORY_CONTEXT_ACK } from './search/formatting.js';
+import { getCachedRecall, setCachedRecall, getMemoryVersion } from './recall-cache.js';
 
 // Re-export public types and functions so existing importers continue to work
 export type {
@@ -161,10 +162,12 @@ async function collectAndMergeCandidates(
 
   // -- Phase 2: expensive searches (skipped on early termination) --
   let semantic: Candidate[] = [];
+  let semanticSearchFailed = false;
   if (queryVector && !canTerminateEarly) {
     try {
       semantic = await semanticSearch(queryVector, opts?.provider ?? 'unknown', opts?.model ?? 'unknown', config.memory.retrieval.semanticTopK, excludeMessageIds, scopeIds);
     } catch (err) {
+      semanticSearchFailed = true;
       if (isQdrantConnectionError(err)) {
         log.warn({ err }, 'Qdrant is unavailable — semantic search disabled, memory recall will be degraded');
       } else {
@@ -174,6 +177,7 @@ async function collectAndMergeCandidates(
   }
 
   let entity: Candidate[] = [];
+  let candidateDepths: Map<string, number> | undefined;
   let relationSeedEntityCount = 0;
   let relationTraversedEdgeCount = 0;
   let relationNeighborEntityCount = 0;
@@ -186,6 +190,7 @@ async function collectAndMergeCandidates(
       excludeMessageIds,
     );
     entity = entitySearchResult.candidates;
+    candidateDepths = entitySearchResult.candidateDepths;
     relationSeedEntityCount = entitySearchResult.relationSeedEntityCount;
     relationTraversedEdgeCount = entitySearchResult.relationTraversedEdgeCount;
     relationNeighborEntityCount = entitySearchResult.relationNeighborEntityCount;
@@ -202,7 +207,10 @@ async function collectAndMergeCandidates(
   const relationScoreMultiplier = config.memory.entity.enabled && config.memory.entity.relationRetrieval.enabled
     ? config.memory.entity.relationRetrieval.neighborScoreMultiplier
     : undefined;
-  const merged = mergeCandidates(lexical, semantic, recency, [...entity, ...directItems], config.memory.retrieval.freshness, relationScoreMultiplier);
+  const depthMap = config.memory.entity.enabled && config.memory.entity.relationRetrieval.depthDecay
+    ? candidateDepths
+    : undefined;
+  const merged = mergeCandidates(lexical, semantic, recency, [...entity, ...directItems], config.memory.retrieval.freshness, relationScoreMultiplier, depthMap);
 
   return {
     lexical,
@@ -214,6 +222,7 @@ async function collectAndMergeCandidates(
     relationNeighborEntityCount,
     relationExpandedItemCount,
     earlyTerminated: canTerminateEarly,
+    semanticSearchFailed,
     merged,
   };
 }
@@ -225,6 +234,7 @@ export async function buildMemoryRecall(
   options?: MemoryRecallOptions,
 ): Promise<MemoryRecallResult> {
   const start = Date.now();
+  const versionSnapshot = getMemoryVersion();
   const excludeMessageIds = options?.excludeMessageIds?.filter((id) => id.length > 0) ?? [];
   const signal = options?.signal;
   if (!config.memory.enabled) {
@@ -234,6 +244,14 @@ export async function buildMemoryRecall(
     return emptyResult({ enabled: true, degraded: false, reason: 'memory.aborted', latencyMs: Date.now() - start });
   }
 
+  // Check recall cache — serves identical results instantly when the query
+  // and memory state haven't changed since the last recall.
+  const cached = getCachedRecall(query, conversationId, options);
+  if (cached) {
+    log.debug({ query: truncate(query, 120), latencyMs: Date.now() - start }, 'Memory recall served from cache');
+    return { ...cached, latencyMs: Date.now() - start };
+  }
+
   const backendStatus = getMemoryBackendStatus(config);
   let queryVector: number[] | null = null;
   let provider: string | undefined;
@@ -326,7 +344,15 @@ export async function buildMemoryRecall(
     relationNeighborEntityCount,
     relationExpandedItemCount,
     earlyTerminated,
+    semanticSearchFailed,
   } = collected;
+
+  // Mark as degraded when semantic search failed — the recall is based on
+  // lexical/recency only and should not be cached.
+  if (semanticSearchFailed) {
+    degraded = true;
+    reason = reason ?? 'memory.semantic_search_failure';
+  }
   let merged = applySourceCaps(collected.merged, config);
 
   // LLM re-ranking: send top candidates to Haiku for relevance scoring
@@ -395,7 +421,7 @@ export async function buildMemoryRecall(
     latencyMs,
   }, 'Memory recall completed');
 
-  return {
+  const result: MemoryRecallResult = {
     enabled: true,
     degraded,
     reason,
@@ -418,6 +444,14 @@ export async function buildMemoryRecall(
     latencyMs,
     topCandidates,
   };
+
+  // Only cache non-degraded results — degraded results (e.g. lexical-only
+  // fallback when embeddings fail) would delay quality recovery once the
+  // embedding backend comes back.
+  if (!result.degraded) {
+    setCachedRecall(query, conversationId, options, result, versionSnapshot);
+  }
+  return result;
 }
 
 export function stripMemoryRecallMessages<T extends { role: 'user' | 'assistant'; content: Array<{ type: string; text?: string }> }>(