@vellumai/assistant 0.3.2 → 0.3.4

package/src/memory/embedding-backend.ts

@@ -1,3 +1,4 @@
+import { createHash } from 'node:crypto';
 import type { AssistantConfig } from '../config/types.js';
 import { getLogger } from '../util/logger.js';
 import { GeminiEmbeddingBackend } from './embedding-gemini.js';
@@ -10,9 +11,41 @@ const log = getLogger('memory-embeddings');
 /** Global cache of embedding backend instances, keyed by "provider:model". */
 const backendCache = new Map<string, EmbeddingBackend>();
 
-/** Clear cached embedding backends so new instances pick up fresh credentials. */
+// ── In-memory embedding vector cache ──────────────────────────────
+// LRU cache keyed by sha256(provider + model + text) → embedding vector.
+// Avoids redundant API calls / local compute for identical content.
+const VECTOR_CACHE_MAX_ENTRIES = 4096;
+const vectorCache = new Map<string, number[]>();
+
+function vectorCacheKey(provider: string, model: string, text: string): string {
+  return createHash('sha256').update(`${provider}\0${model}\0${text}`).digest('hex');
+}
+
+function getFromVectorCache(provider: string, model: string, text: string): number[] | undefined {
+  const key = vectorCacheKey(provider, model, text);
+  const v = vectorCache.get(key);
+  if (v !== undefined) {
+    // LRU refresh: move to end of insertion order
+    vectorCache.delete(key);
+    vectorCache.set(key, v);
+  }
+  return v;
+}
+
+function putInVectorCache(provider: string, model: string, text: string, vector: number[]): void {
+  const key = vectorCacheKey(provider, model, text);
+  vectorCache.delete(key);
+  if (vectorCache.size >= VECTOR_CACHE_MAX_ENTRIES) {
+    const oldest = vectorCache.keys().next().value;
+    if (oldest !== undefined) vectorCache.delete(oldest);
+  }
+  vectorCache.set(key, vector);
+}
+
+/** Clear cached embedding backends and the in-memory vector cache. */
 export function clearEmbeddingBackendCache(): void {
   backendCache.clear();
+  vectorCache.clear();
 }
 
 function cacheKey(provider: string, model: string): string {
@@ -153,22 +186,44 @@ export async function embedWithBackend(
     throw new Error(selection.reason ?? 'No memory embedding backend configured');
   }
 
-  // In auto mode, build a fallback list of backends to try
-  const backends: EmbeddingBackend[] = [selection.backend];
-  if (config.memory.embeddings.provider === 'auto' && selection.backend.provider === 'local') {
-    for (const fallback of selectFallbackBackends(config, 'local')) {
-      backends.push(fallback);
-    }
+  const expectedDim = config.memory.qdrant.vectorSize;
+  const { provider: primaryProvider, model: primaryModel } = selection.backend;
+
+  // ── Build fallback backends list (needed for embed fallback) ──
+  const fallbacks: EmbeddingBackend[] =
+    config.memory.embeddings.provider === 'auto' && selection.backend.provider === 'local'
+      ? selectFallbackBackends(config, 'local')
+      : [];
+
+  // ── In-memory cache check (primary provider only) ──────────────
+  const cached: (number[] | null)[] = texts.map(t => {
+    const v = getFromVectorCache(primaryProvider, primaryModel, t);
+    if (v && v.length === expectedDim) return v;
+    return null;
+  });
+  const uncachedIndices: number[] = [];
+  for (let i = 0; i < cached.length; i++) {
+    if (!cached[i]) uncachedIndices.push(i);
+  }
+  if (uncachedIndices.length === 0) {
+    return { provider: primaryProvider, model: primaryModel, vectors: cached as number[][] };
   }
 
+  // ── Embed uncached texts ────────────────────────────────────────
+  const backends: EmbeddingBackend[] = [selection.backend, ...fallbacks];
+
   let lastErr: unknown;
   for (const backend of backends) {
+    const isPrimary = backend === selection.backend;
+    // For the primary backend, only embed uncached texts and merge with cached.
+    // For fallback backends, embed ALL texts since the cache was keyed to the primary.
+    const textsToEmbed = isPrimary ? uncachedIndices.map(i => texts[i]) : texts;
+
     try {
-      const vectors = await backend.embed(texts, options);
-      if (vectors.length !== texts.length) {
-        throw new Error(`Embedding backend returned ${vectors.length} vectors for ${texts.length} texts`);
+      const vectors = await backend.embed(textsToEmbed, options);
+      if (vectors.length !== textsToEmbed.length) {
+        throw new Error(`Embedding backend returned ${vectors.length} vectors for ${textsToEmbed.length} texts`);
       }
-      const expectedDim = config.memory.qdrant.vectorSize;
       for (const vec of vectors) {
         if (vec.length !== expectedDim) {
           throw new Error(
@@ -176,6 +231,19 @@ export async function embedWithBackend(
           );
         }
       }
+
+      // Populate cache with freshly embedded vectors
+      for (let i = 0; i < textsToEmbed.length; i++) {
+        putInVectorCache(backend.provider, backend.model, textsToEmbed[i], vectors[i]);
+      }
+
+      if (isPrimary) {
+        const merged = [...cached] as number[][];
+        for (let i = 0; i < uncachedIndices.length; i++) {
+          merged[uncachedIndices[i]] = vectors[i];
+        }
+        return { provider: backend.provider, model: backend.model, vectors: merged };
+      }
       return { provider: backend.provider, model: backend.model, vectors };
     } catch (err) {
       lastErr = err;

package/src/memory/indexer.ts

@@ -7,6 +7,7 @@ import { getDb } from './db.js';
 import { enqueueMemoryJob, enqueueResolvePendingConflictsForMessageJob } from './jobs-store.js';
 import { extractTextFromStoredMessageContent } from './message-content.js';
 import { segmentText } from './segmenter.js';
+import { bumpMemoryVersion } from './recall-cache.js';
 import { memorySegments } from './schema.js';
 
 const log = getLogger('memory-indexer');
@@ -108,6 +109,7 @@ export function indexMessageNow(
     log.debug(`Skipped ${skippedEmbedJobs}/${segments.length} embed_segment jobs (content unchanged)`);
   }
 
+  bumpMemoryVersion();
   enqueueSummaryRollupJobsIfDue();
 
   const enqueuedJobs = (segments.length - skippedEmbedJobs) + (shouldExtract ? 2 : 1) + (shouldResolveConflicts ? 1 : 0);

package/src/memory/job-utils.ts

@@ -1,6 +1,10 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { eq, and } from 'drizzle-orm';
 import { getLogger } from '../util/logger.js';
 import { embedWithBackend, getMemoryBackendStatus } from './embedding-backend.js';
+import { getDb } from './db.js';
 import { getQdrantClient } from './qdrant-client.js';
+import { memoryEmbeddings } from './schema.js';
 import type { AssistantConfig } from '../config/types.js';
 
 const log = getLogger('memory-jobs-worker');
@@ -111,9 +115,66 @@ export async function embedAndUpsert(
     );
   }
 
-  const embedded = await embedWithBackend(config, [text]);
-  const vector = embedded.vectors[0];
-  if (!vector) return;
+  const contentHash = createHash('sha256').update(text).digest('hex');
+  let provider = status.provider;
+  let model = status.model!;
+  let vector: number[];
+
+  // Check SQLite embedding cache for a matching content hash (primary provider only).
+  const db = getDb();
+  const expectedDim = config.memory.qdrant.vectorSize;
+  let cachedRow = db
+    .select({ vectorJson: memoryEmbeddings.vectorJson, dimensions: memoryEmbeddings.dimensions })
+    .from(memoryEmbeddings)
+    .where(
+      and(
+        eq(memoryEmbeddings.contentHash, contentHash),
+        eq(memoryEmbeddings.provider, provider),
+        eq(memoryEmbeddings.model, model),
+      ),
+    )
+    .get();
+  if (cachedRow && cachedRow.dimensions !== expectedDim) cachedRow = undefined;
+
+  if (cachedRow) {
+    vector = JSON.parse(cachedRow.vectorJson);
+  } else {
+    const embedded = await embedWithBackend(config, [text]);
+    vector = embedded.vectors[0];
+    if (!vector) return;
+    provider = embedded.provider;
+    model = embedded.model;
+  }
+
+  // Persist embedding in SQLite for cross-restart cache
+  const now = Date.now();
+  try {
+    db.insert(memoryEmbeddings)
+      .values({
+        id: randomUUID(),
+        targetType,
+        targetId,
+        provider,
+        model,
+        dimensions: vector.length,
+        vectorJson: JSON.stringify(vector),
+        contentHash,
+        createdAt: now,
+        updatedAt: now,
+      })
+      .onConflictDoUpdate({
+        target: [memoryEmbeddings.targetType, memoryEmbeddings.targetId, memoryEmbeddings.provider, memoryEmbeddings.model],
+        set: {
+          vectorJson: JSON.stringify(vector),
+          dimensions: vector.length,
+          contentHash,
+          updatedAt: now,
+        },
+      })
+      .run();
+  } catch (err) {
+    log.warn({ err, targetType, targetId }, 'Failed to write embedding cache');
+  }
 
   let qdrant;
   try {
@@ -123,7 +184,6 @@ export async function embedAndUpsert(
   }
 
   try {
-    const now = Date.now();
     await qdrant.upsert(targetType, targetId, vector, {
       text,
       created_at: (extraPayload?.created_at as number) ?? now,

package/src/memory/jobs-worker.ts

@@ -18,6 +18,7 @@ import {
   retryDelayForAttempt,
   RETRY_MAX_ATTEMPTS,
 } from './job-utils.js';
+import { bumpMemoryVersion } from './recall-cache.js';
 
 // ── Per-job-type handlers ──────────────────────────────────────────
 
@@ -121,9 +122,14 @@ export async function runMemoryJobsOnce(
           try {
             await processJob(job, config);
             completeMemoryJob(job.id);
+            bumpMemoryVersion();
             groupProcessed += 1;
           } catch (err) {
-            handleJobError(job, err);
+            try {
+              handleJobError(job, err);
+            } catch (handlerErr) {
+              log.error({ err: handlerErr, jobId: job.id, type: job.type }, 'handleJobError itself threw, job left in running status');
+            }
           }
         }
         return groupProcessed;

package/src/memory/recall-cache.ts

@@ -0,0 +1,107 @@
+import { createHash } from 'crypto';
+import type { MemoryRecallResult, MemoryRecallOptions } from './search/types.js';
+
+/**
+ * In-memory cache for memory recall results.
+ *
+ * The full retrieval pipeline (FTS5 + Qdrant + entity graph + RRF merge) is
+ * expensive. When the same query is issued multiple turns in a row (common
+ * when the conversation context hasn't changed), we can serve the cached
+ * result instantly.
+ *
+ * Invalidation: a monotonic version counter is bumped whenever new memory
+ * is indexed (segments, items, embeddings). Cache entries are only valid
+ * when their version matches the current global version.
+ */
+
+interface CacheEntry {
+  version: number;
+  createdAt: number;
+  result: MemoryRecallResult;
+}
+
+const MAX_ENTRIES = 32;
+const TTL_MS = 60_000; // 60 seconds
+
+let _version = 0;
+const _cache = new Map<string, CacheEntry>();
+
+/** Bump the global memory version, invalidating all cached recall results. */
+export function bumpMemoryVersion(): void {
+  _version++;
+}
+
+/** Return the current memory version (for snapshot-based staleness checks). */
+export function getMemoryVersion(): number {
+  return _version;
+}
+
+/** Build a deterministic cache key from the recall inputs. */
+function buildCacheKey(
+  query: string,
+  conversationId: string,
+  options?: MemoryRecallOptions,
+): string {
+  const parts = [
+    query,
+    conversationId,
+    options?.scopeId ?? '',
+    options?.scopePolicyOverride
+      ? `${options.scopePolicyOverride.scopeId}:${options.scopePolicyOverride.fallbackToDefault}`
+      : '',
+    options?.excludeMessageIds ? [...options.excludeMessageIds].sort().join(',') : '',
+    options?.maxInjectTokensOverride != null ? String(options.maxInjectTokensOverride) : '',
+  ];
+  return createHash('sha256').update(parts.join('\0')).digest('hex');
+}
+
+/** Look up a cached recall result. Returns undefined on miss or stale entry. */
+export function getCachedRecall(
+  query: string,
+  conversationId: string,
+  options?: MemoryRecallOptions,
+): MemoryRecallResult | undefined {
+  const key = buildCacheKey(query, conversationId, options);
+  const entry = _cache.get(key);
+  if (!entry) return undefined;
+  if (entry.version !== _version || Date.now() - entry.createdAt > TTL_MS) {
+    _cache.delete(key);
+    return undefined;
+  }
+  return entry.result;
+}
+
+/**
+ * Store a recall result in the cache. Evicts oldest entries when full.
+ *
+ * When `snapshotVersion` is provided, the entry is only stored if the
+ * snapshot still matches the current global version — this prevents a
+ * stale result from being cached under a version that was bumped while
+ * the retrieval pipeline was in flight.
+ */
+export function setCachedRecall(
+  query: string,
+  conversationId: string,
+  options: MemoryRecallOptions | undefined,
+  result: MemoryRecallResult,
+  snapshotVersion?: number,
+): void {
+  // If a snapshot version was provided, only cache when it still matches
+  // the current version — otherwise the result may be stale.
+  if (snapshotVersion !== undefined && snapshotVersion !== _version) return;
+
+  const key = buildCacheKey(query, conversationId, options);
+
+  // Evict oldest entries if at capacity
+  if (_cache.size >= MAX_ENTRIES && !_cache.has(key)) {
+    const oldest = _cache.keys().next().value;
+    if (oldest !== undefined) _cache.delete(oldest);
+  }
+
+  _cache.set(key, { version: _version, createdAt: Date.now(), result });
+}
+
+/** Clear the entire cache (useful for testing). */
+export function clearRecallCache(): void {
+  _cache.clear();
+}

package/src/memory/retriever.ts

@@ -19,6 +19,7 @@ import { semanticSearch, isQdrantConnectionError } from './search/semantic.js';
 import { entitySearch } from './search/entity.js';
 import { mergeCandidates, applySourceCaps, rerankWithLLM, trimToTokenBudget, markItemUsage } from './search/ranking.js';
 import { buildInjectedText, MEMORY_CONTEXT_ACK } from './search/formatting.js';
+import { getCachedRecall, setCachedRecall, getMemoryVersion } from './recall-cache.js';
 
 // Re-export public types and functions so existing importers continue to work
 export type {
@@ -161,10 +162,12 @@ async function collectAndMergeCandidates(
 
   // -- Phase 2: expensive searches (skipped on early termination) --
   let semantic: Candidate[] = [];
+  let semanticSearchFailed = false;
   if (queryVector && !canTerminateEarly) {
     try {
       semantic = await semanticSearch(queryVector, opts?.provider ?? 'unknown', opts?.model ?? 'unknown', config.memory.retrieval.semanticTopK, excludeMessageIds, scopeIds);
     } catch (err) {
+      semanticSearchFailed = true;
       if (isQdrantConnectionError(err)) {
         log.warn({ err }, 'Qdrant is unavailable — semantic search disabled, memory recall will be degraded');
       } else {
@@ -214,6 +217,7 @@ async function collectAndMergeCandidates(
     relationNeighborEntityCount,
     relationExpandedItemCount,
     earlyTerminated: canTerminateEarly,
+    semanticSearchFailed,
     merged,
   };
 }
@@ -225,6 +229,7 @@ export async function buildMemoryRecall(
   options?: MemoryRecallOptions,
 ): Promise<MemoryRecallResult> {
   const start = Date.now();
+  const versionSnapshot = getMemoryVersion();
   const excludeMessageIds = options?.excludeMessageIds?.filter((id) => id.length > 0) ?? [];
   const signal = options?.signal;
   if (!config.memory.enabled) {
@@ -234,6 +239,14 @@ export async function buildMemoryRecall(
     return emptyResult({ enabled: true, degraded: false, reason: 'memory.aborted', latencyMs: Date.now() - start });
   }
 
+  // Check recall cache — serves identical results instantly when the query
+  // and memory state haven't changed since the last recall.
+  const cached = getCachedRecall(query, conversationId, options);
+  if (cached) {
+    log.debug({ query: truncate(query, 120), latencyMs: Date.now() - start }, 'Memory recall served from cache');
+    return { ...cached, latencyMs: Date.now() - start };
+  }
+
   const backendStatus = getMemoryBackendStatus(config);
   let queryVector: number[] | null = null;
   let provider: string | undefined;
@@ -326,7 +339,15 @@ export async function buildMemoryRecall(
     relationNeighborEntityCount,
     relationExpandedItemCount,
     earlyTerminated,
+    semanticSearchFailed,
   } = collected;
+
+  // Mark as degraded when semantic search failed — the recall is based on
+  // lexical/recency only and should not be cached.
+  if (semanticSearchFailed) {
+    degraded = true;
+    reason = reason ?? 'memory.semantic_search_failure';
+  }
   let merged = applySourceCaps(collected.merged, config);
 
   // LLM re-ranking: send top candidates to Haiku for relevance scoring
@@ -395,7 +416,7 @@ export async function buildMemoryRecall(
     latencyMs,
   }, 'Memory recall completed');
 
-  return {
+  const result: MemoryRecallResult = {
     enabled: true,
     degraded,
     reason,
@@ -418,6 +439,14 @@ export async function buildMemoryRecall(
     latencyMs,
     topCandidates,
   };
+
+  // Only cache non-degraded results — degraded results (e.g. lexical-only
+  // fallback when embeddings fail) would delay quality recovery once the
+  // embedding backend comes back.
+  if (!result.degraded) {
+    setCachedRecall(query, conversationId, options, result, versionSnapshot);
+  }
+  return result;
 }
 
 export function stripMemoryRecallMessages<T extends { role: 'user' | 'assistant'; content: Array<{ type: string; text?: string }> }>(