memorix 1.8.8 → 1.9.0

package/src/runtime/vector-backfill-runner.ts

@@ -1,5 +1,14 @@
 import { spawn, type ChildProcess } from 'node:child_process';
-import { existsSync } from 'node:fs';
+import {
+  closeSync,
+  existsSync,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  statSync,
+  unlinkSync,
+  writeFileSync,
+} from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 
@@ -11,8 +20,12 @@ import { initObservationStore } from '../store/obs-store.js';
 import { getDeferredCachedVectorHydration } from '../store/orama-store.js';
 import { closeAllDatabases } from '../store/sqlite-db.js';
 import { MaintenanceJobStore, MaintenanceJobWorker } from './maintenance-jobs.js';
+import { MaintenanceTargetStore, type MaintenanceTarget } from './maintenance-targets.js';
 import { createProjectMaintenanceHandler } from './project-maintenance.js';
 
+const VECTOR_BACKFILL_LOCK_FILE = '.memorix-vector-backfill.lock';
+const STALE_LOCK_RECOVERY_MS = 24 * 60 * 60 * 1_000;
+
 export interface VectorBackfillRequest {
   projectId: string;
   projectRoot: string;
@@ -23,6 +36,20 @@ export interface VectorBackfillLauncherOptions {
   runnerPath?: string;
   exists?: (path: string) => boolean;
   spawn?: typeof spawn;
+  lockPath?: string;
+}
+
+interface VectorBackfillLock {
+  parentPid: number;
+  childPid?: number;
+  startedAt: number;
+  projectId: string;
+}
+
+interface VectorBackfillLockHandle {
+  path: string;
+  setChildPid(pid: number): void;
+  release(): void;
 }
 
 function isNonEmptyString(value: unknown): value is string {
@@ -68,6 +95,111 @@ export function parseVectorBackfillRequest(raw: string): VectorBackfillRequest {
   };
 }
 
+export function vectorBackfillLockPath(dataDir: string): string {
+  return path.join(dataDir, VECTOR_BACKFILL_LOCK_FILE);
+}
+
+function processIsAlive(pid: number | undefined): boolean {
+  if (!Number.isSafeInteger(pid) || pid == null || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    // EPERM means the process exists but this process cannot signal it.
+    return (error as NodeJS.ErrnoException).code === 'EPERM';
+  }
+}
+
+function readVectorBackfillLock(lockPath: string): VectorBackfillLock | undefined {
+  try {
+    const value = JSON.parse(readFileSync(lockPath, 'utf8')) as Partial<VectorBackfillLock>;
+    if (
+      typeof value.parentPid !== 'number'
+      || !Number.isSafeInteger(value.parentPid)
+      || typeof value.startedAt !== 'number'
+      || !Number.isFinite(value.startedAt)
+      || typeof value.projectId !== 'string'
+    ) return undefined;
+    return {
+      parentPid: value.parentPid,
+      ...(typeof value.childPid === 'number' ? { childPid: value.childPid } : {}),
+      startedAt: value.startedAt,
+      projectId: value.projectId,
+    };
+  } catch {
+    return undefined;
+  }
+}
+
+function writeVectorBackfillLock(lockPath: string, value: VectorBackfillLock): void {
+  writeFileSync(lockPath, `${JSON.stringify(value)}\n`, 'utf8');
+}
+
+function acquireVectorBackfillLock(
+  dataDir: string,
+  projectId: string,
+  preferredPath?: string,
+): VectorBackfillLockHandle | undefined {
+  const lockPath = preferredPath ?? vectorBackfillLockPath(dataDir);
+  mkdirSync(path.dirname(lockPath), { recursive: true });
+  const initial: VectorBackfillLock = {
+    parentPid: process.pid,
+    startedAt: Date.now(),
+    projectId,
+  };
+
+  for (let attempt = 0; attempt < 2; attempt++) {
+    try {
+      const fd = openSync(lockPath, 'wx');
+      try {
+        writeFileSync(fd, `${JSON.stringify(initial)}\n`, 'utf8');
+      } finally {
+        closeSync(fd);
+      }
+      return {
+        path: lockPath,
+        setChildPid(pid: number) {
+          try {
+            writeVectorBackfillLock(lockPath, { ...initial, childPid: pid });
+          } catch {
+            // The child can still release the lock on normal shutdown; a dead
+            // owner is recoverable on the next launch.
+          }
+        },
+        release() {
+          try { unlinkSync(lockPath); } catch { /* already released */ }
+        },
+      };
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== 'EEXIST') return undefined;
+      const existing = readVectorBackfillLock(lockPath);
+      const ownerAlive = existing && (processIsAlive(existing.childPid) || processIsAlive(existing.parentPid));
+      if (ownerAlive) return undefined;
+
+      // A malformed lock is not deleted immediately: another process may be
+      // between create and write. A dead, well-formed owner is safe to reap;
+      // an unreadable lock needs a long recovery window so it cannot cause a
+      // second heavy worker during a slow filesystem operation.
+      let stale = false;
+      try {
+        stale = Date.now() - statSync(lockPath).mtimeMs >= STALE_LOCK_RECOVERY_MS;
+      } catch {
+        stale = false;
+      }
+      if (!existing && !stale) return undefined;
+      try { unlinkSync(lockPath); } catch { return undefined; }
+    }
+  }
+  return undefined;
+}
+
+export function releaseVectorBackfillLock(lockPath = process.env.MEMORIX_VECTOR_BACKFILL_LOCK): void {
+  if (!lockPath) return;
+  const lock = readVectorBackfillLock(lockPath);
+  if (lock && lock.childPid !== process.pid) return;
+  try { unlinkSync(lockPath); } catch { /* already released */ }
+}
+
 /**
  * Start a detached one-shot worker. The caller has already persisted the
  * observation and durable vector job, so failure to start is recoverable by a
@@ -81,6 +213,9 @@ export function launchDetachedVectorBackfill(
   const exists = options.exists ?? existsSync;
   if (!exists(runnerPath)) return false;
 
+  const lock = acquireVectorBackfillLock(request.dataDir, request.projectId, options.lockPath);
+  if (!lock) return false;
+
   try {
     const child = (options.spawn ?? spawn)(process.execPath, [runnerPath], {
       cwd: request.projectRoot,
@@ -90,13 +225,24 @@ export function launchDetachedVectorBackfill(
       stdio: 'ignore',
       // The request contains only local project metadata, never credentials.
       // Environment transport avoids a live stdin pipe keeping the CLI alive.
-      env: { ...process.env, MEMORIX_VECTOR_BACKFILL_REQUEST: JSON.stringify(request) },
+      env: {
+        ...process.env,
+        MEMORIX_VECTOR_BACKFILL_REQUEST: JSON.stringify(request),
+        MEMORIX_VECTOR_BACKFILL_LOCK: lock.path,
+      },
       windowsHide: true,
     }) as ChildProcess;
+    if (child.pid) lock.setChildPid(child.pid);
+    else lock.release();
     child.once?.('error', () => {});
+    child.once?.('close', () => lock.release());
+    // Test doubles and unusual spawn adapters may not expose ChildProcess
+    // events. Do not leave a lock behind when there is no lifecycle to own it.
+    if (typeof child.once !== 'function') lock.release();
     child.unref();
-    return true;
+    return Boolean(child.pid);
   } catch {
+    lock.release();
     return false;
   }
 }
@@ -110,12 +256,43 @@ export async function executeVectorBackfill(request: VectorBackfillRequest) {
   await prepareSearchIndex();
   await getDeferredCachedVectorHydration()?.catch(() => {});
 
+  const queue = new MaintenanceJobStore(request.dataDir);
+  const targets = new MaintenanceTargetStore(request.dataDir);
+  let preparedRoot = request.projectRoot;
+  const prepareTarget = async (target: MaintenanceTarget): Promise<void> => {
+    if (target.projectRoot === preparedRoot) return;
+    initProjectRoot(target.projectRoot);
+    loadDotenv(target.projectRoot);
+    preparedRoot = target.projectRoot;
+    // The data directory is shared across projects. The in-process index is
+    // intentionally global, so switching the target only changes the config
+    // and the job scope; it does not create another corpus/index copy.
+  };
+
+  const handler = async (job: Parameters<ReturnType<typeof createProjectMaintenanceHandler>>[0]) => {
+    const target = targets.get(job.projectId)
+      ?? (job.projectId === request.projectId
+        ? { projectId: request.projectId, projectRoot: request.projectRoot, dataDir: request.dataDir, updatedAt: Date.now() }
+        : undefined);
+    if (!target) return { action: 'reschedule' as const, delayMs: 30_000 };
+    await prepareTarget(target);
+    return createProjectMaintenanceHandler(job.projectId, target.dataDir, target.projectRoot)(job);
+  };
+
   const worker = new MaintenanceJobWorker(
-    new MaintenanceJobStore(request.dataDir),
-    createProjectMaintenanceHandler(request.projectId, request.dataDir, request.projectRoot),
-    { projectId: request.projectId, kinds: ['vector-backfill'] },
+    queue,
+    handler,
+    { kinds: ['vector-backfill'] },
   );
-  return worker.runOnce();
+  // Drain currently due vector jobs while this one process owns the lock. A
+  // hook storm therefore creates at most one heavy process, and pending jobs
+  // for other projects are picked up without requiring another hook event.
+  let result: Awaited<ReturnType<typeof worker.runOnce>> = { state: 'idle' };
+  while (true) {
+    result = await worker.runOnce();
+    if (result.state === 'idle') break;
+  }
+  return result;
 }
 
 async function readStdin(): Promise<string> {
@@ -137,6 +314,7 @@ export async function main(): Promise<void> {
     process.stderr.write(`[memorix] vector backfill worker failed: ${detail}\n`);
     process.exitCode = 1;
   } finally {
+    releaseVectorBackfillLock();
     closeAllDatabases();
   }
 }

package/src/store/orama-store.ts

@@ -33,10 +33,19 @@ let embeddingEnabled = false;
 let embeddingDimensions: number | null = null;
 let indexEmbeddingProvider: EmbeddingProvider | null = null;
 let deferredCachedVectorHydration: Promise<void> | null = null;
-const docByObservationKey = new Map<string, MemorixDocument>();
 const NON_CJK_HYBRID_SIMILARITY = 0.45;
 const lastSearchModeByProject = new Map<string, string>();
 const SEARCH_MODE_DEFAULT_KEY = '__global__';
+const MAX_SEARCH_MODE_KEYS = 4096;
+
+function rememberSearchMode(key: string, value: string): void {
+  if (!lastSearchModeByProject.has(key) && lastSearchModeByProject.size >= MAX_SEARCH_MODE_KEYS) {
+    const oldest = lastSearchModeByProject.keys().next().value;
+    if (oldest !== undefined) lastSearchModeByProject.delete(oldest);
+  }
+  lastSearchModeByProject.delete(key);
+  lastSearchModeByProject.set(key, value);
+}
 export function getLastSearchMode(projectId?: string): string {
   return lastSearchModeByProject.get(projectId ?? SEARCH_MODE_DEFAULT_KEY) ?? 'fulltext';
 }
@@ -68,9 +77,6 @@ function makeEntryKey(projectId: string | undefined, observationId: number): str
 function rememberObservationDoc(doc: MemorixDocument): MemorixDocument {
   const publicDoc = { ...doc };
   delete publicDoc.embedding;
-  if (doc.projectId && typeof doc.observationId === 'number') {
-    docByObservationKey.set(makeEntryKey(doc.projectId, doc.observationId), publicDoc);
-  }
   return publicDoc;
 }
 
@@ -244,7 +250,6 @@ export async function resetDb(): Promise<void> {
   indexEmbeddingProvider = null;
   deferredCachedVectorHydration = null;
   lastSearchModeByProject.clear();
-  docByObservationKey.clear();
 }
 
 /**
@@ -336,8 +341,6 @@ export interface HydrateIndexOptions {
   skipCachedVectors?: boolean;
 }
 
-type HydrationCandidate = { observation: any; id: string };
-
 function observationEmbeddingText(observation: any): string {
   return [
     observation.title ?? '',
@@ -361,29 +364,31 @@ function isCompatibleCachedVector(vector: number[] | null): vector is number[] {
 
 async function attachCachedVectors(
   database: AnyOrama,
-  candidates: HydrationCandidate[],
+  observations: any[],
 ): Promise<void> {
-  const cachedVectors = await getCachedEmbeddings(
-    candidates.map(({ observation }) => observationEmbeddingText(observation)),
-  );
+  const batchSize = 200;
+  for (let start = 0; start < observations.length; start += batchSize) {
+    // Keep cache lookup and the temporary vector array bounded to one batch.
+    const batch = observations.slice(start, start + batchSize);
+    const cachedVectors = await getCachedEmbeddings(batch.map(observationEmbeddingText));
 
-  // A reset or project switch created a new index while the cache was loading.
-  // Do not attach stale vectors to that new index.
-  if (db !== database) return;
+    // A reset or project switch created a new index while the cache was loading.
+    // Do not attach stale vectors to that new index.
+    if (db !== database) return;
 
-  for (let index = 0; index < candidates.length; index++) {
-    const vector = cachedVectors[index];
-    if (!isCompatibleCachedVector(vector)) continue;
-    const existing = getByID(database, candidates[index].id) as MemorixDocument | undefined;
-    if (!existing || documentEmbeddingText(existing) !== observationEmbeddingText(candidates[index].observation)) continue;
-    try {
-      await update(database, candidates[index].id, { ...existing, embedding: vector });
-    } catch {
-      // Vector cache hydration is best-effort. The normal backfill lane owns misses.
-    }
-    if (index % 50 === 0) {
-      await new Promise<void>((resolve) => setImmediate(resolve));
+    for (let index = 0; index < batch.length; index++) {
+      const vector = cachedVectors[index];
+      if (!isCompatibleCachedVector(vector)) continue;
+      const id = makeOramaObservationId(batch[index].projectId, batch[index].id);
+      const existing = getByID(database, id) as MemorixDocument | undefined;
+      if (!existing || documentEmbeddingText(existing) !== observationEmbeddingText(batch[index])) continue;
+      try {
+        await update(database, id, { ...existing, embedding: vector });
+      } catch {
+        // Vector cache hydration is best-effort. The normal backfill lane owns misses.
+      }
     }
+    await new Promise<void>((resolve) => setImmediate(resolve));
   }
 }
 
@@ -392,62 +397,54 @@ export async function hydrateIndex(
   options: HydrateIndexOptions = {},
 ): Promise<number> {
   const database = await getDb({ allowNetworkProbe: options.allowNetworkProbe });
-
-  const candidates: HydrationCandidate[] = [];
-  for (const observation of observations) {
-    if (!observation || !observation.id || !observation.projectId) continue;
-    const id = makeOramaObservationId(observation.projectId, observation.id);
-    if (getByID(database, id)) continue;
-    candidates.push({ observation, id });
-  }
-
   const deferCachedVectors = options.deferCachedVectors === true && Boolean(indexEmbeddingProvider?.getCachedEmbeddings);
-  const cachedVectors = deferCachedVectors
-    ? candidates.map(() => null)
-    : await getCachedEmbeddings(candidates.map(({ observation }) => observationEmbeddingText(observation)));
-
-  const documents: MemorixDocument[] = [];
-  for (let index = 0; index < candidates.length; index++) {
-    const { observation: obs, id } = candidates[index];
-    try {
-      const vector = cachedVectors[index];
-      const compatibleVector = isCompatibleCachedVector(vector) ? vector : null;
-      const doc: MemorixDocument = {
-        id,
-        observationId: obs.id,
-        entityName: obs.entityName || '',
-        type: obs.type || 'discovery',
-        title: obs.title || '',
-        narrative: obs.narrative || '',
-        facts: Array.isArray(obs.facts) ? obs.facts.join(' ') : '',
-        filesModified: Array.isArray(obs.filesModified) ? obs.filesModified.join(' ') : '',
-        concepts: Array.isArray(obs.concepts) ? obs.concepts.join(' ') : '',
-        tokens: obs.tokens ?? 0,
-        createdAt: obs.createdAt || '',
-        projectId: obs.projectId,
-        accessCount: obs.accessCount ?? 0,
-        lastAccessedAt: obs.lastAccessedAt || '',
-        status: obs.status ?? 'active',
-        source: obs.source || 'agent',
-        sourceDetail: obs.sourceDetail ?? '',
-        valueCategory: obs.valueCategory ?? '',
-        admissionState: obs.admissionState ?? '',
-        admissionReason: obs.admissionReason ?? '',
-        visibility: obs.visibility ?? 'project',
-        createdByAgentId: obs.createdByAgentId ?? '',
-        sharedWithAgentIds: JSON.stringify(obs.sharedWithAgentIds ?? []),
-        documentType: 'observation',
-        knowledgeLayer: resolveKnowledgeLayer('observation', obs.sourceDetail, obs.source),
-        ...(compatibleVector ? { embedding: compatibleVector } : {}),
-      };
-      documents.push(doc);
-    } catch { /* skip malformed entries */ }
-  }
-
   let inserted = 0;
   const hydrationBatchSize = 200;
-  for (let start = 0; start < documents.length; start += hydrationBatchSize) {
-    const batch = documents.slice(start, start + hydrationBatchSize);
+  for (let start = 0; start < observations.length; start += hydrationBatchSize) {
+    const batchObservations = observations.slice(start, start + hydrationBatchSize)
+      .filter((observation) => observation && observation.id && observation.projectId)
+      .filter((observation) => !getByID(database, makeOramaObservationId(observation.projectId, observation.id)));
+    if (batchObservations.length === 0) continue;
+
+    const cachedVectors = deferCachedVectors || options.skipCachedVectors
+      ? batchObservations.map(() => null)
+      : await getCachedEmbeddings(batchObservations.map(observationEmbeddingText));
+    const batch: MemorixDocument[] = [];
+    for (let index = 0; index < batchObservations.length; index++) {
+      const obs = batchObservations[index];
+      try {
+        const compatibleVector = isCompatibleCachedVector(cachedVectors[index]) ? cachedVectors[index] : null;
+        batch.push({
+          id: makeOramaObservationId(obs.projectId, obs.id),
+          observationId: obs.id,
+          entityName: obs.entityName || '',
+          type: obs.type || 'discovery',
+          title: obs.title || '',
+          narrative: obs.narrative || '',
+          facts: Array.isArray(obs.facts) ? obs.facts.join(' ') : '',
+          filesModified: Array.isArray(obs.filesModified) ? obs.filesModified.join(' ') : '',
+          concepts: Array.isArray(obs.concepts) ? obs.concepts.join(' ') : '',
+          tokens: obs.tokens ?? 0,
+          createdAt: obs.createdAt || '',
+          projectId: obs.projectId,
+          accessCount: obs.accessCount ?? 0,
+          lastAccessedAt: obs.lastAccessedAt || '',
+          status: obs.status ?? 'active',
+          source: obs.source || 'agent',
+          sourceDetail: obs.sourceDetail ?? '',
+          valueCategory: obs.valueCategory ?? '',
+          admissionState: obs.admissionState ?? '',
+          admissionReason: obs.admissionReason ?? '',
+          visibility: obs.visibility ?? 'project',
+          createdByAgentId: obs.createdByAgentId ?? '',
+          sharedWithAgentIds: JSON.stringify(obs.sharedWithAgentIds ?? []),
+          documentType: 'observation',
+          knowledgeLayer: resolveKnowledgeLayer('observation', obs.sourceDetail, obs.source),
+          ...(compatibleVector ? { embedding: compatibleVector } : {}),
+        });
+      } catch { /* skip malformed entries */ }
+    }
+    if (batch.length === 0) continue;
     try {
       // Orama can rebalance its indexes once per batch instead of once per
       // document. Keep batches bounded so HTTP health remains responsive.
@@ -476,7 +473,7 @@ export async function hydrateIndex(
   deferredCachedVectorHydration = options.skipCachedVectors
     ? null
     : deferCachedVectors
-      ? attachCachedVectors(database, candidates).catch(() => {})
+      ? attachCachedVectors(database, observations).catch(() => {})
     : null;
   return inserted;
 }
@@ -548,12 +545,6 @@ export async function updateObservationMetadata(
 export async function removeObservation(oramaId: string): Promise<void> {
   const database = await getDb();
   await remove(database, oramaId);
-  for (const [key, doc] of docByObservationKey.entries()) {
-    if (doc.id === oramaId) {
-      docByObservationKey.delete(key);
-      break;
-    }
-  }
 }
 
 /**
@@ -569,7 +560,7 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
   const modeKey = options.projectId ?? SEARCH_MODE_DEFAULT_KEY;
   const quality = options.quality ?? 'balanced';
   const database = await getDb();
-  lastSearchModeByProject.set(
+  rememberSearchMode(
     modeKey,
     quality === 'fast' ? 'fulltext (fast profile)' : embeddingEnabled ? 'hybrid' : 'fulltext',
   );
@@ -671,7 +662,7 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
       if (provider) {
         const activeVectorDimensions = getVectorDimensions();
         if (activeVectorDimensions !== null && provider.dimensions !== activeVectorDimensions) {
-          lastSearchModeByProject.set(
+          rememberSearchMode(
             modeKey,
             `fulltext (embedding dimension mismatch: provider ${provider.dimensions}d vs index ${activeVectorDimensions}d)`,
           );
@@ -700,7 +691,7 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
           // Detect CJK-heavy queries: BM25 can't tokenize Chinese/Japanese/Korean well
           const cjkRatio = (originalQuery!.match(/[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/g) || []).length / originalQuery!.length;
           const isCJKHeavy = cjkRatio > 0.3;
-          lastSearchModeByProject.set(modeKey, 'hybrid');
+          rememberSearchMode(modeKey, 'hybrid');
           searchParams = {
             ...searchParams,
             mode: 'hybrid',
@@ -719,7 +710,7 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
       }
     } catch (error) {
       // Fallback to fulltext if embedding fails or times out
-      lastSearchModeByProject.set(modeKey, 'fulltext (embedding unavailable)');
+      rememberSearchMode(modeKey, 'fulltext (embedding unavailable)');
       console.error('[memorix] Embedding failed or timed out, falling back to fulltext search');
     }
   }
@@ -730,7 +721,7 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
     results = await search(database, searchParams);
   } catch (error) {
     if (queryVector && isVectorDimensionMismatchError(error)) {
-      lastSearchModeByProject.set(modeKey, 'fulltext (embedding dimension mismatch)');
+      rememberSearchMode(modeKey, 'fulltext (embedding dimension mismatch)');
       console.error('[memorix] Vector search dimension mismatch detected, retrying without embeddings');
       results = await search(database, stripVectorSearchParams(searchParams));
     } else {
@@ -754,7 +745,7 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
         },
         similarity: 0.25,
       };
-      lastSearchModeByProject.set(modeKey, 'vector-only (hybrid empty fallback)');
+      rememberSearchMode(modeKey, 'vector-only (hybrid empty fallback)');
       results = await search(database, vectorOnlyParams);
     } catch {
       // Keep original empty results
@@ -1106,7 +1097,7 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
             .filter((entry): entry is NonNullable<typeof entry> => entry != null);
           if (rerankedTop.length > 0) {
             intermediate = [...rerankedTop, ...intermediate.slice(RERANK_TOP_K)];
-            lastSearchModeByProject.set(modeKey, (lastSearchModeByProject.get(modeKey) ?? 'fulltext') + ' + neural rerank');
+            rememberSearchMode(modeKey, (lastSearchModeByProject.get(modeKey) ?? 'fulltext') + ' + neural rerank');
             mark('rerank(usedNeural=true)');
           }
         }
@@ -1123,7 +1114,7 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
         );
         mark(`rerank(usedLLM=${usedLLM})`);
         if (usedLLM) {
-          lastSearchModeByProject.set(modeKey, (lastSearchModeByProject.get(modeKey) ?? 'fulltext') + ' + LLM rerank');
+          rememberSearchMode(modeKey, (lastSearchModeByProject.get(modeKey) ?? 'fulltext') + ' + LLM rerank');
           const rerankedTop = reranked
             .map((row) => candidateMap.get(row.id))
             .filter((entry): entry is NonNullable<typeof entry> => entry != null);
@@ -1224,14 +1215,6 @@ export async function getObservationsByIds(
   const results: MemorixDocument[] = [];
 
   for (const id of ids) {
-    if (projectId) {
-      const cached = docByObservationKey.get(makeEntryKey(projectId, id));
-      if (cached) {
-        results.push(cached);
-        continue;
-      }
-    }
-
     const searchResult = await search(database, {
       term: '',
       includeVectors: true,