memorix 1.9.0 → 1.9.2

package/src/store/obs-store.ts

@@ -45,6 +45,20 @@ export interface StoreTransaction {
   getGeneration(): Promise<number>;
 }
 
+export interface LexicalSearchOptions {
+  query: string;
+  projectId?: string | string[];
+  status?: string | 'all';
+  type?: string;
+  source?: string;
+  limit?: number;
+}
+
+export interface LexicalSearchHit {
+  observation: Observation;
+  score: number;
+}
+
 export interface ObservationStore {
   // ── Lifecycle ──────────────────────────────────────────────────────
 
@@ -63,7 +77,7 @@ export interface ObservationStore {
   ): Promise<Observation[]>;
 
   /** Count one project's observations without materializing the rows. */
-  countByProject(projectId: string, options?: { status?: string }): Promise<number>;
+  countByProject(projectId: string, options?: { status?: string; visibility?: 'project' }): Promise<number>;
 
   /** Load one observation by its globally unique ID. */
   getById(id: number): Promise<Observation | undefined>;
@@ -71,6 +85,24 @@ export interface ObservationStore {
   /** Load the current next-ID counter value. */
   loadIdCounter(): Promise<number>;
 
+  /** Count all durable observations without materializing their bodies. */
+  countAll?(): Promise<number>;
+
+  /** List project scopes without materializing observation bodies. */
+  listProjectIds?(): Promise<string[]>;
+
+  /**
+   * Search the persistent lexical index without materializing the corpus.
+   * Backends without FTS5 return an empty result and callers use the fallback.
+   */
+  searchLexical?(options: LexicalSearchOptions): Promise<LexicalSearchHit[]>;
+
+  /** Whether a persistent lexical index is available for this backend. */
+  hasLexicalIndex?(): boolean;
+
+  /** Rebuild the derived lexical index from durable observations. */
+  rebuildLexicalIndex?(): Promise<boolean>;
+
   // ── Write — single mutations ───────────────────────────────────────
 
   /** Insert a new observation. Bumps generation (if applicable). */
@@ -235,7 +267,7 @@ export class DegradedBackend implements ObservationStore {
     return [];
   }
 
-  async countByProject(_projectId: string, _options?: { status?: string }): Promise<number> {
+  async countByProject(_projectId: string, _options?: { status?: string; visibility?: 'project' }): Promise<number> {
     return 0;
   }
 
@@ -247,6 +279,26 @@ export class DegradedBackend implements ObservationStore {
     return 1;
   }
 
+  async countAll(): Promise<number> {
+    return 0;
+  }
+
+  async listProjectIds(): Promise<string[]> {
+    return [];
+  }
+
+  async searchLexical(_options: LexicalSearchOptions): Promise<LexicalSearchHit[]> {
+    return [];
+  }
+
+  hasLexicalIndex(): boolean {
+    return false;
+  }
+
+  async rebuildLexicalIndex(): Promise<boolean> {
+    return false;
+  }
+
   async insert(_obs: Observation): Promise<void> {
     throw new Error('[memorix] Cannot write observations: SQLite backend unavailable (degraded mode)');
   }

package/src/store/orama-store.ts

@@ -9,7 +9,7 @@
  */
 
 import { create, insert, insertMultiple, search, remove, update, count, getByID, type AnyOrama } from '@orama/orama';
-import type { MemorixDocument, SearchOptions, IndexEntry, KnowledgeLayer, ObservationReader } from '../types.js';
+import type { MemorixDocument, SearchOptions, IndexEntry, KnowledgeLayer, Observation, ObservationReader } from '../types.js';
 import { OBSERVATION_ICONS, type ObservationType } from '../types.js';
 import { resolveKnowledgeLayer } from '../skills/mini-skills.js';
 import { canReadObservation } from '../memory/visibility.js';
@@ -25,6 +25,14 @@ import { detectQueryIntent, applyIntentBoost } from '../search/intent-detector.j
 import { maybeExpandSearchQuery } from '../search/query-expansion.js';
 import { parseRerankTimeoutMs, shouldAttemptHttpRerank, shouldAttemptLlmRerank } from '../rerank/policy.js';
 import { withTimeout, withTimeoutSignal } from '../timeout.js';
+import { getObservationStore } from './obs-store.js';
+import {
+  createSemanticIndexProfile,
+  searchSemanticVectors,
+  upsertSemanticVectors,
+  type SemanticIndexProfile,
+  type SemanticVectorRecord,
+} from '../search/semantic-index.js';
 
 let db: AnyOrama | null = null;
 let dbInitPromise: Promise<AnyOrama> | null = null;
@@ -49,6 +57,30 @@ function rememberSearchMode(key: string, value: string): void {
 export function getLastSearchMode(projectId?: string): string {
   return lastSearchModeByProject.get(projectId ?? SEARCH_MODE_DEFAULT_KEY) ?? 'fulltext';
 }
+
+function getOptionalObservationStore(): ReturnType<typeof getObservationStore> | null {
+  try {
+    return getObservationStore();
+  } catch {
+    return null;
+  }
+}
+
+async function shouldUsePersistentSearch(
+  projectIds: string[] | null,
+  projectId?: string,
+): Promise<boolean> {
+  const store = getOptionalObservationStore();
+  if (!store?.hasLexicalIndex?.()) return false;
+  const rawThreshold = Number.parseInt(process.env.MEMORIX_ORAMA_HYDRATION_THRESHOLD ?? '', 10);
+  const threshold = Number.isSafeInteger(rawThreshold) && rawThreshold > 0 ? rawThreshold : 10_000;
+  const ids = projectIds ?? (projectId ? [projectId] : null);
+  if (ids) {
+    const count = (await Promise.all(ids.map((id) => store.countByProject(id)))).reduce((sum, value) => sum + value, 0);
+    return count >= threshold;
+  }
+  return (await store.countAll?.() ?? 0) >= threshold;
+}
 // Hard filter: titles starting with these are command execution logs, not knowledge.
 // They are excluded from results entirely (not just demoted) unless the query is command-like.
 const COMMAND_LOG_TITLE = /^(Ran:|Command:|Executed:)\s/i;
@@ -80,6 +112,185 @@ function rememberObservationDoc(doc: MemorixDocument): MemorixDocument {
   return publicDoc;
 }
 
+function observationToSearchDocument(observation: Observation): MemorixDocument {
+  return {
+    id: makeOramaObservationId(observation.projectId, observation.id),
+    observationId: observation.id,
+    entityName: observation.entityName,
+    type: observation.type,
+    title: observation.title,
+    narrative: observation.narrative,
+    facts: observation.facts.join(' '),
+    filesModified: observation.filesModified.join(' '),
+    concepts: observation.concepts.join(', '),
+    attachments: (observation.attachments ?? []).map((attachment) => [
+      attachment.name,
+      attachment.modality,
+      attachment.mimeType,
+      attachment.url,
+    ].filter(Boolean).join(' ')).join('\n'),
+    tokens: observation.tokens,
+    createdAt: observation.createdAt,
+    projectId: observation.projectId,
+    accessCount: 0,
+    lastAccessedAt: '',
+    status: observation.status ?? 'active',
+    source: observation.source ?? 'agent',
+    sourceDetail: observation.sourceDetail ?? '',
+    valueCategory: observation.valueCategory ?? '',
+    admissionState: observation.admissionState ?? '',
+    admissionReason: observation.admissionReason ?? '',
+    visibility: observation.visibility ?? 'project',
+    createdByAgentId: observation.createdByAgentId ?? '',
+    sharedWithAgentIds: JSON.stringify(observation.sharedWithAgentIds ?? []),
+    documentType: 'observation',
+    knowledgeLayer: resolveKnowledgeLayer('observation', observation.sourceDetail, observation.source),
+  };
+}
+
+type SearchHitLike = {
+  id: string;
+  score: number;
+  document: MemorixDocument;
+};
+
+type SearchResultLike = {
+  count: number;
+  hits: SearchHitLike[];
+};
+
+/**
+ * Use the durable SQLite FTS5 index for lexical-only queries. Returning a
+ * search-result-shaped value lets the existing visibility, intent, provenance,
+ * token-budget, and access-tracking pipeline remain shared with Orama.
+ */
+async function searchPersistentLexically(
+  options: SearchOptions,
+  projectIds: string[] | null,
+  limit: number,
+): Promise<SearchResultLike | null> {
+  try {
+    const store = getObservationStore();
+    if (!store.hasLexicalIndex?.() || !store.searchLexical) return null;
+    const hits = await store.searchLexical({
+      query: options.query ?? '',
+      projectId: projectIds ?? options.projectId,
+      status: options.status === 'all' ? 'all' : (options.status ?? 'active'),
+      type: options.type,
+      source: options.source,
+      limit,
+    });
+    return {
+      count: hits.length,
+      hits: hits.map((hit) => ({
+        id: makeOramaObservationId(hit.observation.projectId, hit.observation.id),
+        score: hit.score,
+        document: observationToSearchDocument(hit.observation),
+      })),
+    };
+  } catch {
+    // The derived FTS path is optional. A provider/runtime-specific failure
+    // must fall back to the established Orama search path.
+    return null;
+  }
+}
+
+/**
+ * Fuse the persistent lexical candidate list with the local LanceDB semantic
+ * candidate list. The vector index is a derived accelerator; authoritative
+ * observation rows still come from SQLite before visibility is applied.
+ */
+async function searchPersistentSemantically(
+  options: SearchOptions,
+  projectIds: string[] | null,
+  limit: number,
+  queryVector: number[],
+  provider: EmbeddingProvider,
+): Promise<SearchResultLike | null> {
+  try {
+    const { getObservationDataDir } = await import('../memory/observations.js');
+    const dataDir = getObservationDataDir();
+    if (!dataDir) return null;
+
+    const profile = createSemanticIndexProfile(provider);
+    const vectorHits = await searchSemanticVectors({
+      dataDir,
+      profile,
+      vector: queryVector,
+      projectId: projectIds ?? options.projectId,
+      status: 'all',
+      limit: Math.min(10_000, limit * (options.reader ? 8 : 2)),
+    });
+    if (vectorHits === null) return null;
+
+    const lexical = await searchPersistentLexically(options, projectIds, limit);
+    const store = getObservationStore();
+    const documents = new Map<string, MemorixDocument>();
+    const addObservation = (observation: Observation): void => {
+      const document = observationToSearchDocument(observation);
+      documents.set(makeEntryKey(document.projectId, document.observationId), document);
+    };
+
+    for (const hit of lexical?.hits ?? []) {
+      documents.set(makeEntryKey(hit.document.projectId, hit.document.observationId), hit.document);
+    }
+
+    const vectorObservations = await Promise.all(vectorHits.map(async (hit) => {
+      const observation = await store.getById(hit.observationId);
+      if (!observation) return null;
+      if (projectIds && !projectIds.includes(observation.projectId)) return null;
+      if (options.status !== 'all' && (observation.status ?? 'active') !== (options.status ?? 'active')) return null;
+      if (options.type && observation.type !== options.type) return null;
+      if (options.source && observation.source !== options.source) return null;
+      addObservation(observation);
+      return observation;
+    }));
+
+    // Reciprocal Rank Fusion keeps lexical identifier matches and semantic
+    // paraphrases comparable without pretending their raw score scales match.
+    const RRF_K = 60;
+    const scores = new Map<string, number>();
+    for (const [index, hit] of (lexical?.hits ?? []).entries()) {
+      const key = makeEntryKey(hit.document.projectId, hit.document.observationId);
+      scores.set(key, (scores.get(key) ?? 0) + 0.6 / (RRF_K + index + 1));
+    }
+    for (const [index, hit] of vectorHits.entries()) {
+      const key = makeEntryKey(hit.projectId, hit.observationId);
+      if (!documents.has(key)) continue;
+      scores.set(key, (scores.get(key) ?? 0) + 0.4 / (RRF_K + index + 1));
+    }
+
+    const hits = [...scores.entries()]
+      .sort((left, right) => right[1] - left[1])
+      .slice(0, limit)
+      .map(([key, score]) => ({
+        id: documents.get(key)!.id,
+        score,
+        document: documents.get(key)!,
+      }));
+    // Keep a vector-only result set useful when the FTS tokenizer cannot see
+    // the query language (for example a paraphrase or CJK text).
+    if (hits.length === 0 && vectorObservations.some(Boolean)) {
+      return {
+        count: vectorHits.length,
+        hits: vectorHits
+          .map((hit, index) => {
+            const key = makeEntryKey(hit.projectId, hit.observationId);
+            const document = documents.get(key);
+            return document ? { id: document.id, score: 0.4 / (RRF_K + index + 1), document } : null;
+          })
+          .filter((hit): hit is SearchHitLike => hit !== null)
+          .slice(0, limit),
+      };
+    }
+    return { count: hits.length, hits };
+  } catch {
+    // LanceDB is a derived optional accelerator. If it is unavailable or its
+    // table is mid-rebuild, the caller retains the Orama fallback.
+    return null;
+  }
+}
+
 function isCommandLikeQuery(query: string): boolean {
   return COMMAND_LIKE_QUERY.test(query);
 }
@@ -267,6 +478,29 @@ export function getVectorDimensions(): number | null {
   return embeddingEnabled ? embeddingDimensions : null;
 }
 
+/** Current derived semantic-index profile, if an embedding provider is active. */
+export function getActiveSemanticIndexProfile(): SemanticIndexProfile | null {
+  return indexEmbeddingProvider ? createSemanticIndexProfile(indexEmbeddingProvider) : null;
+}
+
+/** Queue one derived vector without delaying the durable observation write. */
+export async function queueSemanticVector(
+  dataDir: string,
+  record: Omit<SemanticVectorRecord, 'vector'> & { vector: number[] },
+): Promise<void> {
+  // A large-corpus writer can reset the in-memory Orama state after crossing
+  // the hydration threshold. Resolve the provider again so that reset does
+  // not strand newly generated vectors outside the persistent shadow index.
+  const provider = indexEmbeddingProvider ?? await getEmbeddingProvider();
+  const profile = provider ? createSemanticIndexProfile(provider) : null;
+  if (!profile) return;
+  try {
+    await upsertSemanticVectors(dataDir, profile, [record]);
+  } catch {
+    // The vector index is rebuildable; lexical memory remains authoritative.
+  }
+}
+
 /**
  * Generate embedding for text content using the available provider.
  * Returns null if no provider is available.
@@ -366,6 +600,8 @@ async function attachCachedVectors(
   database: AnyOrama,
   observations: any[],
 ): Promise<void> {
+  const semanticDataDir = (await import('../memory/observations.js')).getObservationDataDir();
+  const semanticProfile = getActiveSemanticIndexProfile();
   const batchSize = 200;
   for (let start = 0; start < observations.length; start += batchSize) {
     // Keep cache lookup and the temporary vector array bounded to one batch.
@@ -376,9 +612,20 @@ async function attachCachedVectors(
     // Do not attach stale vectors to that new index.
     if (db !== database) return;
 
+    const semanticRecords: SemanticVectorRecord[] = [];
     for (let index = 0; index < batch.length; index++) {
       const vector = cachedVectors[index];
       if (!isCompatibleCachedVector(vector)) continue;
+      const observation = batch[index];
+      if (semanticDataDir && semanticProfile) {
+        semanticRecords.push({
+          observationId: observation.id,
+          projectId: observation.projectId,
+          status: observation.status ?? 'active',
+          visibility: observation.visibility ?? 'project',
+          vector,
+        });
+      }
       const id = makeOramaObservationId(batch[index].projectId, batch[index].id);
       const existing = getByID(database, id) as MemorixDocument | undefined;
       if (!existing || documentEmbeddingText(existing) !== observationEmbeddingText(batch[index])) continue;
@@ -388,6 +635,11 @@ async function attachCachedVectors(
         // Vector cache hydration is best-effort. The normal backfill lane owns misses.
       }
     }
+    if (semanticDataDir && semanticProfile && semanticRecords.length > 0) {
+      void upsertSemanticVectors(semanticDataDir, semanticProfile, semanticRecords).catch(() => {
+        // A rebuildable semantic index must never make startup hydration fail.
+      });
+    }
     await new Promise<void>((resolve) => setImmediate(resolve));
   }
 }
@@ -656,10 +908,12 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
   // If embedding provider is available and query tier warrants it, use hybrid search
   // Fast-tier queries skip embedding entirely (fulltext is sufficient)
   let queryVector: number[] | null = null;
+  let embeddingProviderForSearch: EmbeddingProvider | null = null;
   if (quality !== 'fast' && embeddingEnabled && hasQuery && tier !== 'fast') {
     try {
       const provider = await getEmbeddingProvider();
       if (provider) {
+        embeddingProviderForSearch = provider;
         const activeVectorDimensions = getVectorDimensions();
         if (activeVectorDimensions !== null && provider.dimensions !== activeVectorDimensions) {
           rememberSearchMode(
@@ -716,19 +970,47 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
   }
 
   mark('preSearch');
-  let results;
-  try {
-    results = await search(database, searchParams);
-  } catch (error) {
-    if (queryVector && isVectorDimensionMismatchError(error)) {
-      rememberSearchMode(modeKey, 'fulltext (embedding dimension mismatch)');
-      console.error('[memorix] Vector search dimension mismatch detected, retrying without embeddings');
-      results = await search(database, stripVectorSearchParams(searchParams));
-    } else {
-      throw error;
+  let results: SearchResultLike | Awaited<ReturnType<typeof search>>;
+  const persistentCorpusEligible = await shouldUsePersistentSearch(projectIds, options.projectId);
+  const canUsePersistentLexical = persistentCorpusEligible && Boolean(hasQuery) && !queryVector &&
+    (quality === 'fast' || tier === 'fast' || !embeddingEnabled);
+  let persistent = persistentCorpusEligible && queryVector && embeddingProviderForSearch
+    ? await searchPersistentSemantically(options, projectIds, requestLimit, queryVector, embeddingProviderForSearch)
+    : canUsePersistentLexical
+      ? await searchPersistentLexically(options, projectIds, requestLimit)
+      : null;
+
+  // In lazy large-corpus mode Orama is intentionally empty. If a semantic
+  // shadow index has not been built yet, retain a correct lexical answer
+  // instead of returning no memories while background indexing catches up.
+  const durableStore = getOptionalObservationStore();
+  if (!persistent && queryVector && durableStore?.hasLexicalIndex?.() && count(database) === 0) {
+    persistent = await searchPersistentLexically(options, projectIds, requestLimit);
+  }
+
+  if (persistent) {
+    results = persistent;
+    // Keep the public mode label compatible with existing integrations while
+    // exposing the concrete accelerator through performance marks/status.
+    rememberSearchMode(
+      modeKey,
+      queryVector ? 'hybrid' : quality === 'fast' ? 'fulltext (fast profile)' : 'fulltext',
+    );
+    mark(queryVector ? 'persistentHybridSearch' : 'sqliteFtsSearch');
+  } else {
+    try {
+      results = await search(database, searchParams);
+    } catch (error) {
+      if (queryVector && isVectorDimensionMismatchError(error)) {
+        rememberSearchMode(modeKey, 'fulltext (embedding dimension mismatch)');
+        console.error('[memorix] Vector search dimension mismatch detected, retrying without embeddings');
+        results = await search(database, stripVectorSearchParams(searchParams));
+      } else {
+        throw error;
+      }
     }
+    mark('oramaSearch');
   }
-  mark('oramaSearch');
 
   // Fallback: if hybrid returned nothing but we have a vector, retry with vector-only
   if (results.count === 0 && queryVector && embeddingEnabled) {
@@ -1345,6 +1627,12 @@ function applyTokenBudget(entries: IndexEntry[], maxTokens: number): IndexEntry[
  * Get total observation count, optionally filtered by project.
  */
 export async function getObservationCount(projectId?: string): Promise<number> {
+  const durableStore = getObservationStore();
+  if (durableStore.countAll && durableStore.countByProject) {
+    return projectId
+      ? durableStore.countByProject(projectId)
+      : durableStore.countAll();
+  }
   const database = await getDb();
   if (!projectId) {
     return await count(database);

package/src/store/sqlite-db.ts

@@ -17,6 +17,7 @@ import path from 'node:path';
 import fs from 'node:fs';
 import { createDatabase, loadSqlite } from './bun-sqlite-compat.js';
 import { assertNotHomeDataDir } from '../project/launch-root.js';
+import { initializeObservationLexicalIndex } from './sqlite-fts.js';
 
 // Dynamic require for SQLite (better-sqlite3, node:sqlite, or bun:sqlite)
 let BetterSqlite3: any;
@@ -57,6 +58,7 @@ CREATE TABLE IF NOT EXISTS observations (
   commitHash      TEXT,
   relatedCommits  TEXT,
   relatedEntities TEXT,
+  attachments     TEXT,
   sourceDetail    TEXT,
   valueCategory   TEXT,
   admissionState  TEXT,
@@ -1177,6 +1179,10 @@ export function getDatabase(dataDir: string): any {
   // Create indexes AFTER all ALTER TABLE migrations so referenced columns exist
   db.exec(CREATE_INDEXES);
 
+  // FTS5 is a derived accelerator. It is initialized after all migrations so
+  // old databases have the attachment column before triggers are created.
+  initializeObservationLexicalIndex(db);
+
   // Seed meta defaults
   db.prepare(`INSERT OR IGNORE INTO meta (key, value) VALUES ('storage_generation', '0')`).run();
   db.prepare(`INSERT OR IGNORE INTO meta (key, value) VALUES ('next_id', '1')`).run();

package/src/store/sqlite-fts.ts

@@ -0,0 +1,164 @@
+import type { Observation } from '../types.js';
+
+/** Rows returned by the derived FTS5 index before normal Observation parsing. */
+export interface SqliteLexicalHit {
+  row: Record<string, unknown>;
+  score: number;
+}
+
+const ftsAvailability = new WeakMap<object, boolean>();
+const FTS_TABLE = 'observations_fts';
+
+const CREATE_OBSERVATION_FTS = `
+CREATE VIRTUAL TABLE IF NOT EXISTS observations_fts USING fts5(
+  entityName,
+  title,
+  narrative,
+  facts,
+  filesModified,
+  concepts,
+  attachments,
+  content='observations',
+  content_rowid='id',
+  tokenize='unicode61 remove_diacritics 2'
+);
+
+CREATE TRIGGER IF NOT EXISTS observations_fts_ai AFTER INSERT ON observations BEGIN
+  INSERT INTO observations_fts(rowid, entityName, title, narrative, facts, filesModified, concepts, attachments)
+  VALUES (new.id, new.entityName, new.title, new.narrative, new.facts, new.filesModified, new.concepts, new.attachments);
+END;
+
+CREATE TRIGGER IF NOT EXISTS observations_fts_ad AFTER DELETE ON observations BEGIN
+  INSERT INTO observations_fts(observations_fts, rowid, entityName, title, narrative, facts, filesModified, concepts, attachments)
+  VALUES ('delete', old.id, old.entityName, old.title, old.narrative, old.facts, old.filesModified, old.concepts, old.attachments);
+END;
+
+CREATE TRIGGER IF NOT EXISTS observations_fts_au AFTER UPDATE ON observations BEGIN
+  INSERT INTO observations_fts(observations_fts, rowid, entityName, title, narrative, facts, filesModified, concepts, attachments)
+  VALUES ('delete', old.id, old.entityName, old.title, old.narrative, old.facts, old.filesModified, old.concepts, old.attachments);
+  INSERT INTO observations_fts(rowid, entityName, title, narrative, facts, filesModified, concepts, attachments)
+  VALUES (new.id, new.entityName, new.title, new.narrative, new.facts, new.filesModified, new.concepts, new.attachments);
+END;
+`;
+
+function normalizeFtsToken(token: string): string {
+  return token.replace(/"/g, '""');
+}
+
+/**
+ * Turn user text into a parameterized FTS query. Every token is quoted so
+ * punctuation such as `OR`, `NEAR`, and quotes remains user data.
+ */
+export function buildFtsQuery(query: string): string | null {
+  const tokens = query.normalize('NFKC').match(/[\p{L}\p{N}_]+/gu) ?? [];
+  const unique = [...new Set(tokens.map((token) => token.trim()).filter(Boolean))].slice(0, 64);
+  if (unique.length === 0) return null;
+  return unique.map((token) => `"${normalizeFtsToken(token)}"`).join(' OR ');
+}
+
+/**
+ * Create and reconcile the external-content FTS5 index for one SQLite handle.
+ * FTS5 is an optional SQLite capability, so an unavailable extension is a
+ * normal fallback rather than a reason to make the durable store unusable.
+ */
+export function initializeObservationLexicalIndex(db: any): boolean {
+  const known = ftsAvailability.get(db);
+  if (known !== undefined) return known;
+
+  try {
+    db.exec(CREATE_OBSERVATION_FTS);
+    const observationCount = Number(db.prepare('SELECT COUNT(*) AS count FROM observations').get()?.count ?? 0);
+    const indexedCount = Number(db.prepare(`SELECT COUNT(*) AS count FROM ${FTS_TABLE}`).get()?.count ?? 0);
+    if (observationCount !== indexedCount) {
+      db.exec(`INSERT INTO ${FTS_TABLE}(${FTS_TABLE}) VALUES ('rebuild')`);
+    }
+    ftsAvailability.set(db, true);
+    return true;
+  } catch (error) {
+    ftsAvailability.set(db, false);
+    console.warn(
+      `[memorix] SQLite FTS5 unavailable; using the existing search fallback: ${error instanceof Error ? error.message : String(error)}`,
+    );
+    return false;
+  }
+}
+
+export function isObservationLexicalIndexEnabled(db: any): boolean {
+  return ftsAvailability.get(db) === true;
+}
+
+export function rebuildObservationLexicalIndex(db: any): boolean {
+  if (!isObservationLexicalIndexEnabled(db)) return false;
+  try {
+    db.exec(`INSERT INTO ${FTS_TABLE}(${FTS_TABLE}) VALUES ('rebuild')`);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export interface SqliteLexicalSearchOptions {
+  query: string;
+  projectId?: string | string[];
+  status?: string | 'all';
+  type?: string;
+  source?: string;
+  limit?: number;
+}
+
+/**
+ * Search only the bounded lexical candidate set. The full observation body is
+ * fetched from SQLite for those candidates, never by loading the corpus.
+ */
+export function searchObservationLexically(db: any, options: SqliteLexicalSearchOptions): SqliteLexicalHit[] {
+  if (!isObservationLexicalIndexEnabled(db)) return [];
+  const match = buildFtsQuery(options.query);
+  if (!match) return [];
+
+  const projectIds = options.projectId == null
+    ? undefined
+    : (Array.isArray(options.projectId) ? options.projectId : [options.projectId]);
+  if (projectIds && projectIds.length === 0) return [];
+
+  const where: string[] = [`${FTS_TABLE} MATCH ?`];
+  const params: unknown[] = [match];
+  if (projectIds) {
+    where.push(`o.projectId IN (${projectIds.map(() => '?').join(', ')})`);
+    params.push(...projectIds);
+  }
+  if (options.status && options.status !== 'all') {
+    where.push('o.status = ?');
+    params.push(options.status);
+  }
+  if (options.type) {
+    where.push('o.type = ?');
+    params.push(options.type);
+  }
+  if (options.source) {
+    where.push('o.source = ?');
+    params.push(options.source);
+  }
+
+  const requestedLimit = Number.isFinite(options.limit) ? Math.floor(options.limit!) : 20;
+  // This is a per-query working-set guard, not a corpus limit. Large callers
+  // can page or request up to 10k candidates without changing durable data.
+  const limit = Math.max(1, Math.min(requestedLimit, 10_000));
+  params.push(limit);
+
+  const rows = db.prepare(`
+    SELECT o.*, -bm25(${FTS_TABLE}, 2.0, 4.0, 1.0, 0.7, 0.5, 1.5, 0.5) AS lexical_score
+    FROM ${FTS_TABLE}
+    JOIN observations AS o ON o.id = ${FTS_TABLE}.rowid
+    WHERE ${where.join(' AND ')}
+    ORDER BY bm25(${FTS_TABLE}, 2.0, 4.0, 1.0, 0.7, 0.5, 1.5, 0.5) ASC, o.id DESC
+    LIMIT ?
+  `).all(...params) as Array<Record<string, unknown>>;
+
+  return rows.map((row) => ({
+    row,
+    score: Math.max(0, Number(row.lexical_score ?? 0)),
+  }));
+}
+
+/** Keep the import visible to TypeScript consumers that build store adapters. */
+export type SqliteLexicalObservation = Observation;