@lincoln504/pi-research 1.6.18 → 1.6.20

package/src/index.ts

@@ -91,9 +91,11 @@ function extractResultText(result: AgentToolResult<unknown>): string {
  * Pi Research Extension
  */
 export default async function (pi: ExtensionAPI) {
-  // Runtime version check — must match the @earendil-works/* dependency minimum (>=0.84.0).
-  // The floor is 0.84.0 because setRuntimeApiKey() lost its `allowNetwork` option there
-  // and only 0.84.0+ hardcodes the equivalent guard internally (see pi-version.ts). The
+  // Runtime version check — must match the @earendil-works/* dependency minimum (>=0.87.0).
+  // The floor was raised to 0.84.0 because setRuntimeApiKey() lost its `allowNetwork` option
+  // there and only 0.84.0+ hardcodes the equivalent guard internally (see pi-version.ts), then
+  // to 0.85.0 to track the pi extension package line, and to 0.87.0 for the 0.86
+  // TranscriptContext / 0.87 SessionEntry-union surface (see pi-version.ts). The
   // APIs it rests on arrived in 0.80.8, which introduced ModelRuntime and removed
   // AuthStorage/ModelRegistry.create(); buildModelRegistry (model-registry-factory.ts)
   // unconditionally calls ModelRuntime.create(), and createAgentSession() is invoked

package/src/infrastructure/browser/thread-worker.mjs

@@ -2699,6 +2699,13 @@ var ConfigSchema = Type.Object({
   WORKER_CONCURRENCY: Type.Number({ minimum: 1, maximum: 10, default: 2 }),
   /** Knowledge store isolation mode (default: 'global' — one shared store across every directory) */
   KNOWLEDGE_STORE_MODE: Type.Union([Type.Literal("none"), Type.Literal("project"), Type.Literal("global")], { default: "global" }),
+  /** Knowledge store retrieval strategy (default: 'vector' — current hybrid dense+sparse behavior).
+   *  'vector' = hybrid search (embedding similarity + BM25/FTS fused via RRF; requires the
+   *  optional @huggingface/transformers dependency and a downloaded embedding model).
+   *  'bm25'   = pure lexical/BM25 retrieval over the same stored findings — NO embedding model:
+   *  never initialized, downloaded, or invoked. Uses a separate lexical table in the same
+   *  LanceDB database, so existing vector stores are untouched (no migration). */
+  KNOWLEDGE_STORE_RETRIEVAL: Type.Union([Type.Literal("vector"), Type.Literal("bm25")], { default: "vector" }),
   /** Embedding model to use for the knowledge store */
   EMBEDDING_MODEL: Type.String({ default: "onnx-community/granite-embedding-small-english-r2-ONNX" }),
   /** Hardware backend for embeddings: 'auto' (probe WebGPU viability out-of-process, fall back to CPU), 'webgpu' (force), or 'cpu' (force).

package/src/infrastructure/knowledge-store-service.ts

@@ -50,6 +50,10 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
   // mode change (project↔global re-scope, or enabled→none) and re-initialize so a /research-config
   // change applies without a Pi restart — not just the DISABLED→enabled revival.
   private _initializedMode: string | undefined = undefined;
+  // The retrieval strategy the current build reflects ('vector' hybrid / 'bm25'
+  // lexical) — drives the embedder-optional invariants below and the live
+  // re-init detection on a /research-config retrieval change.
+  private _initializedRetrieval: 'vector' | 'bm25' | undefined = undefined;
 
   // Initialization promise to prevent concurrent initialization
   private _initializationPromise: Promise<void> | null = null;
@@ -88,7 +92,12 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
     //  - INITIALIZED and mode != the built mode        : project↔global re-scope, or enabled→'none'.
     // A native-unavailable DISABLED ('native') stays memoized — retrying is futile on that platform.
     if ((this.lifecycle === ServiceLifecycle.INITIALIZED || this.lifecycle === ServiceLifecycle.DISABLED) && this._cwd === newCwd) {
-      const liveMode: string = (ctx?.config || getConfig(this._cwd)).KNOWLEDGE_STORE_MODE;
+      const liveConfig = ctx?.config || getConfig(this._cwd);
+      const liveMode: string = liveConfig.KNOWLEDGE_STORE_MODE;
+      // Retrieval strategy ('vector' hybrid / 'bm25' lexical) participates in the same
+      // live-change detection: a /research-config switch must re-scope the store onto
+      // the OTHER table (lexical vs vector schema) without a Pi restart.
+      const liveRetrieval: string = liveConfig.KNOWLEDGE_STORE_RETRIEVAL;
       // Reviving a DISABLED store creates handles (no live state to tear down), so it is safe even
       // from a lazy call and uses the live config. Re-scoping/disabling a LIVE (INITIALIZED) store,
       // however, disposes handles — restrict that to an EXPLICIT ctx.config (mirrors the ctx.cwd
@@ -101,10 +110,15 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
         ctx?.config !== undefined &&
         this._initializedMode !== undefined &&
         this._initializedMode !== ctx.config.KNOWLEDGE_STORE_MODE;
-      if (!reviveFromMode && !modeChanged) {
+      const retrievalChanged =
+        this.lifecycle === ServiceLifecycle.INITIALIZED &&
+        ctx?.config !== undefined &&
+        this._initializedRetrieval !== undefined &&
+        this._initializedRetrieval !== ctx.config.KNOWLEDGE_STORE_RETRIEVAL;
+      if (!reviveFromMode && !modeChanged && !retrievalChanged) {
         return;
       }
-      logger.log(`[KnowledgeStoreService] Knowledge Mode changed (${this._initializedMode ?? this._disabledReason} → ${liveMode}); re-initializing store (no restart needed).`);
+      logger.log(`[KnowledgeStoreService] Knowledge config changed (${this._initializedMode ?? this._disabledReason} → ${liveMode}, retrieval ${this._initializedRetrieval ?? '?'} → ${liveRetrieval}); re-initializing store (no restart needed).`);
       // An INITIALIZED store holds live handles (embedder/LanceDB/writer) — dispose before rebuilding.
       if (this.lifecycle === ServiceLifecycle.INITIALIZED) {
         await this.dispose();
@@ -170,6 +184,7 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
         const config = ctx?.config || getConfig(this._cwd);
         // Record the mode this build reflects so a later runtime mode change is detected above.
         this._initializedMode = config.KNOWLEDGE_STORE_MODE;
+        this._initializedRetrieval = config.KNOWLEDGE_STORE_RETRIEVAL;
         const embedderFactory = () => getEmbedder(config);
         const reconnectFactory = async () => {
           // MUST await: clearEmbeddingInstance() only nulls the cached instance AFTER its
@@ -260,7 +275,7 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
           //    calls don't re-run init, and so the healthcheck/config surfaces
           //    report the package-level reason rather than a user choice the user
           //    never made.
-          const depsMissing = !probeKnowledgeStoreAvailability().available;
+          const depsMissing = !probeKnowledgeStoreAvailability(config.KNOWLEDGE_STORE_RETRIEVAL).available;
           logger.debug(`[KnowledgeStoreService] Knowledge store is disabled (${depsMissing ? 'dependencies missing' : 'mode = none'}). Setting lifecycle to DISABLED.`);
           this.lifecycle = ServiceLifecycle.DISABLED;
           this._disabledReason = depsMissing ? 'native' : 'mode';
@@ -431,6 +446,11 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
    * Check if the knowledge store is ready
    */
   isReady(): boolean {
+    // bm25 mode has no embedder by design — the store+writer pair is the readiness
+    // contract there. Vector mode keeps the original three-way check.
+    if (this._initializedRetrieval === 'bm25') {
+      return this._store !== null && this._writerQueue !== null;
+    }
     return this._embedder !== null && this._store !== null && this._writerQueue !== null;
   }
 
@@ -458,6 +478,11 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
     if (this.lifecycle === ServiceLifecycle.DISABLED) {
       return null;
     }
+    if (this._initializedRetrieval === 'bm25') {
+      // Lexical mode has no embedder — null is the CORRECT answer here, not an
+      // error. Callers must treat it as 'no embedding model configured'.
+      return null;
+    }
     if (!this._embedder) {
       throw new Error('[KnowledgeStoreService] Embedder not initialized');
     }

package/src/knowledge/availability.ts

@@ -46,17 +46,36 @@ export interface KnowledgeStoreAvailability {
 }
 
 /**
- * The packages the store cannot run without. `@lancedb/lancedb` is the vector
- * store itself; `@huggingface/transformers` is the only embedding backend
- * (see transformers-loader.ts — there is no remote-embedding fallback).
+ * The packages the store cannot run without, per retrieval mode.
+ *
+ * Every mode needs `@lancedb/lancedb` — it backs the store in ALL modes (its
+ * native FTS/Tantivy engine provides BM25 ranking in lexical mode too).
+ *
+ * Only the 'vector' (hybrid) retrieval mode additionally needs
+ * `@huggingface/transformers` (the only embedding backend — see
+ * transformers-loader.ts, there is no remote-embedding fallback). Selecting
+ * retrieval='bm25' is a HARD dependency boundary: the embedding package is
+ * never resolved, loaded, or downloaded in lexical mode, so a bm25-only host
+ * (e.g. an install with --omit=optional) runs the store fully.
  */
-const REQUIRED_PACKAGES: readonly string[] = ['@lancedb/lancedb', '@huggingface/transformers'];
+function requiredPackages(retrieval: KnowledgeStoreRetrieval): readonly string[] {
+  return retrieval === 'bm25'
+    ? ['@lancedb/lancedb']
+    : ['@lancedb/lancedb', '@huggingface/transformers'];
+}
+
+/** Knowledge store retrieval strategy. 'vector' = hybrid dense+sparse (default,
+ *  embedding-backed); 'bm25' = pure lexical, no embedding model. */
+export type KnowledgeStoreRetrieval = 'vector' | 'bm25';
 
 // Memoized per-process: resolution touches the filesystem on every miss, the
 // answer cannot change within a live process without a reinstall mid-flight,
 // and the probe sits on settings/health paths that run often. Tests reset it
 // via clearAvailabilityCache.
 let _cached: KnowledgeStoreAvailability | null = null;
+// The retrieval mode the cached probe answer was computed for — the cache must
+// be keyed on it because the required-package set differs between modes.
+let _cachedRetrieval: KnowledgeStoreRetrieval | null = null;
 
 /** Resolver signature — injectable so tests never need the real filesystem. */
 export type PackageResolver = (specifier: string) => string;
@@ -75,10 +94,10 @@ const defaultResolver: PackageResolver = (specifier) =>
  * Probe whether the knowledge store's required packages are resolvable.
  * Memoized after the first call; {@link clearAvailabilityCache} resets.
  */
-export function probeKnowledgeStoreAvailability(resolver: PackageResolver = defaultResolver): KnowledgeStoreAvailability {
-  if (cachedIsValidFor(resolver)) return _cached!;
+export function probeKnowledgeStoreAvailability(retrieval: KnowledgeStoreRetrieval = 'vector', resolver: PackageResolver = defaultResolver): KnowledgeStoreAvailability {
+  if (cachedIsValidFor(retrieval, resolver)) return _cached!;
   const missing: string[] = [];
-  for (const pkg of REQUIRED_PACKAGES) {
+  for (const pkg of requiredPackages(retrieval)) {
     try {
       resolver(pkg);
     } catch {
@@ -88,17 +107,21 @@ export function probeKnowledgeStoreAvailability(resolver: PackageResolver = defa
     }
   }
   const result: KnowledgeStoreAvailability = { available: missing.length === 0, missing };
-  if (resolver === defaultResolver) _cached = result;
+  if (resolver === defaultResolver) {
+    _cached = result;
+    _cachedRetrieval = retrieval;
+  }
   return result;
 }
 
-function cachedIsValidFor(resolver: PackageResolver): boolean {
-  return _cached !== null && resolver === defaultResolver;
+function cachedIsValidFor(retrieval: KnowledgeStoreRetrieval, resolver: PackageResolver): boolean {
+  return _cached !== null && _cachedRetrieval === retrieval && resolver === defaultResolver;
 }
 
 /** Reset the memoized probe result (test isolation). */
 export function clearAvailabilityCache(): void {
   _cached = null;
+  _cachedRetrieval = null;
 }
 
 /**

package/src/knowledge/index.ts

@@ -33,7 +33,9 @@ function getMigrationStrategy(config: Config): MigrationStrategy | undefined {
  * Result of knowledge store initialization
  */
 export interface KnowledgeStoreComponents {
-  embedder: IEmbedder;
+  /** null in bm25 (lexical) retrieval mode — the store runs with NO embedding
+   *  model; non-null in 'vector' (hybrid) mode. */
+  embedder: IEmbedder | null;
   store: IKnowledgeStore;
   writerQueue: WriterQueue;
 }
@@ -51,7 +53,7 @@ export async function createKnowledgeStoreComponents(
   withLock?: <T>(fn: () => Promise<T>) => Promise<T>,
   config: Config = getConfig(),
   workspace: string = process.cwd(),
-  availability: KnowledgeStoreAvailability = probeKnowledgeStoreAvailability()
+  availability: KnowledgeStoreAvailability = probeKnowledgeStoreAvailability(config.KNOWLEDGE_STORE_RETRIEVAL)
 ): Promise<KnowledgeStoreComponents | null> {
   if (config.KNOWLEDGE_STORE_MODE === 'none') {
     logger.debug('[knowledge] Knowledge store is disabled in configuration');
@@ -76,15 +78,20 @@ export async function createKnowledgeStoreComponents(
   for (let attempt = 1; attempt <= MAX_INIT_RETRIES; attempt++) {
     let store: KnowledgeStore | undefined;
     try {
-      logger.info(`[knowledge] Creating Knowledge Store components (attempt ${attempt}/${MAX_INIT_RETRIES})...`);
+      logger.info(`[knowledge] Creating Knowledge Store components (attempt ${attempt}/${MAX_INIT_RETRIES}, retrieval=${config.KNOWLEDGE_STORE_RETRIEVAL})...`);
 
-      const embedder = await embedderFactory();
+      // Hard dependency boundary: in bm25 (lexical) retrieval mode the embedder
+      // factory is NEVER invoked — no @huggingface/transformers resolution, no
+      // model download, no initialization, no inference. The store runs purely
+      // on LanceDB's FTS/BM25 engine.
+      const embedder = config.KNOWLEDGE_STORE_RETRIEVAL === 'bm25' ? null : await embedderFactory();
       const migrationStrategy = getMigrationStrategy(config);
 
       store = new KnowledgeStore({
         dbDir: getDbDir(config, workspace),
         embedder,
         modelName: config.EMBEDDING_MODEL,
+        retrieval: config.KNOWLEDGE_STORE_RETRIEVAL,
         migrationStrategy: migrationStrategy,
         reconnectFactory,
         withLock,
@@ -149,8 +156,12 @@ export async function forceDeleteKnowledgeStore(config?: Config, workspace?: str
   const manifestPath = path.join(dbDir, 'store-manifest.json');
   try {
     if (fs.existsSync(manifestPath)) {
-      const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8')) as { activeTableName?: string };
-      if (manifest.activeTableName) tableName = manifest.activeTableName;
+      const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8')) as { activeTableName?: string; activeBm25TableName?: string };
+      // Read the manifest key that belongs to the CONFIGURED retrieval mode —
+      // each mode owns its own key so a shared dbDir never cross-wires them.
+      const key = config?.KNOWLEDGE_STORE_RETRIEVAL === 'bm25' ? 'activeBm25TableName' : 'activeTableName';
+      const saved = key === 'activeBm25TableName' ? manifest.activeBm25TableName : manifest.activeTableName;
+      if (saved) tableName = saved;
     }
   } catch (err) {
     logger.warn(`[knowledge] Failed to read manifest before forced delete, defaulting to table '${tableName}':`, err);

package/src/knowledge/store-operations.ts

@@ -188,6 +188,183 @@ export async function searchStore(
   return filteredResults;
 }
 
+/**
+ * Add documents to the BM25 (lexical) table — the vector write path minus the
+ * embedding step. Rows carry no vector at all; lexical ranking comes from the
+ * Tantivy FTS indexes (see store-schema.ts). Everything else — row shape,
+ * scope columns, ingestion_type fallback, metrics — matches addDocumentsToStore
+ * so the two backends stay interchangeable behind IKnowledgeStore.
+ */
+export async function addDocumentsBm25ToStore(
+  table: lancedb.Table,
+  docs: StoreDocument[],
+  isClosing: () => boolean,
+  workspace: string,
+  isGlobal: boolean
+): Promise<AddDocumentsOutcome> {
+  if (docs.length === 0) return 'empty';
+  if (isClosing()) {
+    logger.warn('[store] Ignoring addDocuments during close');
+    return 'ignored_closing';
+  }
+
+  const startTime = Date.now();
+
+  try {
+    const data = docs.map((doc) => ({
+      url: doc.url,
+      text: doc.text,
+      content: doc.content ?? null,
+      metadata: JSON.stringify(doc.metadata),
+      workspace,
+      is_global: isGlobal,
+      ingestion_type: doc.ingestion_type || (doc.metadata['ingestionType'] as string) || 'unknown',
+      timestamp: BigInt(doc.timestamp),
+    }));
+
+    await table.add(data);
+    const duration = Date.now() - startTime;
+    metrics.observe('knowledge_store_add_documents_duration_ms', duration, { retrieval: 'bm25' });
+    metrics.increment('knowledge_store_chunks_added_total', docs.length);
+    logger.log(`[store] Added ${docs.length} chunk(s) for ${docs[0]?.url} [retrieval=bm25, workspace=${workspace}, global=${isGlobal}]`);
+    return 'written';
+  } catch (err) {
+    const duration = Date.now() - startTime;
+    metrics.observe('knowledge_store_add_documents_duration_ms', duration, { status: 'error', retrieval: 'bm25' });
+    // Same accounting posture as the vector path: attempt-level failures are
+    // thrown; outcome accounting belongs to KnowledgeStore.addDocuments.
+    logger.debug(`[store] bm25 add attempt failed after ${duration}ms: ${err instanceof Error ? err.message : String(err)}`);
+    throw err;
+  }
+}
+
+/**
+ * Search the BM25 (lexical) table — pure full-text retrieval, no embedding.
+ *
+ * Runs the SAME lexical half the hybrid path already uses: Tantivy BM25 over
+ * BOTH 'text' (synthesis descriptions) and 'content' (full article body),
+ * ranked by BM25 score alone (no nearestTo, no RRF fusion — there is no second
+ * signal to fuse). Filters, result mapping, and metrics match searchStore so
+ * callers see identical StoreDocument results either way.
+ */
+export async function searchStoreBm25(
+  table: lancedb.Table,
+  query: string,
+  limit: number,
+  scopeFilter?: string
+): Promise<StoreDocument[]> {
+  const startTime = Date.now();
+
+  const rowCount = scopeFilter ? await table.countRows(scopeFilter) : await table.countRows();
+  if (rowCount === 0) {
+    metrics.increment('knowledge_store_search_total', 1, { status: 'empty', retrieval: 'bm25' });
+    return [];
+  }
+
+  let filter = "ingestion_type = 'synthesis-description'";
+  if (scopeFilter) {
+    filter = `(${filter}) AND (${scopeFilter})`;
+  }
+
+  // Pure FTS query: no nearestTo() — a Query (not a VectorQuery) ranked by BM25.
+  const results = await table
+    .query()
+    .where(filter)
+    .fullTextSearch(query, { columns: ['text', 'content'] })
+    .limit(limit)
+    .toArray();
+
+  const filteredResults = results
+    .map(r => {
+      let metadata: Record<string, unknown>;
+      try {
+        metadata = JSON.parse(r.metadata as string) as Record<string, unknown>;
+      } catch {
+        logger.debug('[store] Corrupted metadata in searchStoreBm25, skipping row');
+        return null;
+      }
+      return {
+        url: r.url as string,
+        text: r.text as string,
+        content: (r.content as string | null) ?? undefined,
+        metadata,
+        timestamp: Number(r.timestamp),
+      };
+    })
+    .filter((doc): doc is NonNullable<typeof doc> => doc !== null);
+
+  const duration = Date.now() - startTime;
+  metrics.observe('knowledge_store_search_duration_ms', duration);
+  metrics.increment('knowledge_store_search_total', 1, { status: 'success', retrieval: 'bm25' });
+  metrics.increment('knowledge_store_search_results_total', filteredResults.length);
+
+  return filteredResults;
+}
+
+/**
+ * Find relevant URLs in the BM25 (lexical) table — the vector variant minus
+ * the embedding step: BM25 ranking over 'text' + 'content', same filters,
+ * same provenance decoration, same result shape as findRelevantUrls.
+ */
+export async function findRelevantUrlsBm25(
+  table: lancedb.Table,
+  query: string,
+  limit: number,
+  scopeFilter?: string
+): Promise<{ url: string; description: string; provenance?: string }[]> {
+  const startTime = Date.now();
+  const rowCount = scopeFilter ? await table.countRows(scopeFilter) : await table.countRows();
+  if (rowCount === 0) {
+    metrics.increment('knowledge_store_find_urls_total', 1, { status: 'empty', retrieval: 'bm25' });
+    return [];
+  }
+
+  let filter = "ingestion_type = 'synthesis-description'";
+  if (scopeFilter) {
+    filter = `(${filter}) AND (${scopeFilter})`;
+  }
+
+  const results = await table
+    .query()
+    .where(filter)
+    .fullTextSearch(query, { columns: ['text', 'content'] })
+    .limit(limit)
+    .toArray();
+
+  const entries: { url: string; description: string; provenance?: string }[] = [];
+  const seen = new Set<string>();
+  for (const r of results) {
+    const url = r.url as string;
+    if (seen.has(url)) continue;
+    seen.add(url);
+
+    let description = '';
+    let provenance = 'description-unverified';
+    try {
+      const meta = JSON.parse(r.metadata as string);
+      description = meta.description as string ?? '';
+      // Surface provenance metadata so consumers can prefer verified entries
+      if (meta.provenance) provenance = meta.provenance;
+      else if (meta.hasContent === true) provenance = 'scraped-verified';
+    } catch { logger.debug('[store] Corrupted metadata in findRelevantUrlsBm25, skipping row'); }
+
+    if (!description) description = (r.text as string ?? '').substring(0, 300);
+
+    // Add store scope to provenance
+    const scope = r.is_global === true ? 'Global Store' : `Local Project (${r.workspace || 'unknown'})`;
+    const finalProvenance = `${provenance} [${scope}]`;
+
+    entries.push({ url, description, provenance: finalProvenance });
+  }
+
+  const duration = Date.now() - startTime;
+  metrics.observe('knowledge_store_find_urls_duration_ms', duration);
+  metrics.increment('knowledge_store_find_urls_total', 1, { status: 'success', retrieval: 'bm25' });
+  metrics.increment('knowledge_store_urls_found_total', entries.length);
+
+  return entries;
+}
+
 /**
  * Find documents by URL
  */

package/src/knowledge/store-schema.ts

@@ -40,6 +40,38 @@ function createStoreSchema(dim: number, modelName: string): Schema {
   ]));
 }
 
+/**
+ * Schema metadata for BM25 (lexical) tables.
+ *
+ * BM25 tables carry NO vector column at all — rows are stored without any
+ * embedding and ranking comes entirely from the Tantivy FTS inverted indexes
+ * over 'text' and 'content' (computed at query time, nothing score-shaped is
+ * persisted). The schema is otherwise identical to the vector schema, so the
+ * same store operations, scope filters, and result mapping apply.
+ */
+export const BM25_TABLE_NAME = 'knowledge-bm25';
+export const BM25_EMBEDDING_MODEL_TAG = 'bm25-lexical';
+
+/**
+ * Create the BM25 (lexical) store table schema — the vector schema minus the
+ * 'vector' column, tagged so the two table kinds are distinguishable on disk.
+ */
+function createBm25StoreSchema(): Schema {
+  return new Schema([
+    new Field('url', new Utf8(), false),
+    new Field('text', new Utf8(), false),
+    new Field('content', new Utf8(), true), // full page markdown, nullable
+    new Field('metadata', new Utf8(), false), // JSON stringified
+    new Field('workspace', new Utf8(), false), // local workspace path or 'global'
+    new Field('is_global', new Bool(), false), // boolean indicating if it's shared globally
+    new Field('ingestion_type', new Utf8(), false), // e.g. 'synthesis-description'
+    new Field('timestamp', new Int64(), false),
+  ], new Map([
+    ['embedding_model', BM25_EMBEDDING_MODEL_TAG],
+    ['schema_version', CURRENT_SCHEMA_VERSION]
+  ]));
+}
+
 /**
  * Create an empty table with the knowledge store schema
  */
@@ -50,6 +82,32 @@ export async function createStoreTable(
   modelName: string
 ): Promise<lancedb.Table> {
   const schema = createStoreSchema(dim, modelName);
+  return createKnowledgeTable(db, name, schema);
+}
+
+/**
+ * Create an empty BM25 (lexical) table — same indexes as the vector table, no
+ * vector column. Used by retrieval mode 'bm25' (separate table so existing
+ * vector stores are never touched — no migration either way).
+ */
+export async function createBm25StoreTable(
+  db: lancedb.Connection,
+  name: string = BM25_TABLE_NAME
+): Promise<lancedb.Table> {
+  const schema = createBm25StoreSchema();
+  return createKnowledgeTable(db, name, schema);
+}
+
+/**
+ * Shared table creation: FTS indexes for lexical search + btree scalar indexes
+ * for scope/eviction performance. Identical for both table kinds — the ONLY
+ * difference between them is the presence of the 'vector' column.
+ */
+async function createKnowledgeTable(
+  db: lancedb.Connection,
+  name: string,
+  schema: Schema
+): Promise<lancedb.Table> {
   const { Index } = await getLancedb();
 
   // Create empty table with schema and metadata