@lincoln504/pi-research 1.6.19 → 1.6.20

package/src/knowledge/availability.ts

@@ -46,17 +46,36 @@ export interface KnowledgeStoreAvailability {
 }
 
 /**
- * The packages the store cannot run without. `@lancedb/lancedb` is the vector
- * store itself; `@huggingface/transformers` is the only embedding backend
- * (see transformers-loader.ts — there is no remote-embedding fallback).
+ * The packages the store cannot run without, per retrieval mode.
+ *
+ * Every mode needs `@lancedb/lancedb` — it backs the store in ALL modes (its
+ * native FTS/Tantivy engine provides BM25 ranking in lexical mode too).
+ *
+ * Only the 'vector' (hybrid) retrieval mode additionally needs
+ * `@huggingface/transformers` (the only embedding backend — see
+ * transformers-loader.ts, there is no remote-embedding fallback). Selecting
+ * retrieval='bm25' is a HARD dependency boundary: the embedding package is
+ * never resolved, loaded, or downloaded in lexical mode, so a bm25-only host
+ * (e.g. an install with --omit=optional) runs the store fully.
  */
-const REQUIRED_PACKAGES: readonly string[] = ['@lancedb/lancedb', '@huggingface/transformers'];
+function requiredPackages(retrieval: KnowledgeStoreRetrieval): readonly string[] {
+  return retrieval === 'bm25'
+    ? ['@lancedb/lancedb']
+    : ['@lancedb/lancedb', '@huggingface/transformers'];
+}
+
+/** Knowledge store retrieval strategy. 'vector' = hybrid dense+sparse (default,
+ *  embedding-backed); 'bm25' = pure lexical, no embedding model. */
+export type KnowledgeStoreRetrieval = 'vector' | 'bm25';
 
 // Memoized per-process: resolution touches the filesystem on every miss, the
 // answer cannot change within a live process without a reinstall mid-flight,
 // and the probe sits on settings/health paths that run often. Tests reset it
 // via clearAvailabilityCache.
 let _cached: KnowledgeStoreAvailability | null = null;
+// The retrieval mode the cached probe answer was computed for — the cache must
+// be keyed on it because the required-package set differs between modes.
+let _cachedRetrieval: KnowledgeStoreRetrieval | null = null;
 
 /** Resolver signature — injectable so tests never need the real filesystem. */
 export type PackageResolver = (specifier: string) => string;
@@ -75,10 +94,10 @@ const defaultResolver: PackageResolver = (specifier) =>
  * Probe whether the knowledge store's required packages are resolvable.
  * Memoized after the first call; {@link clearAvailabilityCache} resets.
  */
-export function probeKnowledgeStoreAvailability(resolver: PackageResolver = defaultResolver): KnowledgeStoreAvailability {
-  if (cachedIsValidFor(resolver)) return _cached!;
+export function probeKnowledgeStoreAvailability(retrieval: KnowledgeStoreRetrieval = 'vector', resolver: PackageResolver = defaultResolver): KnowledgeStoreAvailability {
+  if (cachedIsValidFor(retrieval, resolver)) return _cached!;
   const missing: string[] = [];
-  for (const pkg of REQUIRED_PACKAGES) {
+  for (const pkg of requiredPackages(retrieval)) {
     try {
       resolver(pkg);
     } catch {
@@ -88,17 +107,21 @@ export function probeKnowledgeStoreAvailability(resolver: PackageResolver = defa
     }
   }
   const result: KnowledgeStoreAvailability = { available: missing.length === 0, missing };
-  if (resolver === defaultResolver) _cached = result;
+  if (resolver === defaultResolver) {
+    _cached = result;
+    _cachedRetrieval = retrieval;
+  }
   return result;
 }
 
-function cachedIsValidFor(resolver: PackageResolver): boolean {
-  return _cached !== null && resolver === defaultResolver;
+function cachedIsValidFor(retrieval: KnowledgeStoreRetrieval, resolver: PackageResolver): boolean {
+  return _cached !== null && _cachedRetrieval === retrieval && resolver === defaultResolver;
 }
 
 /** Reset the memoized probe result (test isolation). */
 export function clearAvailabilityCache(): void {
   _cached = null;
+  _cachedRetrieval = null;
 }
 
 /**

package/src/knowledge/index.ts

@@ -33,7 +33,9 @@ function getMigrationStrategy(config: Config): MigrationStrategy | undefined {
  * Result of knowledge store initialization
  */
 export interface KnowledgeStoreComponents {
-  embedder: IEmbedder;
+  /** null in bm25 (lexical) retrieval mode — the store runs with NO embedding
+   *  model; non-null in 'vector' (hybrid) mode. */
+  embedder: IEmbedder | null;
   store: IKnowledgeStore;
   writerQueue: WriterQueue;
 }
@@ -51,7 +53,7 @@ export async function createKnowledgeStoreComponents(
   withLock?: <T>(fn: () => Promise<T>) => Promise<T>,
   config: Config = getConfig(),
   workspace: string = process.cwd(),
-  availability: KnowledgeStoreAvailability = probeKnowledgeStoreAvailability()
+  availability: KnowledgeStoreAvailability = probeKnowledgeStoreAvailability(config.KNOWLEDGE_STORE_RETRIEVAL)
 ): Promise<KnowledgeStoreComponents | null> {
   if (config.KNOWLEDGE_STORE_MODE === 'none') {
     logger.debug('[knowledge] Knowledge store is disabled in configuration');
@@ -76,15 +78,20 @@ export async function createKnowledgeStoreComponents(
   for (let attempt = 1; attempt <= MAX_INIT_RETRIES; attempt++) {
     let store: KnowledgeStore | undefined;
     try {
-      logger.info(`[knowledge] Creating Knowledge Store components (attempt ${attempt}/${MAX_INIT_RETRIES})...`);
+      logger.info(`[knowledge] Creating Knowledge Store components (attempt ${attempt}/${MAX_INIT_RETRIES}, retrieval=${config.KNOWLEDGE_STORE_RETRIEVAL})...`);
 
-      const embedder = await embedderFactory();
+      // Hard dependency boundary: in bm25 (lexical) retrieval mode the embedder
+      // factory is NEVER invoked — no @huggingface/transformers resolution, no
+      // model download, no initialization, no inference. The store runs purely
+      // on LanceDB's FTS/BM25 engine.
+      const embedder = config.KNOWLEDGE_STORE_RETRIEVAL === 'bm25' ? null : await embedderFactory();
       const migrationStrategy = getMigrationStrategy(config);
 
       store = new KnowledgeStore({
         dbDir: getDbDir(config, workspace),
         embedder,
         modelName: config.EMBEDDING_MODEL,
+        retrieval: config.KNOWLEDGE_STORE_RETRIEVAL,
         migrationStrategy: migrationStrategy,
         reconnectFactory,
         withLock,
@@ -149,8 +156,12 @@ export async function forceDeleteKnowledgeStore(config?: Config, workspace?: str
   const manifestPath = path.join(dbDir, 'store-manifest.json');
   try {
     if (fs.existsSync(manifestPath)) {
-      const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8')) as { activeTableName?: string };
-      if (manifest.activeTableName) tableName = manifest.activeTableName;
+      const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8')) as { activeTableName?: string; activeBm25TableName?: string };
+      // Read the manifest key that belongs to the CONFIGURED retrieval mode —
+      // each mode owns its own key so a shared dbDir never cross-wires them.
+      const key = config?.KNOWLEDGE_STORE_RETRIEVAL === 'bm25' ? 'activeBm25TableName' : 'activeTableName';
+      const saved = key === 'activeBm25TableName' ? manifest.activeBm25TableName : manifest.activeTableName;
+      if (saved) tableName = saved;
     }
   } catch (err) {
     logger.warn(`[knowledge] Failed to read manifest before forced delete, defaulting to table '${tableName}':`, err);

package/src/knowledge/store-operations.ts

@@ -188,6 +188,183 @@ export async function searchStore(
   return filteredResults;
 }
 
+/**
+ * Add documents to the BM25 (lexical) table — the vector write path minus the
+ * embedding step. Rows carry no vector at all; lexical ranking comes from the
+ * Tantivy FTS indexes (see store-schema.ts). Everything else — row shape,
+ * scope columns, ingestion_type fallback, metrics — matches addDocumentsToStore
+ * so the two backends stay interchangeable behind IKnowledgeStore.
+ */
+export async function addDocumentsBm25ToStore(
+  table: lancedb.Table,
+  docs: StoreDocument[],
+  isClosing: () => boolean,
+  workspace: string,
+  isGlobal: boolean
+): Promise<AddDocumentsOutcome> {
+  if (docs.length === 0) return 'empty';
+  if (isClosing()) {
+    logger.warn('[store] Ignoring addDocuments during close');
+    return 'ignored_closing';
+  }
+
+  const startTime = Date.now();
+
+  try {
+    const data = docs.map((doc) => ({
+      url: doc.url,
+      text: doc.text,
+      content: doc.content ?? null,
+      metadata: JSON.stringify(doc.metadata),
+      workspace,
+      is_global: isGlobal,
+      ingestion_type: doc.ingestion_type || (doc.metadata['ingestionType'] as string) || 'unknown',
+      timestamp: BigInt(doc.timestamp),
+    }));
+
+    await table.add(data);
+    const duration = Date.now() - startTime;
+    metrics.observe('knowledge_store_add_documents_duration_ms', duration, { retrieval: 'bm25' });
+    metrics.increment('knowledge_store_chunks_added_total', docs.length);
+    logger.log(`[store] Added ${docs.length} chunk(s) for ${docs[0]?.url} [retrieval=bm25, workspace=${workspace}, global=${isGlobal}]`);
+    return 'written';
+  } catch (err) {
+    const duration = Date.now() - startTime;
+    metrics.observe('knowledge_store_add_documents_duration_ms', duration, { status: 'error', retrieval: 'bm25' });
+    // Same accounting posture as the vector path: attempt-level failures are
+    // thrown; outcome accounting belongs to KnowledgeStore.addDocuments.
+    logger.debug(`[store] bm25 add attempt failed after ${duration}ms: ${err instanceof Error ? err.message : String(err)}`);
+    throw err;
+  }
+}
+
+/**
+ * Search the BM25 (lexical) table — pure full-text retrieval, no embedding.
+ *
+ * Runs the SAME lexical half the hybrid path already uses: Tantivy BM25 over
+ * BOTH 'text' (synthesis descriptions) and 'content' (full article body),
+ * ranked by BM25 score alone (no nearestTo, no RRF fusion — there is no second
+ * signal to fuse). Filters, result mapping, and metrics match searchStore so
+ * callers see identical StoreDocument results either way.
+ */
+export async function searchStoreBm25(
+  table: lancedb.Table,
+  query: string,
+  limit: number,
+  scopeFilter?: string
+): Promise<StoreDocument[]> {
+  const startTime = Date.now();
+
+  const rowCount = scopeFilter ? await table.countRows(scopeFilter) : await table.countRows();
+  if (rowCount === 0) {
+    metrics.increment('knowledge_store_search_total', 1, { status: 'empty', retrieval: 'bm25' });
+    return [];
+  }
+
+  let filter = "ingestion_type = 'synthesis-description'";
+  if (scopeFilter) {
+    filter = `(${filter}) AND (${scopeFilter})`;
+  }
+
+  // Pure FTS query: no nearestTo() — a Query (not a VectorQuery) ranked by BM25.
+  const results = await table
+    .query()
+    .where(filter)
+    .fullTextSearch(query, { columns: ['text', 'content'] })
+    .limit(limit)
+    .toArray();
+
+  const filteredResults = results
+    .map(r => {
+      let metadata: Record<string, unknown>;
+      try {
+        metadata = JSON.parse(r.metadata as string) as Record<string, unknown>;
+      } catch {
+        logger.debug('[store] Corrupted metadata in searchStoreBm25, skipping row');
+        return null;
+      }
+      return {
+        url: r.url as string,
+        text: r.text as string,
+        content: (r.content as string | null) ?? undefined,
+        metadata,
+        timestamp: Number(r.timestamp),
+      };
+    })
+    .filter((doc): doc is NonNullable<typeof doc> => doc !== null);
+
+  const duration = Date.now() - startTime;
+  metrics.observe('knowledge_store_search_duration_ms', duration);
+  metrics.increment('knowledge_store_search_total', 1, { status: 'success', retrieval: 'bm25' });
+  metrics.increment('knowledge_store_search_results_total', filteredResults.length);
+
+  return filteredResults;
+}
+
+/**
+ * Find relevant URLs in the BM25 (lexical) table — the vector variant minus
+ * the embedding step: BM25 ranking over 'text' + 'content', same filters,
+ * same provenance decoration, same result shape as findRelevantUrls.
+ */
+export async function findRelevantUrlsBm25(
+  table: lancedb.Table,
+  query: string,
+  limit: number,
+  scopeFilter?: string
+): Promise<{ url: string; description: string; provenance?: string }[]> {
+  const startTime = Date.now();
+  const rowCount = scopeFilter ? await table.countRows(scopeFilter) : await table.countRows();
+  if (rowCount === 0) {
+    metrics.increment('knowledge_store_find_urls_total', 1, { status: 'empty', retrieval: 'bm25' });
+    return [];
+  }
+
+  let filter = "ingestion_type = 'synthesis-description'";
+  if (scopeFilter) {
+    filter = `(${filter}) AND (${scopeFilter})`;
+  }
+
+  const results = await table
+    .query()
+    .where(filter)
+    .fullTextSearch(query, { columns: ['text', 'content'] })
+    .limit(limit)
+    .toArray();
+
+  const entries: { url: string; description: string; provenance?: string }[] = [];
+  const seen = new Set<string>();
+  for (const r of results) {
+    const url = r.url as string;
+    if (seen.has(url)) continue;
+    seen.add(url);
+
+    let description = '';
+    let provenance = 'description-unverified';
+    try {
+      const meta = JSON.parse(r.metadata as string);
+      description = meta.description as string ?? '';
+      // Surface provenance metadata so consumers can prefer verified entries
+      if (meta.provenance) provenance = meta.provenance;
+      else if (meta.hasContent === true) provenance = 'scraped-verified';
+    } catch { logger.debug('[store] Corrupted metadata in findRelevantUrlsBm25, skipping row'); }
+
+    if (!description) description = (r.text as string ?? '').substring(0, 300);
+
+    // Add store scope to provenance
+    const scope = r.is_global === true ? 'Global Store' : `Local Project (${r.workspace || 'unknown'})`;
+    const finalProvenance = `${provenance} [${scope}]`;
+
+    entries.push({ url, description, provenance: finalProvenance });
+  }
+
+  const duration = Date.now() - startTime;
+  metrics.observe('knowledge_store_find_urls_duration_ms', duration);
+  metrics.increment('knowledge_store_find_urls_total', 1, { status: 'success', retrieval: 'bm25' });
+  metrics.increment('knowledge_store_urls_found_total', entries.length);
+
+  return entries;
+}
+
 /**
  * Find documents by URL
  */

package/src/knowledge/store-schema.ts

@@ -40,6 +40,38 @@ function createStoreSchema(dim: number, modelName: string): Schema {
   ]));
 }
 
+/**
+ * Schema metadata for BM25 (lexical) tables.
+ *
+ * BM25 tables carry NO vector column at all — rows are stored without any
+ * embedding and ranking comes entirely from the Tantivy FTS inverted indexes
+ * over 'text' and 'content' (computed at query time, nothing score-shaped is
+ * persisted). The schema is otherwise identical to the vector schema, so the
+ * same store operations, scope filters, and result mapping apply.
+ */
+export const BM25_TABLE_NAME = 'knowledge-bm25';
+export const BM25_EMBEDDING_MODEL_TAG = 'bm25-lexical';
+
+/**
+ * Create the BM25 (lexical) store table schema — the vector schema minus the
+ * 'vector' column, tagged so the two table kinds are distinguishable on disk.
+ */
+function createBm25StoreSchema(): Schema {
+  return new Schema([
+    new Field('url', new Utf8(), false),
+    new Field('text', new Utf8(), false),
+    new Field('content', new Utf8(), true), // full page markdown, nullable
+    new Field('metadata', new Utf8(), false), // JSON stringified
+    new Field('workspace', new Utf8(), false), // local workspace path or 'global'
+    new Field('is_global', new Bool(), false), // boolean indicating if it's shared globally
+    new Field('ingestion_type', new Utf8(), false), // e.g. 'synthesis-description'
+    new Field('timestamp', new Int64(), false),
+  ], new Map([
+    ['embedding_model', BM25_EMBEDDING_MODEL_TAG],
+    ['schema_version', CURRENT_SCHEMA_VERSION]
+  ]));
+}
+
 /**
  * Create an empty table with the knowledge store schema
  */
@@ -50,6 +82,32 @@ export async function createStoreTable(
   modelName: string
 ): Promise<lancedb.Table> {
   const schema = createStoreSchema(dim, modelName);
+  return createKnowledgeTable(db, name, schema);
+}
+
+/**
+ * Create an empty BM25 (lexical) table — same indexes as the vector table, no
+ * vector column. Used by retrieval mode 'bm25' (separate table so existing
+ * vector stores are never touched — no migration either way).
+ */
+export async function createBm25StoreTable(
+  db: lancedb.Connection,
+  name: string = BM25_TABLE_NAME
+): Promise<lancedb.Table> {
+  const schema = createBm25StoreSchema();
+  return createKnowledgeTable(db, name, schema);
+}
+
+/**
+ * Shared table creation: FTS indexes for lexical search + btree scalar indexes
+ * for scope/eviction performance. Identical for both table kinds — the ONLY
+ * difference between them is the presence of the 'vector' column.
+ */
+async function createKnowledgeTable(
+  db: lancedb.Connection,
+  name: string,
+  schema: Schema
+): Promise<lancedb.Table> {
   const { Index } = await getLancedb();
 
   // Create empty table with schema and metadata