@lincoln504/pi-research 1.6.18 → 1.6.20

package/dist/cli.mjs

@@ -2389,10 +2389,13 @@ var init_atomic_replace = __esm({
 
 // src/knowledge/availability.ts
 import { createRequire as createRequire3 } from "node:module";
-function probeKnowledgeStoreAvailability(resolver = defaultResolver) {
-  if (cachedIsValidFor(resolver)) return _cached;
+function requiredPackages(retrieval) {
+  return retrieval === "bm25" ? ["@lancedb/lancedb"] : ["@lancedb/lancedb", "@huggingface/transformers"];
+}
+function probeKnowledgeStoreAvailability(retrieval = "vector", resolver = defaultResolver) {
+  if (cachedIsValidFor(retrieval, resolver)) return _cached;
   const missing = [];
-  for (const pkg of REQUIRED_PACKAGES) {
+  for (const pkg of requiredPackages(retrieval)) {
     try {
       resolver(pkg);
     } catch {
@@ -2400,11 +2403,14 @@ function probeKnowledgeStoreAvailability(resolver = defaultResolver) {
     }
   }
   const result = { available: missing.length === 0, missing };
-  if (resolver === defaultResolver) _cached = result;
+  if (resolver === defaultResolver) {
+    _cached = result;
+    _cachedRetrieval = retrieval;
+  }
   return result;
 }
-function cachedIsValidFor(resolver) {
-  return _cached !== null && resolver === defaultResolver;
+function cachedIsValidFor(retrieval, resolver) {
+  return _cached !== null && _cachedRetrieval === retrieval && resolver === defaultResolver;
 }
 function describeKnowledgeStoreUnavailability(availability) {
   if (availability.available) return "";
@@ -2413,12 +2419,12 @@ function describeKnowledgeStoreUnavailability(availability) {
   );
   return `knowledge store is OFF: required package${parts.length > 1 ? "s" : ""} not resolvable \u2014 ${parts.join("; ")}. Repair with a full install of optional dependencies (e.g. 'npm install' including optional deps, or pi install npm:@lincoln504/pi-research without --omit=optional).`;
 }
-var REQUIRED_PACKAGES, _cached, defaultResolver;
+var _cached, _cachedRetrieval, defaultResolver;
 var init_availability = __esm({
   "src/knowledge/availability.ts"() {
     "use strict";
-    REQUIRED_PACKAGES = ["@lancedb/lancedb", "@huggingface/transformers"];
     _cached = null;
+    _cachedRetrieval = null;
     defaultResolver = (specifier) => (
       // import.meta.url survives esbuild's ESM output; createRequire only builds a
       // resolver — it never evaluates the specifier.
@@ -2611,37 +2617,39 @@ function getDbDir(config, cwd = process.cwd()) {
 }
 function describeKnowledgeStoreMode(cwd = process.cwd(), iface) {
   const cfg = getConfig(cwd, iface);
-  const KEY = "PI_RESEARCH_KNOWLEDGE_STORE_MODE";
-  const fileHasKey = (p) => {
+  const fileHasKey = (key, p) => {
     try {
-      return fs5.existsSync(p) && parseDotEnv(fs5.readFileSync(p, "utf-8"))[KEY] !== void 0;
+      return fs5.existsSync(p) && parseDotEnv(fs5.readFileSync(p, "utf-8"))[key] !== void 0;
     } catch {
       return false;
     }
   };
-  let origin = "built-in default";
-  if (process.env[KEY] !== void 0) {
-    origin = "environment variable";
-  } else {
+  const originFor = (KEY) => {
+    if (process.env[KEY] !== void 0) {
+      return "environment variable";
+    }
     let registryHasKey = false;
     try {
       const registry = loadProjectSettingsRegistry();
       registryHasKey = findRegistryEntry(registry, normalizeWorkspacePath(cwd))?.[KEY] !== void 0;
     } catch {
     }
-    if (registryHasKey) origin = "this directory (project settings)";
-    else if (iface && fileHasKey(getInterfaceEnvFilePath(iface))) origin = `${iface}.env overlay`;
-    else if (fileHasKey(getGlobalEnvFilePath())) origin = "config.env (machine-wide default)";
-  }
+    if (registryHasKey) return "this directory (project settings)";
+    if (iface && fileHasKey(KEY, getInterfaceEnvFilePath(iface))) return `${iface}.env overlay`;
+    if (fileHasKey(KEY, getGlobalEnvFilePath())) return "config.env (machine-wide default)";
+    return "built-in default";
+  };
   return {
     mode: cfg.KNOWLEDGE_STORE_MODE,
-    origin,
+    origin: originFor("PI_RESEARCH_KNOWLEDGE_STORE_MODE"),
+    retrieval: cfg.KNOWLEDGE_STORE_RETRIEVAL,
+    retrievalOrigin: originFor("PI_RESEARCH_KNOWLEDGE_STORE_RETRIEVAL"),
     dbDir: getDbDir(cfg, cwd),
-    ...availabilityFields()
+    ...availabilityFields(cfg)
   };
 }
-function availabilityFields() {
-  const availability = probeKnowledgeStoreAvailability();
+function availabilityFields(cfg) {
+  const availability = probeKnowledgeStoreAvailability(cfg?.KNOWLEDGE_STORE_RETRIEVAL ?? getConfig().KNOWLEDGE_STORE_RETRIEVAL);
   return {
     available: availability.available,
     unavailableReason: describeKnowledgeStoreUnavailability(availability)
@@ -2787,6 +2795,7 @@ function saveConfig(config, scope = "local", cwd = process.cwd(), changedKeys) {
     PI_RESEARCH_WORKER_THREADS: String(config.WORKER_THREADS),
     PI_RESEARCH_WORKER_CONCURRENCY: String(config.WORKER_CONCURRENCY),
     PI_RESEARCH_KNOWLEDGE_STORE_MODE: config.KNOWLEDGE_STORE_MODE,
+    PI_RESEARCH_KNOWLEDGE_STORE_RETRIEVAL: config.KNOWLEDGE_STORE_RETRIEVAL,
     PI_RESEARCH_EMBEDDING_MODEL: config.EMBEDDING_MODEL,
     PI_RESEARCH_EMBEDDING_DEVICE: config.EMBEDDING_DEVICE,
     PI_RESEARCH_SCRAPE_TIMEOUT_MS: String(config.SCRAPE_TIMEOUT_MS),
@@ -2947,6 +2956,7 @@ function createConfig(env, processEnv) {
     WORKER_THREADS: parseEnvNumber(e, "PI_RESEARCH_WORKER_THREADS", DEFAULTS.WORKER_THREADS, 1, 10, true),
     WORKER_CONCURRENCY: parseEnvNumber(e, "PI_RESEARCH_WORKER_CONCURRENCY", DEFAULTS.WORKER_CONCURRENCY, 1, 10, true),
     KNOWLEDGE_STORE_MODE: parseEnvEnum(e, "PI_RESEARCH_KNOWLEDGE_STORE_MODE", ["none", "project", "global"], "global"),
+    KNOWLEDGE_STORE_RETRIEVAL: parseEnvEnum(e, "PI_RESEARCH_KNOWLEDGE_STORE_RETRIEVAL", ["vector", "bm25"], "vector"),
     EMBEDDING_MODEL: parseEnvString(e, "PI_RESEARCH_EMBEDDING_MODEL", DEFAULTS.EMBEDDING_MODEL),
     EMBEDDING_DEVICE: parseEnvEnum(e, "PI_RESEARCH_EMBEDDING_DEVICE", ["auto", "webgpu", "cpu"], DEFAULTS.EMBEDDING_DEVICE),
     SCRAPE_TIMEOUT_MS: parseEnvNumber(e, "PI_RESEARCH_SCRAPE_TIMEOUT_MS", DEFAULTS.SCRAPE_TIMEOUT_MS, 5e3, 12e4),
@@ -3082,6 +3092,13 @@ var init_config = __esm({
       WORKER_CONCURRENCY: Type.Number({ minimum: 1, maximum: 10, default: 2 }),
       /** Knowledge store isolation mode (default: 'global' — one shared store across every directory) */
       KNOWLEDGE_STORE_MODE: Type.Union([Type.Literal("none"), Type.Literal("project"), Type.Literal("global")], { default: "global" }),
+      /** Knowledge store retrieval strategy (default: 'vector' — current hybrid dense+sparse behavior).
+       *  'vector' = hybrid search (embedding similarity + BM25/FTS fused via RRF; requires the
+       *  optional @huggingface/transformers dependency and a downloaded embedding model).
+       *  'bm25'   = pure lexical/BM25 retrieval over the same stored findings — NO embedding model:
+       *  never initialized, downloaded, or invoked. Uses a separate lexical table in the same
+       *  LanceDB database, so existing vector stores are untouched (no migration). */
+      KNOWLEDGE_STORE_RETRIEVAL: Type.Union([Type.Literal("vector"), Type.Literal("bm25")], { default: "vector" }),
       /** Embedding model to use for the knowledge store */
       EMBEDDING_MODEL: Type.String({ default: "onnx-community/granite-embedding-small-english-r2-ONNX" }),
       /** Hardware backend for embeddings: 'auto' (probe WebGPU viability out-of-process, fall back to CPU), 'webgpu' (force), or 'cpu' (force).
@@ -3225,7 +3242,8 @@ var init_config = __esm({
     DEFAULTS = Value.Create(ConfigSchema);
     LOCAL_SCOPE_KEYS = /* @__PURE__ */ new Set([
       "PI_RESEARCH_DEFAULT_RESEARCH_DEPTH",
-      "PI_RESEARCH_KNOWLEDGE_STORE_MODE"
+      "PI_RESEARCH_KNOWLEDGE_STORE_MODE",
+      "PI_RESEARCH_KNOWLEDGE_STORE_RETRIEVAL"
     ]);
     USER_MIGRATION_KEYS = [
       // Core user-scoped from schema
@@ -7400,8 +7418,35 @@ function createStoreSchema(dim, modelName) {
     ["schema_version", CURRENT_SCHEMA_VERSION]
   ]));
 }
+function createBm25StoreSchema() {
+  return new Schema([
+    new Field("url", new Utf8(), false),
+    new Field("text", new Utf8(), false),
+    new Field("content", new Utf8(), true),
+    // full page markdown, nullable
+    new Field("metadata", new Utf8(), false),
+    // JSON stringified
+    new Field("workspace", new Utf8(), false),
+    // local workspace path or 'global'
+    new Field("is_global", new Bool(), false),
+    // boolean indicating if it's shared globally
+    new Field("ingestion_type", new Utf8(), false),
+    // e.g. 'synthesis-description'
+    new Field("timestamp", new Int64(), false)
+  ], /* @__PURE__ */ new Map([
+    ["embedding_model", BM25_EMBEDDING_MODEL_TAG],
+    ["schema_version", CURRENT_SCHEMA_VERSION]
+  ]));
+}
 async function createStoreTable(db, name, dim, modelName) {
   const schema = createStoreSchema(dim, modelName);
+  return createKnowledgeTable(db, name, schema);
+}
+async function createBm25StoreTable(db, name = BM25_TABLE_NAME) {
+  const schema = createBm25StoreSchema();
+  return createKnowledgeTable(db, name, schema);
+}
+async function createKnowledgeTable(db, name, schema) {
   const { Index } = await getLancedb();
   const table = await db.createTable({
     name,
@@ -7417,12 +7462,14 @@ async function createStoreTable(db, name, dim, modelName) {
   await table.createIndex("ingestion_type", { config: Index.btree() });
   return table;
 }
-var CURRENT_SCHEMA_VERSION;
+var CURRENT_SCHEMA_VERSION, BM25_TABLE_NAME, BM25_EMBEDDING_MODEL_TAG;
 var init_store_schema = __esm({
   "src/knowledge/store-schema.ts"() {
     "use strict";
     init_lancedb_loader();
     CURRENT_SCHEMA_VERSION = "4";
+    BM25_TABLE_NAME = "knowledge-bm25";
+    BM25_EMBEDDING_MODEL_TAG = "bm25-lexical";
   }
 });
 
@@ -7522,6 +7569,110 @@ async function searchStore(table, embedder, query, getReranker, limit, scopeFilt
   metrics.increment("knowledge_store_search_results_total", filteredResults.length);
   return filteredResults;
 }
+async function addDocumentsBm25ToStore(table, docs, isClosing, workspace, isGlobal) {
+  if (docs.length === 0) return "empty";
+  if (isClosing()) {
+    logger.warn("[store] Ignoring addDocuments during close");
+    return "ignored_closing";
+  }
+  const startTime = Date.now();
+  try {
+    const data = docs.map((doc) => ({
+      url: doc.url,
+      text: doc.text,
+      content: doc.content ?? null,
+      metadata: JSON.stringify(doc.metadata),
+      workspace,
+      is_global: isGlobal,
+      ingestion_type: doc.ingestion_type || doc.metadata["ingestionType"] || "unknown",
+      timestamp: BigInt(doc.timestamp)
+    }));
+    await table.add(data);
+    const duration = Date.now() - startTime;
+    metrics.observe("knowledge_store_add_documents_duration_ms", duration, { retrieval: "bm25" });
+    metrics.increment("knowledge_store_chunks_added_total", docs.length);
+    logger.log(`[store] Added ${docs.length} chunk(s) for ${docs[0]?.url} [retrieval=bm25, workspace=${workspace}, global=${isGlobal}]`);
+    return "written";
+  } catch (err) {
+    const duration = Date.now() - startTime;
+    metrics.observe("knowledge_store_add_documents_duration_ms", duration, { status: "error", retrieval: "bm25" });
+    logger.debug(`[store] bm25 add attempt failed after ${duration}ms: ${err instanceof Error ? err.message : String(err)}`);
+    throw err;
+  }
+}
+async function searchStoreBm25(table, query, limit, scopeFilter) {
+  const startTime = Date.now();
+  const rowCount = scopeFilter ? await table.countRows(scopeFilter) : await table.countRows();
+  if (rowCount === 0) {
+    metrics.increment("knowledge_store_search_total", 1, { status: "empty", retrieval: "bm25" });
+    return [];
+  }
+  let filter = "ingestion_type = 'synthesis-description'";
+  if (scopeFilter) {
+    filter = `(${filter}) AND (${scopeFilter})`;
+  }
+  const results = await table.query().where(filter).fullTextSearch(query, { columns: ["text", "content"] }).limit(limit).toArray();
+  const filteredResults = results.map((r) => {
+    let metadata;
+    try {
+      metadata = JSON.parse(r.metadata);
+    } catch {
+      logger.debug("[store] Corrupted metadata in searchStoreBm25, skipping row");
+      return null;
+    }
+    return {
+      url: r.url,
+      text: r.text,
+      content: r.content ?? void 0,
+      metadata,
+      timestamp: Number(r.timestamp)
+    };
+  }).filter((doc) => doc !== null);
+  const duration = Date.now() - startTime;
+  metrics.observe("knowledge_store_search_duration_ms", duration);
+  metrics.increment("knowledge_store_search_total", 1, { status: "success", retrieval: "bm25" });
+  metrics.increment("knowledge_store_search_results_total", filteredResults.length);
+  return filteredResults;
+}
+async function findRelevantUrlsBm25(table, query, limit, scopeFilter) {
+  const startTime = Date.now();
+  const rowCount = scopeFilter ? await table.countRows(scopeFilter) : await table.countRows();
+  if (rowCount === 0) {
+    metrics.increment("knowledge_store_find_urls_total", 1, { status: "empty", retrieval: "bm25" });
+    return [];
+  }
+  let filter = "ingestion_type = 'synthesis-description'";
+  if (scopeFilter) {
+    filter = `(${filter}) AND (${scopeFilter})`;
+  }
+  const results = await table.query().where(filter).fullTextSearch(query, { columns: ["text", "content"] }).limit(limit).toArray();
+  const entries = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const r of results) {
+    const url = r.url;
+    if (seen.has(url)) continue;
+    seen.add(url);
+    let description = "";
+    let provenance = "description-unverified";
+    try {
+      const meta = JSON.parse(r.metadata);
+      description = meta.description ?? "";
+      if (meta.provenance) provenance = meta.provenance;
+      else if (meta.hasContent === true) provenance = "scraped-verified";
+    } catch {
+      logger.debug("[store] Corrupted metadata in findRelevantUrlsBm25, skipping row");
+    }
+    if (!description) description = (r.text ?? "").substring(0, 300);
+    const scope = r.is_global === true ? "Global Store" : `Local Project (${r.workspace || "unknown"})`;
+    const finalProvenance = `${provenance} [${scope}]`;
+    entries.push({ url, description, provenance: finalProvenance });
+  }
+  const duration = Date.now() - startTime;
+  metrics.observe("knowledge_store_find_urls_duration_ms", duration);
+  metrics.increment("knowledge_store_find_urls_total", 1, { status: "success", retrieval: "bm25" });
+  metrics.increment("knowledge_store_urls_found_total", entries.length);
+  return entries;
+}
 async function findDocumentsByUrl(table, url, scopeFilter) {
   const startTime = Date.now();
   const escapedUrl = url.replace(/'/g, "''");
@@ -7631,7 +7782,12 @@ var init_store = __esm({
       // Coalesces concurrent open() calls (see open()).
       _openPromise = null;
       options;
-      tableName = "knowledge";
+      tableName;
+      // The OTHER retrieval mode's active table name, preserved across manifest
+      // save/load so the two modes can share one dbDir without clobbering each
+      // other's migration-renamed tables ('vector' owns activeTableName,
+      // 'bm25' owns activeBm25TableName).
+      otherModeTableName = null;
       manifestPath;
       isClosing = false;
       activeWrites = /* @__PURE__ */ new Set();
@@ -7660,8 +7816,23 @@ var init_store = __esm({
       });
       constructor(options) {
         this.options = options;
+        this.tableName = this.retrieval === "bm25" ? BM25_TABLE_NAME : "knowledge";
         this.manifestPath = path15.join(this.options.dbDir, "store-manifest.json");
       }
+      /** Normalized retrieval strategy: an unset option means the historical
+       *  'vector' (hybrid) behavior, not bm25. Every mode check goes through this. */
+      get retrieval() {
+        return this.options.retrieval ?? "vector";
+      }
+      /** The embedding backend. Only callable on the vector (hybrid) path — the
+       *  single choke point that makes a null embedder in bm25 mode fail loud here
+       *  instead of surfacing as a mysterious TypeError deep in an embed call. */
+      get embedder() {
+        if (this.retrieval === "bm25" || !this.options.embedder) {
+          throw new Error("[store] embedder accessed in bm25 (lexical) mode \u2014 internal invariant violated");
+        }
+        return this.options.embedder;
+      }
       getScopeFilter() {
         const ws = this.getWorkspace().replace(/'/g, "''");
         switch (this.options.knowledgeMode) {
@@ -7687,8 +7858,12 @@ var init_store = __esm({
           if (fs13.existsSync(this.manifestPath)) {
             const content = await fsPromises.readFile(this.manifestPath, "utf-8");
             const manifest = JSON.parse(content);
-            if (manifest.activeTableName) {
-              this.tableName = manifest.activeTableName;
+            const savedVectorTable = typeof manifest.activeTableName === "string" ? manifest.activeTableName : null;
+            const savedBm25Table = typeof manifest.activeBm25TableName === "string" ? manifest.activeBm25TableName : null;
+            this.otherModeTableName = this.retrieval === "bm25" ? savedVectorTable : savedBm25Table;
+            const savedOwnTable = this.retrieval === "bm25" ? savedBm25Table : savedVectorTable;
+            if (savedOwnTable) {
+              this.tableName = savedOwnTable;
               logger.debug(`[store] Loaded active table name from manifest: ${this.tableName}`);
             }
             if (manifest.lastMigration && typeof manifest.lastMigration.model === "string") {
@@ -7707,7 +7882,11 @@ var init_store = __esm({
         const tempPath = `${this.manifestPath}.tmp`;
         try {
           const content = JSON.stringify({
-            activeTableName: this.tableName,
+            activeTableName: this.retrieval === "vector" ? this.tableName : this.otherModeTableName,
+            // bm25 keeps its OWN manifest key so a lexical store can share the dbDir
+            // with a vector store — neither mode's table renames (migrations,
+            // backups) can strand the other's manifest pointer.
+            activeBm25TableName: this.retrieval === "bm25" ? this.tableName : this.otherModeTableName,
             // Record of the two most recent model/schema migrations. Read back on open
             // to detect ALTERNATION (two configurations sharing this dbDir each
             // migrating the table back and forth) — previously that thrash was
@@ -7732,6 +7911,7 @@ var init_store = __esm({
       }
       async open() {
         if (this.db) return;
+        await this.loadManifest();
         if (this.isClosing) {
           throw new Error("KnowledgeStore.open() called after close() \u2014 construct a fresh store instead.");
         }
@@ -7757,7 +7937,13 @@ var init_store = __esm({
           }
           this.db = await (await getLancedb()).connect(this.options.dbDir);
           const tableNames = await this.db.tableNames();
-          if (tableNames.includes(this.tableName)) {
+          if (this.retrieval === "bm25") {
+            if (tableNames.includes(this.tableName)) {
+              this.table = await this.db.openTable(this.tableName);
+            } else {
+              this.table = await this.createTable();
+            }
+          } else if (tableNames.includes(this.tableName)) {
             this.table = await this.db.openTable(this.tableName);
             let migrationInProgress = false;
             try {
@@ -7776,8 +7962,8 @@ var init_store = __esm({
                 const vectorField = schema.fields.find((f) => f.name === "vector");
                 if (vectorField && vectorField.type.constructor.name === "FixedSizeList") {
                   const dim = vectorField.type.listSize;
-                  if (this.options.embedder.getDimension() === null) {
-                    this.options.embedder.setDimension(dim);
+                  if (this.embedder.getDimension() === null) {
+                    this.embedder.setDimension(dim);
                     logger.debug(`[store] Extracted dimension ${dim} from existing table schema`);
                   }
                 }
@@ -7787,7 +7973,7 @@ var init_store = __esm({
                 logger.warn(`[store] ${reason} detected: ${storedModel} (v${storedVersion}) \u2192 ${this.options.modelName} (v${CURRENT_SCHEMA_VERSION})`);
                 const strategy = this.options.migrationStrategy || "backup";
                 migrationInProgress = true;
-                if (this.options.embedder.getDimension() === null) {
+                if (this.embedder.getDimension() === null) {
                   await this.withEmbedderReconnect((embedder) => embedder.embedMany([" "]));
                 }
                 try {
@@ -7822,7 +8008,7 @@ var init_store = __esm({
               logger.warn("[store] Failed to read table schema:", schemaErr);
             }
           } else {
-            if (this.options.embedder.getDimension() === null) {
+            if (this.embedder.getDimension() === null) {
               await this.withEmbedderReconnect((embedder) => embedder.embedMany([" "]));
             }
             this.table = await this.createTable();
@@ -7934,7 +8120,7 @@ var init_store = __esm({
             });
             const texts = batchDocs.map((d) => d.text);
             const vectors = await this.withEmbedderReconnect((embedder) => embedder.embedMany(texts));
-            assertValidEmbeddingVectors(vectors, batchDocs.length, this.options.embedder.getDimension?.());
+            assertValidEmbeddingVectors(vectors, batchDocs.length, this.embedder.getDimension?.());
             const records = batchDocs.map((doc, idx) => ({
               vector: Array.from(vectors[idx]),
               url: doc.url,
@@ -8014,11 +8200,17 @@ var init_store = __esm({
       }
       async createTable(name = this.tableName) {
         if (!this.db) throw new Error("Database not connected");
-        const dim = this.options.embedder.getDimension();
+        if (this.retrieval === "bm25") {
+          return createBm25StoreTable(this.db, name);
+        }
+        const dim = this.embedder.getDimension();
         if (dim === null) throw new Error("[store] Cannot create table: embedder dimension unknown (not yet initialized)");
         return createStoreTable(this.db, name, dim, this.options.modelName);
       }
       async withEmbedderReconnect(fn) {
+        if (this.retrieval === "bm25" || !this.options.embedder) {
+          throw new Error("[store] withEmbedderReconnect called without an embedder (bm25 mode?) \u2014 internal invariant violated");
+        }
         try {
           return await fn(this.options.embedder);
         } catch (err) {
@@ -8100,17 +8292,26 @@ var init_store = __esm({
           }
           let table = await this.getFreshTable();
           const workspace = this.getWorkspace();
-          const outcome = await this.withEmbedderReconnect(async (embedder) => {
+          const runWriteAttempt = async () => {
             let retryCount = 0;
             const MAX_RETRIES = 5;
             const BASE_DELAY = 100;
             while (retryCount <= MAX_RETRIES) {
               try {
                 const isGlobal = this.options.knowledgeMode === "global";
+                if (this.retrieval === "bm25") {
+                  return await addDocumentsBm25ToStore(
+                    table,
+                    docs,
+                    () => this.isClosing,
+                    workspace,
+                    isGlobal
+                  );
+                }
                 return await addDocumentsToStore(
                   table,
                   docs,
-                  embedder,
+                  this.embedder,
                   () => this.isClosing,
                   workspace,
                   isGlobal
@@ -8129,7 +8330,8 @@ var init_store = __esm({
               }
             }
             throw new Error("[store] addDocuments retry loop exited without result");
-          });
+          };
+          const outcome = this.retrieval === "bm25" ? await runWriteAttempt() : await this.withEmbedderReconnect(runWriteAttempt);
           if (outcome !== "empty") {
             metrics.increment("knowledge_store_add_documents_total", 1, {
               status: outcome === "ignored_closing" ? "ignored_closing" : "success"
@@ -8154,6 +8356,9 @@ var init_store = __esm({
         return this.trackOperation("search", [], async () => {
           const table = await this.getFreshTable();
           const scopeFilter = this.getScopeFilter();
+          if (this.retrieval === "bm25") {
+            return this.circuitBreaker.execute(() => searchStoreBm25(table, query, options.limit ?? 5, scopeFilter));
+          }
           return this.circuitBreaker.execute(
             () => this.withEmbedderReconnect(
               (embedder) => searchStore(table, embedder, query, this.getReranker.bind(this), options.limit ?? 5, scopeFilter)
@@ -8308,8 +8513,9 @@ var init_store = __esm({
               url: r.url,
               // The text is the summary/description
               text: r.text,
-              // The vector is the model-dimension embedding
-              v: Array.from(r.vector),
+              // The vector is the model-dimension embedding — vector/hybrid mode only;
+              // a bm25 store has no vector column, so the export omits it entirely.
+              ...this.retrieval === "vector" ? { v: Array.from(r.vector) } : {},
               // Minimal metadata needed for the UI
               m: {
                 d: metadata["description"] || "",
@@ -8390,6 +8596,9 @@ var init_store = __esm({
         return this.trackOperation("findRelevantUrls", [], async () => {
           const table = await this.getFreshTable();
           const scopeFilter = this.getScopeFilter();
+          if (this.retrieval === "bm25") {
+            return this.circuitBreaker.execute(() => findRelevantUrlsBm25(table, query, options.limit ?? 20, scopeFilter));
+          }
           return this.circuitBreaker.execute(
             () => this.withEmbedderReconnect(
               (embedder) => findRelevantUrls(table, embedder, query, this.getReranker.bind(this), options.limit ?? 20, scopeFilter)
@@ -9078,7 +9287,7 @@ function getMigrationStrategy(config) {
   logger.warn(`[knowledge] Invalid migration strategy '${strategy}'. Valid options: drop, re-embed, backup. Falling back to default (backup).`);
   return void 0;
 }
-async function createKnowledgeStoreComponents(embedderFactory, reconnectFactory, withLock, config = getConfig(), workspace = process.cwd(), availability = probeKnowledgeStoreAvailability()) {
+async function createKnowledgeStoreComponents(embedderFactory, reconnectFactory, withLock, config = getConfig(), workspace = process.cwd(), availability = probeKnowledgeStoreAvailability(config.KNOWLEDGE_STORE_RETRIEVAL)) {
   if (config.KNOWLEDGE_STORE_MODE === "none") {
     logger.debug("[knowledge] Knowledge store is disabled in configuration");
     return null;
@@ -9091,13 +9300,14 @@ async function createKnowledgeStoreComponents(embedderFactory, reconnectFactory,
   for (let attempt = 1; attempt <= MAX_INIT_RETRIES; attempt++) {
     let store;
     try {
-      logger.info(`[knowledge] Creating Knowledge Store components (attempt ${attempt}/${MAX_INIT_RETRIES})...`);
-      const embedder = await embedderFactory();
+      logger.info(`[knowledge] Creating Knowledge Store components (attempt ${attempt}/${MAX_INIT_RETRIES}, retrieval=${config.KNOWLEDGE_STORE_RETRIEVAL})...`);
+      const embedder = config.KNOWLEDGE_STORE_RETRIEVAL === "bm25" ? null : await embedderFactory();
       const migrationStrategy = getMigrationStrategy(config);
       store = new KnowledgeStore({
         dbDir: getDbDir(config, workspace),
         embedder,
         modelName: config.EMBEDDING_MODEL,
+        retrieval: config.KNOWLEDGE_STORE_RETRIEVAL,
         migrationStrategy,
         reconnectFactory,
         withLock,
@@ -9139,7 +9349,9 @@ async function forceDeleteKnowledgeStore(config, workspace) {
   try {
     if (fs14.existsSync(manifestPath)) {
       const manifest = JSON.parse(fs14.readFileSync(manifestPath, "utf-8"));
-      if (manifest.activeTableName) tableName = manifest.activeTableName;
+      const key = config?.KNOWLEDGE_STORE_RETRIEVAL === "bm25" ? "activeBm25TableName" : "activeTableName";
+      const saved = key === "activeBm25TableName" ? manifest.activeBm25TableName : manifest.activeTableName;
+      if (saved) tableName = saved;
     }
   } catch (err) {
     logger.warn(`[knowledge] Failed to read manifest before forced delete, defaulting to table '${tableName}':`, err);
@@ -9209,6 +9421,10 @@ var init_knowledge_store_service = __esm({
       // mode change (project↔global re-scope, or enabled→none) and re-initialize so a /research-config
       // change applies without a Pi restart — not just the DISABLED→enabled revival.
       _initializedMode = void 0;
+      // The retrieval strategy the current build reflects ('vector' hybrid / 'bm25'
+      // lexical) — drives the embedder-optional invariants below and the live
+      // re-init detection on a /research-config retrieval change.
+      _initializedRetrieval = void 0;
       // Initialization promise to prevent concurrent initialization
       _initializationPromise = null;
       // Disposal promise: lets a concurrent initialize() wait out an in-flight
@@ -9224,13 +9440,16 @@ var init_knowledge_store_service = __esm({
         }
         const newCwd = ctx?.cwd || this._cwd;
         if ((this.lifecycle === "initialized" /* INITIALIZED */ || this.lifecycle === "disabled" /* DISABLED */) && this._cwd === newCwd) {
-          const liveMode = (ctx?.config || getConfig(this._cwd)).KNOWLEDGE_STORE_MODE;
+          const liveConfig = ctx?.config || getConfig(this._cwd);
+          const liveMode = liveConfig.KNOWLEDGE_STORE_MODE;
+          const liveRetrieval = liveConfig.KNOWLEDGE_STORE_RETRIEVAL;
           const reviveFromMode = this.lifecycle === "disabled" /* DISABLED */ && this._disabledReason === "mode" && liveMode !== "none";
           const modeChanged = this.lifecycle === "initialized" /* INITIALIZED */ && ctx?.config !== void 0 && this._initializedMode !== void 0 && this._initializedMode !== ctx.config.KNOWLEDGE_STORE_MODE;
-          if (!reviveFromMode && !modeChanged) {
+          const retrievalChanged = this.lifecycle === "initialized" /* INITIALIZED */ && ctx?.config !== void 0 && this._initializedRetrieval !== void 0 && this._initializedRetrieval !== ctx.config.KNOWLEDGE_STORE_RETRIEVAL;
+          if (!reviveFromMode && !modeChanged && !retrievalChanged) {
             return;
           }
-          logger.log(`[KnowledgeStoreService] Knowledge Mode changed (${this._initializedMode ?? this._disabledReason} \u2192 ${liveMode}); re-initializing store (no restart needed).`);
+          logger.log(`[KnowledgeStoreService] Knowledge config changed (${this._initializedMode ?? this._disabledReason} \u2192 ${liveMode}, retrieval ${this._initializedRetrieval ?? "?"} \u2192 ${liveRetrieval}); re-initializing store (no restart needed).`);
           if (this.lifecycle === "initialized" /* INITIALIZED */) {
             await this.dispose();
           }
@@ -9265,6 +9484,7 @@ var init_knowledge_store_service = __esm({
             this._cwd = ctx?.cwd || this._cwd;
             const config = ctx?.config || getConfig(this._cwd);
             this._initializedMode = config.KNOWLEDGE_STORE_MODE;
+            this._initializedRetrieval = config.KNOWLEDGE_STORE_RETRIEVAL;
             const embedderFactory = () => getEmbedder(config);
             const reconnectFactory = async () => {
               await clearEmbeddingInstance();
@@ -9332,7 +9552,7 @@ var init_knowledge_store_service = __esm({
               }
             }
             if (!components) {
-              const depsMissing = !probeKnowledgeStoreAvailability().available;
+              const depsMissing = !probeKnowledgeStoreAvailability(config.KNOWLEDGE_STORE_RETRIEVAL).available;
               logger.debug(`[KnowledgeStoreService] Knowledge store is disabled (${depsMissing ? "dependencies missing" : "mode = none"}). Setting lifecycle to DISABLED.`);
               this.lifecycle = "disabled" /* DISABLED */;
               this._disabledReason = depsMissing ? "native" : "mode";
@@ -9430,6 +9650,9 @@ var init_knowledge_store_service = __esm({
        * Check if the knowledge store is ready
        */
       isReady() {
+        if (this._initializedRetrieval === "bm25") {
+          return this._store !== null && this._writerQueue !== null;
+        }
         return this._embedder !== null && this._store !== null && this._writerQueue !== null;
       }
       /** The working directory this service initialized against (from ctx.cwd). */
@@ -9453,6 +9676,9 @@ var init_knowledge_store_service = __esm({
         if (this.lifecycle === "disabled" /* DISABLED */) {
           return null;
         }
+        if (this._initializedRetrieval === "bm25") {
+          return null;
+        }
         if (!this._embedder) {
           throw new Error("[KnowledgeStoreService] Embedder not initialized");
         }
@@ -18127,7 +18353,21 @@ function registerHealthChecks(registry, container = getServiceContainer()) {
       }
       const store = await service.getStore();
       const counts = store ? await store.countScoped() : { local: 0, global: 0, projects: 0 };
+      const retrieval = getConfig(service.getCwd()).KNOWLEDGE_STORE_RETRIEVAL;
       const embedder = await service.getEmbedder();
+      if (retrieval === "bm25") {
+        return {
+          healthy: true,
+          diagnostic: {
+            status: "initialized",
+            retrieval: "bm25",
+            model: "none (lexical BM25)",
+            localEntries: counts.local,
+            globalEntries: counts.global,
+            totalProjects: counts.projects
+          }
+        };
+      }
       if (!embedder) {
         return { healthy: false, error: "Embedder service not available" };
       }
@@ -18158,6 +18398,7 @@ function registerHealthChecks(registry, container = getServiceContainer()) {
         healthy: true,
         diagnostic: {
           status: "initialized",
+          retrieval,
           device,
           model: config2.EMBEDDING_MODEL,
           localEntries: counts.local,
@@ -27400,6 +27641,29 @@ knowledge store mode: ${cfg.KNOWLEDGE_STORE_MODE}   (default depth: ${cfg.DEFAUL
 async function cmdKnowledgeConfig(kc) {
   const cwd = process.cwd();
   try {
+    if (kc.action === "set-retrieval") {
+      const before = getConfig(cwd, "cli").KNOWLEDGE_STORE_RETRIEVAL;
+      const cfg = { ...getConfig(cwd, "cli"), KNOWLEDGE_STORE_RETRIEVAL: kc.retrieval };
+      saveConfig(cfg, "local", cwd, ["PI_RESEARCH_KNOWLEDGE_STORE_RETRIEVAL"]);
+      resetConfig();
+      const info2 = describeKnowledgeStoreMode(cwd, "cli");
+      const overridden = info2.retrieval !== kc.retrieval;
+      if (kc.json) {
+        toStdout(pretty({ command: "knowledge-config", action: "set-retrieval", requested: kc.retrieval, previous: before, saved: true, effectiveOverriddenBy: overridden ? info2.retrievalOrigin : null, ...info2, cwd }));
+        return EXIT.OK;
+      }
+      toStdout(
+        `knowledge store retrieval for this directory: ${before} -> ${kc.retrieval}  (saved per-directory)
+  strategy: ${kc.retrieval === "bm25" ? "lexical BM25 \u2014 no embedding model needed" : "hybrid (embedding + BM25, fused) \u2014 needs the optional embedding dependency"}
+  saved:    per-directory (${cwd})
+  db dir:   ${info2.dbDir}
+` + (info2.available ? "" : `  state:    ${info2.unavailableReason}
+`) + (overridden ? `
+NOTE: the effective retrieval here is still '${info2.retrieval}' because a higher-precedence ${info2.retrievalOrigin} overrides it. Clear that source for this setting to take effect.
+` : "")
+      );
+      return EXIT.OK;
+    }
     if (kc.action === "set") {
       const before = getConfig(cwd, "cli").KNOWLEDGE_STORE_MODE;
       const cfg = { ...getConfig(cwd, "cli"), KNOWLEDGE_STORE_MODE: kc.mode };
@@ -27430,12 +27694,15 @@ NOTE: the effective mode here is still '${info2.mode}' because a higher-preceden
     }
     toStdout(
       `knowledge store mode: ${info.mode}   (source: ${info.origin})
-  scope:   ${info.mode === "none" ? "disabled here" : info.mode === "project" ? "this directory only" : "shared across all directories"}
-  db dir:  ${info.dbDir}
-` + (info.available ? "" : `  state:   ${info.unavailableReason}
+knowledge store retrieval: ${info.retrieval}   (source: ${info.retrievalOrigin})
+  scope:     ${info.mode === "none" ? "disabled here" : info.mode === "project" ? "this directory only" : "shared across all directories"}
+  strategy:  ${info.retrieval === "bm25" ? "lexical BM25 \u2014 no embedding model needed" : "hybrid (embedding + BM25, fused)"}
+  db dir:    ${info.dbDir}
+` + (info.available ? "" : `  state:     ${info.unavailableReason}
 `) + `
-change it for THIS directory:  ${BINARY_NAME} knowledge-config set <none|project|global>
-change the machine-wide default: set PI_RESEARCH_KNOWLEDGE_STORE_MODE=<mode> in ${resolvedConfigPaths().configEnv}
+change scope for THIS directory:     ${BINARY_NAME} knowledge-config set <none|project|global>
+change retrieval for THIS directory: ${BINARY_NAME} knowledge-config set retrieval <vector|bm25>
+change the machine-wide defaults: set PI_RESEARCH_KNOWLEDGE_STORE_MODE=<mode> and/or PI_RESEARCH_KNOWLEDGE_STORE_RETRIEVAL=<vector|bm25> in ${resolvedConfigPaths().configEnv}
 `
     );
     return EXIT.OK;
@@ -27680,6 +27947,10 @@ COMMANDS
     --json                       Emit a JSON object.
   knowledge-config set <mode>    Set the mode for THIS directory (none | project | global);
                                  persisted per-directory. Takes effect on the next run.
+  knowledge-config set retrieval <vector|bm25>
+                                 Set the retrieval strategy for THIS directory: 'vector'
+                                 (hybrid embedding+BM25, default) or 'bm25' (pure lexical,
+                                 no embedding model). Persisted per-directory.
 
   status                         Show detected config, model/key, and readiness.
     --config <path>              Layer this config file on top (highest file precedence; base
@@ -27713,6 +27984,7 @@ CONFIGURE
 
   Knowledge store \u2014 ON by default in every directory (mode 'global', one shared store).
     per-directory:  ${BINARY_NAME} knowledge-config set <none|project|global>
+                    ${BINARY_NAME} knowledge-config set retrieval <vector|bm25>
     machine-wide:   PI_RESEARCH_KNOWLEDGE_STORE_MODE=<mode>  in ${p.configEnv}
     'project' scopes the store to the current directory; 'none' disables it here.
 
@@ -27814,6 +28086,22 @@ function parseArgs(argv) {
       return out;
     }
     if (action === "set") {
+      if (positional[1] === "retrieval") {
+        const retrieval = positional[2];
+        if (retrieval === void 0) {
+          throw new UsageError("knowledge-config set retrieval requires a strategy: vector | bm25.");
+        }
+        if (retrieval !== "vector" && retrieval !== "bm25") {
+          throw new UsageError(`invalid retrieval strategy "${retrieval}". Use one of: vector | bm25.`);
+        }
+        if (positional.length > 3) {
+          throw new UsageError(`unexpected argument "${positional[3]}" after "knowledge-config set retrieval ${retrieval}".`);
+        }
+        out.command = "knowledge-config";
+        out.knowledgeConfig = { action: "set-retrieval", retrieval, json };
+        out.configPath = configPath;
+        return out;
+      }
       const mode = positional[1];
       if (mode === void 0) {
         throw new UsageError("knowledge-config set requires a mode: none | project | global.");
@@ -27829,7 +28117,7 @@ function parseArgs(argv) {
       out.configPath = configPath;
       return out;
     }
-    throw new UsageError(`unknown knowledge-config action "${action}". Use: show | set <none|project|global>.`);
+    throw new UsageError(`unknown knowledge-config action "${action}". Use: show | set <none|project|global> | set retrieval <vector|bm25>.`);
   }
   if (cmd === "skill") {
     let json = false;