knowledge-rag 4.8.3 → 4.8.4

package/README.md

@@ -230,19 +230,32 @@ See [Changelog](#changelog) for full history.
 flowchart TB
     subgraph MCP["MCP SERVER (FastMCP)"]
         direction TB
-        TOOLS["13 MCP Tools<br/>search | get | add | update | remove<br/>reindex | reindex_status | list | stats | url | similar | evaluate"]
+        TOOLS["13 MCP Tools (frozen, LEI 1)<br/>search_knowledge | get_document | search_similar<br/>add_document | add_from_url | update_document | remove_document<br/>reindex_documents | get_reindex_status<br/>list_categories | list_documents | get_index_stats | evaluate_retrieval"]
     end
 
-    subgraph SEARCH["HYBRID SEARCH ENGINE"]
+    subgraph SEARCH["SEARCH DISPATCH (v4.8.2+)"]
+        direction TB
+        SM["search_method dispatch<br/>auto | hybrid | fts5"]
+        QROUTER["QueryRouter (regex)<br/>lexical vs semantic<br/>(ADR-002)"]
+        FTS5PATH["FTS5 Fast-Path<br/>(exact identifiers <10ms)"]
+        HYBRIDPATH["Hybrid Pipeline<br/>(conceptual queries)"]
+
+        SM --> QROUTER
+        QROUTER -->|lexical| FTS5PATH
+        QROUTER -->|semantic| HYBRIDPATH
+        FTS5PATH -.->|not_ready / low_hits<br/>fallback| HYBRIDPATH
+    end
+
+    subgraph HYBRID["HYBRID PIPELINE"]
         direction LR
-        ROUTER["Keyword Router<br/>(word boundaries)"]
+        KWROUTER["Keyword Router<br/>(word boundaries → category filter)"]
         SEMANTIC["Semantic Search<br/>(ChromaDB)"]
         BM25["BM25 Keyword<br/>(inverted-index + expansion)"]
         RRF["Reciprocal Rank<br/>Fusion (RRF)"]
         RERANK["Cross-Encoder<br/>Reranker"]
 
-        ROUTER --> SEMANTIC
-        ROUTER --> BM25
+        KWROUTER --> SEMANTIC
+        KWROUTER --> BM25
         SEMANTIC --> RRF
         BM25 --> RRF
         RRF --> RERANK
@@ -250,9 +263,11 @@ flowchart TB
 
     subgraph STORAGE["STORAGE LAYER"]
         direction LR
-        CHROMA[("ChromaDB<br/>Vector Database")]
-        COLLECTIONS["Collections<br/>security | ctf<br/>logscale | development"]
+        CHROMA[("ChromaDB<br/>data/chroma_db/<br/>(vector store)")]
+        FTS5DB[("SQLite FTS5<br/>data/fts5_index.db<br/>(lexical index, ADR-001)")]
+        COLLECTIONS["8 Categories<br/>redteam | blueteam | ctf<br/>security | logscale<br/>development | aar | general"]
         CHROMA --- COLLECTIONS
+        FTS5DB --- COLLECTIONS
     end
 
     subgraph EMBED["EMBEDDINGS (In-Process)"]
@@ -264,36 +279,55 @@ flowchart TB
     subgraph INGEST["DOCUMENT INGESTION"]
         PARSERS["20 Parsers<br/>MD | PDF | TXT | PY | C | H | CPP | JS | JSX | TS | TSX | JSON | XML | CSV<br/>DOCX | XLSX | PPTX | IPYNB | MQH | MQ4"]
         CHUNKER["Chunking<br/>MD: section-aware<br/>Other: 1000 chars + 200 overlap"]
-        PARSERS --> CHUNKER
+        FTS5SYNC["FTS5 CRUD sync<br/>(v4.8.2 task 05)"]
+        PARSERS --> CHUNKER --> FTS5SYNC
     end
 
     CLAUDE["Claude Code"] --> MCP
     MCP --> SEARCH
-    SEARCH --> STORAGE
+    HYBRIDPATH --> HYBRID
+    FTS5PATH --> FTS5DB
+    HYBRID --> STORAGE
     STORAGE --> EMBED
     INGEST --> EMBED
     EMBED --> STORAGE
+    FTS5SYNC --> FTS5DB
 ```
 
 ### Query Processing Flow
 
 ```mermaid
 flowchart TB
-    QUERY["User Query<br/>'mimikatz credential dump'"] --> EXPAND
+    QUERY["User Query<br/>'mimikatz credential dump' | 'CVE-2021-4034'"] --> METHOD
+
+    subgraph DISPATCH["Dispatch (v4.8.2+, ADR-006)"]
+        METHOD{"search_method<br/>auto | hybrid | fts5"}
+        AUTOROUTE["QueryRouter.classify()<br/>regex-based lexical detection"]
+        FASTPATH["FTS5 Fast-Path<br/>SQLite MATCH<br/>(&lt;10ms cold, &lt;2ms hot)"]
+        NOTREADY{"FTS5 ready<br/>AND hits >= min_hits?"}
+
+        METHOD -->|auto| AUTOROUTE
+        METHOD -->|fts5| FASTPATH
+        METHOD -->|hybrid| EXPAND
+        AUTOROUTE -->|lexical| FASTPATH
+        AUTOROUTE -->|semantic| EXPAND
+        FASTPATH --> NOTREADY
+        NOTREADY -->|no, fallback| EXPAND
+    end
 
-    subgraph EXPANSION["Query Expansion"]
+    subgraph EXPANSION["Query Expansion (hybrid path)"]
         EXPAND["Synonym Expansion<br/>mimikatz -> mimikatz, sekurlsa, logonpasswords"]
     end
 
-    EXPAND --> ROUTER
+    EXPAND --> KWROUTER
 
-    subgraph ROUTING["Keyword Routing"]
-        ROUTER["Keyword Router"]
+    subgraph ROUTING["Keyword Routing (category filter)"]
+        KWROUTER["Keyword Router<br/>(word boundaries)"]
         MATCH{"Word Boundary<br/>Match?"}
         CATEGORY["Filter: redteam"]
         NOFILTER["No Filter"]
 
-        ROUTER --> MATCH
+        KWROUTER --> MATCH
         MATCH -->|Yes| CATEGORY
         MATCH -->|No| NOFILTER
     end
@@ -326,7 +360,8 @@ flowchart TB
     BM25 --> RRF
 
     ADJ --> MINSCORE
-    SNIPPET --> RESULTS["Results<br/>search_method: hybrid|semantic|keyword<br/>score + filtered_by_score + content_length"]
+    NOTREADY -->|yes, hit| MINSCORE
+    SNIPPET --> RESULTS["Results<br/>search_method: fts5 | hybrid | semantic | keyword<br/>routed_by: fts5_router | none<br/>score + filtered_by_score + content_length"]
 ```
 
 ### Document Ingestion Flow
@@ -334,7 +369,7 @@ flowchart TB
 ```mermaid
 flowchart LR
     subgraph INPUT["Input"]
-        FILES["documents/<br/>├── security/<br/>├── development/<br/>├── ctf/<br/>└── general/"]
+        FILES["documents/<br/>├── security/<br/>│   ├── redteam/<br/>│   ├── blueteam/<br/>│   └── ctf/<br/>├── aar/<br/>├── logscale/<br/>├── development/<br/>└── general/"]
     end
 
     subgraph PARSE["Parse (20 formats)"]
@@ -354,9 +389,18 @@ flowchart LR
         FASTEMBED["FastEmbed ONNX<br/>bge-small-en-v1.5<br/>(384D, CPU or GPU)"]
     end
 
+    subgraph DISPATCH["Write Dispatch (v4.8.3, #161)"]
+        WCOL{"_write_collection<br/>routes writes"}
+        STAGING["Staging Collection<br/>(nuclear_rebuild only,<br/>keeps prod serving queries)"]
+        PROD["Production Collection<br/>(default)"]
+        WCOL -->|_staging_target set| STAGING
+        WCOL -->|default| PROD
+    end
+
     subgraph STORE["Store"]
-        CHROMADB[("ChromaDB")]
-        BM25IDX["BM25 Index"]
+        CHROMADB[("ChromaDB<br/>data/chroma_db/")]
+        BM25IDX["BM25 Index<br/>(in-memory,<br/>rebuilt on write)"]
+        FTS5DB[("SQLite FTS5<br/>data/fts5_index.db<br/>(CRUD-synced, ADR-008)")]
     end
 
     FILES --> MD & PDF & OFFICE & CODE
@@ -365,14 +409,20 @@ flowchart LR
     MDSPLIT --> DEDUP
     TXTSPLIT --> DEDUP
     DEDUP --> EMBED
-    EMBED --> STORE
+    EMBED --> DISPATCH
+    STAGING --> CHROMADB
+    PROD --> CHROMADB
+    PROD --> BM25IDX
+    PROD --> FTS5DB
 ```
 
 ### hybrid_alpha Parameter Effect
 
+`hybrid_alpha` weights RRF fusion between semantic and BM25 rankings on the **hybrid pipeline only**. When `search_method="auto"` and the QueryRouter classifies the query as lexical (e.g. `CVE-2021-4034`, `MDR-AD002`, `T1078.001`, file hashes), the FTS5 fast-path fires and `hybrid_alpha` is not consulted — FTS5 uses SQLite's native `bm25()` scoring exclusively. Pass `search_method="hybrid"` to force RRF fusion + rerank on every query if you want deterministic hybrid semantics regardless of the query shape.
+
 ```mermaid
 flowchart LR
-    subgraph ALPHA["hybrid_alpha values"]
+    subgraph ALPHA["hybrid_alpha values (hybrid path only)"]
         A0["0.0<br/>Pure BM25<br/>Instant"]
         A3["0.3 (default)<br/>Keyword-heavy<br/>Fast"]
         A5["0.5<br/>Balanced"]
@@ -381,7 +431,7 @@ flowchart LR
     end
 
     subgraph USE["Best For"]
-        U0["CVEs, tool names<br/>exact matches"]
+        U0["CVEs, tool names<br/>exact matches<br/>(consider FTS5 fast-path)"]
         U3["Technical queries<br/>specific terms"]
         U5["General queries"]
         U7["Conceptual queries<br/>related topics"]
@@ -1435,6 +1485,23 @@ Common issues:
 
 ### Unreleased
 
+### v4.8.4 (2026-08-13) — Patch: security + durability + defensive fixes
+
+**Recommended upgrade for anyone running HTTP/SSE transport with bearer auth on a non-loopback host, or relying on `reindex_documents(force=True)`.** Four independent fixes accumulated since v4.8.3: one previously blocked remote-with-auth deployments (#174), one silently corrupted durable index metadata after a failed rebuild (#173), and two defensive fixes hardened `.ipynb` ingestion (#175) and the nightly chaos suite (#164).
+
+**Fixed:**
+
+- **fix(security)** — create the HTTP MCP app with the configured `server_host`. On the bearer-auth path the app factory was called without a host, so it defaulted to `127.0.0.1` and MCP 2.0.0's localhost-only DNS-rebinding allowlist rejected a configured non-loopback host with `421 Invalid Host header` before auth ran — even though uvicorn was bound to that host. Remote/LAN deployments with auth are now reachable (#174).
+- **fix(reindex)** — restore durable metadata on a failed zero-downtime rebuild. The staging populate path persisted the staging map to `index_metadata.json` before validation ran; when validation failed, rollback restored only the in-memory `_indexed_docs`, leaving the file describing staging while memory held production. After a restart `_load_metadata` trusted the stale file, corrupting skip/update and embedding-dimension decisions. Rollback now re-persists the restored production metadata (#173).
+- **fix(ingestion)** — `_parse_ipynb` handled invalid JSON gracefully but then assumed the parsed notebook was a well-formed dict. A valid-JSON notebook with a null `metadata` or `cells` field, a non-dict cell, a bare list, or a dict cell whose `source` is not `str | list[str]` (a number, a dict, or a list with non-string items) crashed with `AttributeError`/`TypeError`. Guarded the notebook, `metadata`, `cells`, cell, and cell `source` lookups so a parseable-but-malformed notebook is handled the same as any other odd input (#175, thanks @eeshsaxena).
+- **fix(ci)** — the nightly chaos job (`pytest -m chaos`) aborted collection with `ModuleNotFoundError: No module named 'hypothesis'` because `tests/test_query_router.py` imported hypothesis at module top-level while the chaos job installs only `pytest psutil numpy`. Guarded the import with `pytest.importorskip("hypothesis")` so the module is skipped cleanly instead of failing the whole nightly run (#164).
+
+**Docs:**
+
+- **docs(architecture)** — sync the 4 Mermaid diagrams in `## Architecture` (System Overview, Query Processing Flow, Document Ingestion Flow, hybrid_alpha Parameter Effect) with the v4.8.2 FTS5 fast-path (ADR-002/003/006), the v4.8.3 `_write_collection` staging dispatcher (#161), and the current 8-category layout. The diagrams previously described only the pre-v4.8 hybrid pipeline and 4-category storage; they now also show FTS5 dispatch, fallback semantics, FTS5 CRUD sync (ADR-008), and the zero-downtime staging write path. `hybrid_alpha` now clarifies that the fast-path bypasses RRF entirely so the weighting only matters on the hybrid path.
+
+**No breaking changes.** Backwards compat verified by `check_api_surface.py` (zero diff vs baseline). All 13 MCP tool parameter names frozen (verified by `tests/test_backwards_compat.py`).
+
 ### v4.8.3 (2026-08-10) — Critical hotfix: nuclear-rebuild + smart-reindex hardening
 
 **Recommended upgrade for any corpus with >1000 chunks or any user relying on `reindex_documents(force=True)` after a config change.** The v4.8.0 nuclear rebuild + v4.8.2 FTS5 fast-path have several latent state-management bugs that only trigger under a real-world combination (large corpus + rebuild + concurrent queries + FTS5 opt-in). Enterprise users with hacktricks-scale corpora (~50k+ chunks) are affected.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "knowledge-rag",
-  "version": "4.8.3",
+  "version": "4.8.4",
   "description": "Local RAG System for Claude Code — Hybrid search + Cross-encoder Reranking + 13 MCP Tools + 20 Format Parsers. Zero external servers.",
   "bin": {
     "knowledge-rag": "./bin/cli.js"