n2-qln 3.3.4 → 3.4.0

package/README.ko.md

@@ -29,7 +29,9 @@
 
 🔍 **하나의 도구로 모든 것을** — AI는 `n2_qln_call` (~200 토큰)만 봅니다. 1,000개의 개별 도구가 아닙니다. 99.6% 컨텍스트 절감.
 
-⚡ **5ms 이하 검색** — 3단계 검색 엔진 (트리거 + 키워드 + 시맨틱)이 1,000개 이상의 도구에서도 5ms 이내에 최적 도구를 찾습니다.
+⚡ **5ms 이하 검색** — 3단계 검색 엔진 (트리거 + BM25 키워드 + 시맨틱)이 1,000개 이상의 도구에서도 5ms 이내에 최적 도구를 찾습니다.
+
+🎯 **BM25 키워드 랭킹** *(v3.4)* — Stage 2에 [Okapi BM25](https://en.wikipedia.org/wiki/Okapi_BM25) 알고리즘 적용. 희귀한 단어일수록 높은 점수, 문서 길이 정규화. Google, Elasticsearch, Wikipedia 검색의 핵심 알고리즘.
 
 📈 **자동 학습 랭킹** — 많이 사용되고 성공률이 높은 도구는 자동으로 상위에 랭크됩니다. 수동 튜닝 불필요.
 
@@ -184,14 +186,14 @@ QLN은 세 단계의 검색으로 적합한 도구를 찾습니다:
 | 단계 | 방식 | 속도 | 작동 원리 |
 |:---:|--------|:---:|---------|
 | **1** | 트리거 매칭 | ⚡ <1ms | 도구 이름과 트리거 키워드 정확 매칭 |
-| **2** | 키워드 검색 | ⚡ 1-3ms | 설명, 태그, 예제 전문 검색 |
+| **2** | BM25 키워드 | ⚡ 1-3ms | [Okapi BM25](https://en.wikipedia.org/wiki/Okapi_BM25) 랭킹 검색 — IDF 가중치 + 문서 길이 정규화 *(v3.4)* |
 | **3** | 시맨틱 검색 | 🧠 5-15ms | 임베딩 벡터 유사도 검색 *(선택, Ollama 필요)* |
 
 모든 단계의 결과를 병합 후 랭킹:
 
 ```
 final_score = trigger_score × 3.0
-            + keyword_score × 1.0
+            + bm25_keyword_score × 1.0
             + semantic_score × 2.0
             + log2(usage_count + 1) × 0.5
             + success_rate × 1.0
@@ -397,7 +399,7 @@ n2-qln/
 │   ├── schema.js       # 도구 스키마 정규화 + 검색 텍스트 빌더
 │   ├── validator.js    # 강제 검증 (이름, 설명, 카테고리)
 │   ├── registry.js     # 도구 CRUD + 사용량 추적 + 임베딩 캐시
-│   ├── router.js       # 3단계 검색 엔진
+│   ├── router.js       # 3단계 검색 엔진 (BM25 v3.4)
 │   ├── vector-index.js # Float32 벡터 인덱스 (centroid hierarchy)
 │   ├── embedding.js    # Ollama 임베딩 클라이언트 (nomic-embed-text)
 │   ├── executor.js     # HTTP/함수 도구 실행기

package/README.md

@@ -29,7 +29,9 @@
 
 🔍 **One tool to rule them all** — Your AI sees `n2_qln_call` (~200 tokens), not 1,000 individual tools. 99.6% context reduction.
 
-⚡ **Sub-5ms search** — 3-stage search engine (trigger + keyword + semantic) finds the right tool in under 5ms, even with 1,000+ tools indexed.
+⚡ **Sub-5ms search** — 3-stage search engine (trigger + BM25 keyword + semantic) finds the right tool in under 5ms, even with 1,000+ tools indexed.
+
+🎯 **BM25 keyword ranking** *(v3.4)* — Stage 2 uses [Okapi BM25](https://en.wikipedia.org/wiki/Okapi_BM25) for keyword search. Rare terms score higher, document length is normalized. The same algorithm behind Google, Elasticsearch, and Wikipedia search.
 
 📈 **Self-learning ranking** — Tools that get used more and succeed more are automatically ranked higher over time. No manual tuning needed.
 
@@ -184,14 +186,14 @@ QLN finds the right tool using three parallel search stages:
 | Stage | Method | Speed | How it works |
 |:---:|--------|:---:|------|
 | **1** | Trigger Match | ⚡ <1ms | Matches exact words in tool names and trigger keywords |
-| **2** | Keyword Search | ⚡ 1-3ms | Full-text search across descriptions, tags, and examples |
+| **2** | BM25 Keyword | ⚡ 1-3ms | [Okapi BM25](https://en.wikipedia.org/wiki/Okapi_BM25) ranked search — IDF weighting + document length normalization *(v3.4)* |
 | **3** | Semantic Search | 🧠 5-15ms | Vector similarity using embeddings *(optional, requires Ollama)* |
 
 Results from all stages are merged and ranked:
 
 ```
 final_score = trigger_score × 3.0
-            + keyword_score × 1.0
+            + bm25_keyword_score × 1.0
             + semantic_score × 2.0
             + log2(usage_count + 1) × 0.5
             + success_rate × 1.0
@@ -417,7 +419,7 @@ n2-qln/
 │   ├── schema.js       # Tool schema normalization + search text builder
 │   ├── validator.js    # Enforced validation (name, description, category)
 │   ├── registry.js       # Tool CRUD + usage tracking + embedding cache
-│   ├── router.js         # 3-stage parallel search engine
+│   ├── router.js         # 3-stage parallel search engine (BM25 v3.4)
 │   ├── vector-index.js   # Float32 vector index with centroid hierarchy
 │   ├── embedding.js      # Ollama embedding client (nomic-embed-text)
 │   ├── executor.js       # HTTP/function tool executor

package/lib/router.js

@@ -1,12 +1,12 @@
 // QLN — L1 Router (3-Stage parallel search engine)
-// Query → Stage1(Trigger) + Stage2(Keyword) + Stage3(Semantic) → Merge → Top-K
+// Query → Stage1(Trigger) + Stage2(BM25 Keyword) + Stage3(Semantic) → Merge → Top-K
 const { buildSearchText } = require('./schema');
 
 /**
  * 3-Stage search engine.
  *
  * Score formula:
- *   final = trigger×3.0 + keyword×1.0 + semantic×2.0
+ *   final = trigger×3.0 + bm25_keyword×1.0 + semantic×2.0
  *         + log2(usageCount+1)×0.5 + successRate×1.0
  */
 class Router {
@@ -19,6 +19,15 @@ class Router {
         this._registry = registry;
         this._vectorIndex = vectorIndex;
         this._embedding = embedding;
+
+        // BM25 parameters (standard Okapi BM25 defaults)
+        this._k1 = 1.2;   // Term frequency saturation
+        this._b = 0.75;    // Document length normalization
+
+        // IDF cache (rebuilt when tools change)
+        this._idfCache = new Map();
+        this._avgDocLen = 0;
+        this._idfDirty = true;
     }
 
     /**
@@ -34,14 +43,17 @@ class Router {
         const timing = { stage1: 0, stage2: 0, stage3: 0, merge: 0, total: 0 };
         const t0 = Date.now();
 
+        // Rebuild IDF if registry changed
+        if (this._idfDirty) this._buildIDF();
+
         // Stage 1: Trigger exact match (fastest)
         const t1 = Date.now();
         this._stage1TriggerMatch(query, scores);
         timing.stage1 = Date.now() - t1;
 
-        // Stage 2: Keyword match (search_text LIKE)
+        // Stage 2: BM25 keyword search
         const t2 = Date.now();
-        this._stage2KeywordMatch(query, scores);
+        this._stage2BM25(query, scores);
         timing.stage2 = Date.now() - t2;
 
         // Stage 3: Semantic vector search (when embedding available)
@@ -73,20 +85,101 @@ class Router {
         }
     }
 
-    /** Stage 2: search_text keyword match. Weight: 1.0 */
-    _stage2KeywordMatch(query, scores) {
-        const queryWords = query.toLowerCase().split(/\s+/).filter(w => w.length > 2);
+    /** Stage 2: BM25 keyword search. Weight: 1.0 */
+    _stage2BM25(query, scores) {
+        const queryTerms = this._tokenize(query);
+        if (queryTerms.length === 0) return;
+
         for (const tool of this._registry.getAll()) {
             const text = (tool.searchText || buildSearchText(tool)).toLowerCase();
-            let matchCount = 0;
-            for (const word of queryWords) {
-                if (text.includes(word)) matchCount++;
+            const bm25 = this._bm25Score(queryTerms, text);
+            if (bm25 > 0) {
+                this._getOrCreate(scores, tool.name).stage2 = bm25 * 1.0;
             }
-            if (matchCount > 0) {
-                this._getOrCreate(scores, tool.name).stage2 =
-                    (matchCount / Math.max(queryWords.length, 1)) * 1.0;
+        }
+    }
+
+    /**
+     * Calculate BM25 score for a query against a document.
+     * @param {string[]} queryTerms - Tokenized query terms
+     * @param {string} docText - Document text (lowercased)
+     * @returns {number} BM25 score
+     */
+    _bm25Score(queryTerms, docText) {
+        const docTerms = docText.split(/[\s_\-./]+/).filter(w => w.length > 1);
+        const docLen = docTerms.length;
+        if (docLen === 0) return 0;
+
+        // Build term frequency map for this document
+        const tf = new Map();
+        for (const term of docTerms) {
+            tf.set(term, (tf.get(term) || 0) + 1);
+        }
+
+        let score = 0;
+        for (const term of queryTerms) {
+            const idf = this._idfCache.get(term) || 0;
+            const freq = tf.get(term) || 0;
+            if (freq === 0) continue;
+
+            // BM25 formula: IDF × (f × (k1+1)) / (f + k1 × (1 - b + b × |d|/avgDL))
+            const numerator = freq * (this._k1 + 1);
+            const denominator = freq + this._k1 * (1 - this._b + this._b * (docLen / this._avgDocLen));
+            score += idf * (numerator / denominator);
+        }
+
+        return score;
+    }
+
+    /**
+     * Build IDF cache from all registered tools.
+     * IDF(term) = ln((N - n(t) + 0.5) / (n(t) + 0.5) + 1)
+     * where N = total docs, n(t) = docs containing term
+     */
+    _buildIDF() {
+        const tools = this._registry.getAll();
+        const N = tools.length;
+        if (N === 0) {
+            this._idfDirty = false;
+            return;
+        }
+
+        // Tokenize all documents and count document frequencies
+        const docFreq = new Map();
+        let totalLen = 0;
+
+        for (const tool of tools) {
+            const text = (tool.searchText || buildSearchText(tool)).toLowerCase();
+            const terms = text.split(/[\s_\-./]+/).filter(w => w.length > 1);
+            totalLen += terms.length;
+
+            // Unique terms per document
+            const uniqueTerms = new Set(terms);
+            for (const term of uniqueTerms) {
+                docFreq.set(term, (docFreq.get(term) || 0) + 1);
             }
         }
+
+        this._avgDocLen = totalLen / N;
+
+        // Calculate IDF for each term
+        this._idfCache.clear();
+        for (const [term, df] of docFreq) {
+            // BM25 IDF: ln((N - df + 0.5) / (df + 0.5) + 1)
+            const idf = Math.log((N - df + 0.5) / (df + 0.5) + 1);
+            this._idfCache.set(term, idf);
+        }
+
+        this._idfDirty = false;
+    }
+
+    /**
+     * Tokenize query string into search terms.
+     * @param {string} query
+     * @returns {string[]}
+     */
+    _tokenize(query) {
+        return query.toLowerCase().split(/[\s_\-./]+/).filter(w => w.length > 2);
     }
 
     /** Stage 3: Semantic vector search. Weight: 2.0 */
@@ -167,11 +260,17 @@ class Router {
         return ranked;
     }
 
-    /** Build vector index */
+    /** Build vector index and refresh IDF cache */
     buildIndex() {
+        this._idfDirty = true;
         return this._vectorIndex.build(this._registry.getAll());
     }
 
+    /** Mark IDF cache as dirty (call after tool registration changes) */
+    invalidateIDF() {
+        this._idfDirty = true;
+    }
+
     /** @private */
     _getOrCreate(scores, name) {
         if (!scores.has(name)) scores.set(name, { stage1: 0, stage2: 0, stage3: 0 });
@@ -184,8 +283,15 @@ class Router {
             registrySize: this._registry.size,
             vectorIndex: this._vectorIndex.stats(),
             embeddingAvailable: !!this._embedding,
+            bm25: {
+                idfTerms: this._idfCache.size,
+                avgDocLen: Math.round(this._avgDocLen * 10) / 10,
+                k1: this._k1,
+                b: this._b,
+            },
         };
     }
 }
 
 module.exports = { Router };
+

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "n2-qln",
-    "version": "3.3.4",
+    "version": "3.4.0",
     "description": "Query Layer Network — Semantic tool dispatcher for MCP. Route 1000 tools through 1 router.",
     "main": "index.js",
     "bin": {