@sdsrs/llm-wiki 0.6.7 → 0.7.0

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## 0.7.0 (2026-07-11)
+
+Two retrieval-default changes (hence a minor bump). Both are opt-out via
+`wiki.config.json`; no KB migration, no re-index needed. Suite 210 → 213.
+
+**What changes for you:**
+
+- **BM25 now weights page titles.** A query term in a page's title outranks the
+  same term buried in another page's body. Measured on the dogfood KB (24 probes):
+  Recall@5 0.688 → 0.729, MRR 0.646 → 0.675 (the lift is entirely on fact-style
+  queries; cross-language and multi-hop are unchanged). **Opt out** by setting
+  `"bm25TitleWeight": 1` in `wiki.config.json` (default `3`) — `1` restores the
+  previous flat single-field indexing.
+- **`ask` now budgets tokens pessimistically.** Page size against `askTokenBudget`
+  is now estimated at a worst-case ~2 chars/token (was ~4), so dense
+  markdown/code pages no longer silently overflow a small model context window.
+  The visible effect: on a tight budget, `ask` may load fewer whole pages before
+  trimming the lowest-ranked ones. **To include more pages**, raise
+  `"askTokenBudget"` in `wiki.config.json` (default `32000`) — pages are still
+  always sent whole, never truncated.
+
 ## 0.6.7 (2026-07-11)
 
 Contract/skill clarifications, an env-var convenience, and an explicit platform

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sdsrs/llm-wiki",
-  "version": "0.6.7",
+  "version": "0.7.0",
   "description": "Compile messy document directories into Karpathy-style llm_wiki knowledge bases — standalone Q&A CLI + Claude Code/Codex skills",
   "directories": {
     "test": "test"

package/src/ask.mjs

@@ -4,17 +4,22 @@ import { kbPaths } from './paths.mjs'
 import { loadKbConfig } from './templates.mjs'
 import { listWikiPages, isInvalidated } from './pages.mjs'
 import { buildBm25Index, searchBm25 } from './bm25.mjs'
-import { estimateTokens } from './scanner.mjs'
+import { worstCaseTokens } from './scanner.mjs'
 import { loadLlmConfig, makeTransport } from './llm-config.mjs'
 import { loadVectorStore, normalize, cosineTopK } from './vector.mjs'
 import { embedTexts } from './embed.mjs'
 import { fetchWithRetry } from './retry.mjs'
 
 export function retrievePages(kbRoot, question, k = 6) {
+  // Title is a field: repeat it bm25TitleWeight times in the indexed text so a
+  // title term outweighs the same term buried in the body (measured +0.04 Recall@5
+  // on the dogfood KB at ×3). Set bm25TitleWeight: 1 in wiki.config.json to restore
+  // flat single-field indexing.
+  const w = Math.max(1, Math.trunc(loadKbConfig(kbRoot).bm25TitleWeight ?? 1))
   const pages = listWikiPages(kbRoot).filter(p => !p.error && !isInvalidated(p))
   const idx = buildBm25Index(pages.map(p => ({
     id: p.relPath,
-    text: [p.data.title, p.data.description, (p.data.tags ?? []).join(' '), p.body].join('\n'),
+    text: [Array(w).fill(p.data.title ?? '').join('\n'), p.data.description, (p.data.tags ?? []).join(' '), p.body].join('\n'),
   })))
   return searchBm25(idx, question, k).map(h => ({ relPath: h.id, score: h.score }))
 }
@@ -164,7 +169,9 @@ export async function askKb(kbRoot, question, { k = 6, retrieveOnly = false, fet
     // off every lower-ranked page too (no greedy backfill with smaller pages).
     if (trimmed.length > 0) { trimmed.push(h.relPath); continue }
     const text = fs.readFileSync(path.join(p.wiki, h.relPath), 'utf8')
-    const tokens = estimateTokens(text)
+    // Pessimistic estimate (worst-case ~2 chars/token) so dense pages don't overflow
+    // a small context window; raise askTokenBudget to include more pages per query.
+    const tokens = worstCaseTokens(text)
     if (loaded.length > 0 && used + tokens > kbCfg.askTokenBudget) { trimmed.push(h.relPath); continue }
     used += tokens
     loaded.push({ ...h, text })

package/src/scanner.mjs

@@ -49,6 +49,17 @@ export function estimateTokens(text) {
   return Math.round(cjk / 1.6 + (text.length - cjk) / 4)
 }
 
+// Pessimistic token estimate for budgeting model input (ask). Dense markdown/code
+// runs ~2-3.5 chars/BPE-token, so estimateTokens' chars/4 under-counts and can
+// overflow a small context window; assume ~2 chars/token for non-CJK and ~1
+// token/char for CJK. (Mirror of embed.mjs's local worstCaseEmbedTokens, which
+// stays local by its own note; this is the shared budgeting copy.)
+export function worstCaseTokens(text) {
+  let cjk = 0
+  for (const ch of text) if (/[　-鿿豈-﫿]/.test(ch)) cjk++
+  return Math.round(cjk + (text.length - cjk) / 2)
+}
+
 // Symlinked directories are not followed (loop safety) but are recorded in
 // `skippedDirs` so they surface in the scan report instead of vanishing silently.
 // Symlinked files are followed ONLY when their target resolves inside the source

package/src/templates.mjs

@@ -12,6 +12,7 @@ export const DEFAULT_CONFIG = {
   linkStyle: 'wikilink',
   askTokenBudget: 32000,
   maxFileBytes: 52428800,
+  bm25TitleWeight: 3,
   vectorEnabled: false,
   relationTypes: ['implements', 'uses', 'depends_on', 'part_of', 'instance_of', 'derived_from', 'contrasts_with', 'causes'],
 }