@sdsrs/llm-wiki 0.7.3 → 0.8.0

package/CHANGELOG.md

@@ -1,5 +1,56 @@
 # Changelog
 
+## 0.8.0 (2026-07-12)
+
+Retrieval `auto` mode gains a cross-language noise guard — an additive config
+key (`lexicalGuard`, default on) with an opt-out. No KB migration, no API
+change, no new dependency. Suite 234 → 246.
+
+**Changed:**
+
+- **`auto` retrieval ranks vector-only when the lexical and semantic channels
+  disagree completely.** When BM25's top-k and the vector top-k return a
+  *disjoint* set of pages, BM25 contributed nothing the semantic channel
+  corroborates — on a cross-language query (e.g. a Chinese question against an
+  English KB) its incidental-token matches are noise that rank-based RRF would
+  blend in, pushing correct pages out of top-k. `auto` now drops BM25 and ranks
+  by the vector channel alone in that case. Measured on a 500-page KB: pure
+  cross-language Recall@5 rose 0.538 → 0.769 (matching pure vector) while
+  same-language fact retrieval stayed at 1.000 — the guard never fired on a
+  fact probe, so lexical retrieval is untouched where it is doing the work. The
+  guard is `auto`-only; explicit `hybrid` mode keeps pure RRF fusion. Set
+  `"lexicalGuard": false` in `wiki.config.json` to restore the previous
+  always-fuse behavior.
+
+**Added:**
+
+- `lexicalGuard` config key (boolean, default `true`), emitted into new KBs'
+  `wiki.config.json` by `init`.
+
+## 0.7.4 (2026-07-12)
+
+Two robustness fixes surfaced by an adversarial QA pass over the CLI. No config
+schema change, no KB migration, no retrieval-default change. Suite 230 → 234.
+
+**Fixes:**
+
+- **`index` warns when a page is skipped for invalid frontmatter.** `buildIndex`
+  silently excluded any page with unparseable YAML frontmatter from
+  `index.md`/`graph.json`/`llms.txt`. Because the docs route hand-editors to
+  `index` (not `lint`) after editing, a broken-YAML edit made the page vanish
+  from every query surface with no signal — `indexed N pages`, exit 0, nothing
+  else. `buildIndex` now returns the skipped pages and the CLI prints a one-line
+  stderr warning naming them and pointing at `lint`; stdout stays the clean
+  count, and `lint` still owns the detailed per-page report.
+- **`scan` near-duplicate detection uses 128 minhash permutations (was 32).**
+  Near-dup similarity was estimated from a 32-permutation minhash and compared
+  against the 0.85 threshold, but at 32 perms the estimate's standard error
+  (~0.05) let a genuine near-duplicate (true Jaccard 0.898) estimate 0.844 and
+  be reported as `near: 0`. 128 perms tightens the estimate (SD ~0.026) so real
+  near-dups clear the threshold; the cost is trivial next to the per-file
+  sha256+read scan already does, and the signature is stripped before the plan
+  is persisted (no on-disk change).
+
 ## 0.7.3 (2026-07-12)
 
 Two robustness fixes from the final targeted dogfooding sweep. No config schema

package/bin/llm-wiki.mjs

@@ -65,6 +65,12 @@ program.command('index')
   .action((opts) => {
     const r = buildIndex(opts.kb)
     console.log(`indexed ${r.pageCount} pages${r.topicsSplit ? ' (split into topics/)' : ''}`)
+    // A page with unparseable frontmatter is excluded from index.md/graph.json/llms.txt. Surface
+    // it on stderr (stdout stays the clean count) so a hand-edit that broke the YAML doesn't
+    // silently vanish — the README points hand-editors here, not at `lint`.
+    if (r.skipped?.length) {
+      console.error(`warning: skipped ${r.skipped.length} page(s) with invalid frontmatter (run 'lint' for details): ${r.skipped.map(s => s.relPath).join(', ')}`)
+    }
   })
 
 program.command('embed')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sdsrs/llm-wiki",
-  "version": "0.7.3",
+  "version": "0.8.0",
   "description": "Compile messy document directories into Karpathy-style llm_wiki knowledge bases — standalone Q&A CLI + Claude Code/Codex skills",
   "directories": {
     "test": "test"

package/src/ask.mjs

@@ -41,16 +41,39 @@ export function rrfFuse(lists, k) {
   return [...acc.values()].sort((a, b) => b.score - a.score).slice(0, k)
 }
 
+// auto-mode guard: when the lexical (BM25) and semantic (vector) channels return
+// disjoint top-k page sets, BM25 contributed nothing the vector channel corroborates
+// — on a cross-language query its incidental-token hits are noise that rank-based RRF
+// would blend in and push correct pages out of top-k. In that case drop BM25 and rank
+// vector-only. Parameter-free: the overlap test uses the same k the call retrieves.
+// vector.length > 0 is required so the guard never returns an empty list.
+export function fuseChannels({ bm25, vector }, k, { lexicalGuard = true } = {}) {
+  if (lexicalGuard && vector.length > 0) {
+    const bm25Ids = new Set(bm25.map(h => h.relPath))
+    const disjoint = !vector.some(h => bm25Ids.has(h.relPath))
+    if (disjoint) {
+      return { hits: vector.slice(0, k).map(h => ({ ...h, sources: ['vector'] })), guardApplied: true }
+    }
+  }
+  return {
+    hits: rrfFuse([{ source: 'bm25', hits: bm25 }, { source: 'vector', hits: vector }], k),
+    guardApplied: false,
+  }
+}
+
 // Three modes. 'auto' (default): BM25 always; vector channel only when opted
 // in (vectorEnabled) AND the sidecar exists AND an embeddingModel is
 // configured — fail-open, any vector error degrades to BM25 with a single
-// stderr warning. 'bm25': lexical only, returns before any vector access.
+// stderr warning. When both channels run but return disjoint top-k page sets,
+// the lexicalGuard (config, default on) drops BM25 and ranks vector-only so a
+// cross-language query's lexical noise cannot displace correct semantic hits.
+// 'bm25': lexical only, returns before any vector access.
 // 'hybrid': explicit BM25+vector fusion — every missing prerequisite (and any
 // vector error) throws instead of degrading, and vectorEnabled is ignored.
 export async function locatePages(kbRoot, question, { k = 6, fetchImpl, retrieval = 'auto', retry } = {}) {
   if (!['auto', 'bm25', 'hybrid'].includes(retrieval)) throw new Error(`unknown retrieval mode: ${retrieval}`)
   const bm25 = retrievePages(kbRoot, question, k)
-  const asBm25 = () => ({ hits: bm25.map(h => ({ ...h, sources: ['bm25'] })), usedVector: false })
+  const asBm25 = () => ({ hits: bm25.map(h => ({ ...h, sources: ['bm25'] })), usedVector: false, guardApplied: false })
   if (retrieval === 'bm25') return asBm25()
   // 'hybrid' is an explicit request: prerequisites failing is an error, not a
   // silent degrade. 'auto' keeps the opt-in + fail-open contract unchanged.
@@ -77,7 +100,9 @@ export async function locatePages(kbRoot, question, { k = 6, fetchImpl, retrieva
     // ENOENTs reading a vanished file).
     const valid = new Set(listWikiPages(kbRoot).filter(pg => !pg.error && !isInvalidated(pg)).map(pg => pg.relPath))
     const vecHitsValid = vecHits.filter(h => valid.has(h.relPath))
-    return { hits: rrfFuse([{ source: 'bm25', hits: bm25 }, { source: 'vector', hits: vecHitsValid }], k), usedVector: true }
+    const { hits, guardApplied } = fuseChannels({ bm25, vector: vecHitsValid }, k,
+      { lexicalGuard: retrieval === 'auto' && loadKbConfig(kbRoot).lexicalGuard })
+    return { hits, usedVector: true, guardApplied }
   } catch (err) {
     if (retrieval === 'hybrid') throw err
     process.stderr.write(`warning: vector retrieval unavailable (${err.message}); falling back to BM25\n`)

package/src/hashing.mjs

@@ -13,7 +13,12 @@ function fnv1a(str) {
   return h
 }
 
-export function minhashSignature(text, perms = 32) {
+// 128 permutations, not 32: the near-dup estimate's standard error is ~sqrt(J(1-J)/perms),
+// so at 32 perms a genuine near-duplicate (true Jaccard ~0.90) estimated as low as 0.84 and
+// fell below NEAR_DUP_THRESHOLD (0.85), silently missing it (QA73-002). 128 perms tightens the
+// estimate (SD ~0.026) so real near-dups clear the threshold; the cost is trivial next to the
+// per-file sha256+read scan already does, and `_sig` is stripped before the plan is persisted.
+export function minhashSignature(text, perms = 128) {
   const norm = text.toLowerCase().replace(/\s+/g, ' ')
   const shingles = new Set()
   for (let i = 0; i <= norm.length - 5; i++) shingles.add(norm.slice(i, i + 5))

package/src/indexer.mjs

@@ -26,7 +26,13 @@ const inline = (s) => String(s ?? '').replace(/[\r\n]+/g, ' ')
 export function buildIndex(kbRoot) {
   const p = kbPaths(kbRoot)
   const cfg = loadKbConfig(kbRoot)
-  const pages = listWikiPages(kbRoot).filter(pg => !pg.error)
+  const allPages = listWikiPages(kbRoot)
+  const pages = allPages.filter(pg => !pg.error)
+  // Pages with unparseable/missing frontmatter are excluded from every derived file. Return
+  // them so the CLI can warn instead of silently dropping a page — the README routes
+  // hand-editors to `index` (not `lint`), so a bad edit would otherwise vanish with no signal
+  // (QA73-001). lint still owns the detailed per-page report.
+  const skipped = allPages.filter(pg => pg.error).map(pg => ({ relPath: pg.relPath, error: pg.error }))
 
   // Preserve the pending section, bounded at the next `## ` heading — a greedy
   // match to EOF would swallow user-added sections into "pending" forever.
@@ -122,5 +128,5 @@ export function buildIndex(kbRoot) {
     ...pages.filter(pg => !isInvalidated(pg)).map(pg => `- [${inline(pg.data.title)}](wiki/${pg.relPath}): ${inline(pg.data.description)}`)].join('\n') + '\n'
   writeFileAtomic(p.llmsTxt, llms)
 
-  return { pageCount: pages.length, topicsSplit }
+  return { pageCount: pages.length, topicsSplit, skipped }
 }

package/src/templates.mjs

@@ -14,6 +14,7 @@ export const DEFAULT_CONFIG = {
   maxFileBytes: 52428800,
   bm25TitleWeight: 3,
   vectorEnabled: false,
+  lexicalGuard: true,
   relationTypes: ['implements', 'uses', 'depends_on', 'part_of', 'instance_of', 'derived_from', 'contrasts_with', 'causes'],
 }
 
@@ -38,6 +39,11 @@ export function loadKbConfig(kbRoot) {
     const n = Math.trunc(Number(merged[k]))
     merged[k] = Number.isFinite(n) && n >= 1 ? n : DEFAULT_CONFIG[k]
   }
+  // lexicalGuard is a boolean gate; a non-boolean override (string/number/null) must
+  // not read as truthy. Only an explicit true/false is honored — anything else falls
+  // back to the default. (Targeted, not a general boolean sweep: vectorEnabled keeps
+  // its existing truthy semantics to avoid changing behavior on malformed input.)
+  merged.lexicalGuard = typeof merged.lexicalGuard === 'boolean' ? merged.lexicalGuard : DEFAULT_CONFIG.lexicalGuard
   return merged
 }