@sdsrs/llm-wiki 0.6.7 → 0.7.1

package/CHANGELOG.md

@@ -1,5 +1,53 @@
 # Changelog
 
+## 0.7.1 (2026-07-11)
+
+Four defects found by end-to-end dogfooding — one crash-class fix plus three
+robustness/UX fixes. No config change, no KB migration, no retrieval-default
+change. Suite 213 → 221.
+
+**Fixes:**
+
+- **A single malformed page no longer breaks the whole KB.** A frontmatter list
+  field authored as a bare scalar (`tags: cache` instead of `tags: [cache]`)
+  parses as a string, which slipped past the `?? []` guards and then crashed
+  `ask`/`embed` (`.join` on a string) or was iterated character-by-character
+  (`lint` noise; `index` synthesized a bogus graph edge per character, corrupting
+  `graph.json`). List fields now route through an `Array.isArray` guard in every
+  consumer, so one bad page can no longer take down retrieval or the graph.
+  `lint` now flags `tags must be a YAML list` so the author can fix the shape.
+- **`lint` contradiction-scan skips invalidated pages.** Retired knowledge is
+  already excluded from stale-scan and the orphan rule; the shared-tag
+  contradiction scan now matches, so you are never asked to reconcile a
+  contradiction against a dead page.
+- **`convert` exits non-zero when every file fails.** `converted 0, failed N`
+  used to exit `0`, hiding a total failure from CI/scripts. It now exits `1` on
+  total failure; a partial success (some files converted) still exits `0`.
+- **Network errors name the endpoint.** A mistyped `baseURL` or offline host used
+  to surface undici's opaque `fetch failed`; `ask`/`embed` now report
+  `could not reach the LLM/embedding endpoint <url> — check baseURL/network`.
+
+## 0.7.0 (2026-07-11)
+
+Two retrieval-default changes (hence a minor bump). Both are opt-out via
+`wiki.config.json`; no KB migration, no re-index needed. Suite 210 → 213.
+
+**What changes for you:**
+
+- **BM25 now weights page titles.** A query term in a page's title outranks the
+  same term buried in another page's body. Measured on the dogfood KB (24 probes):
+  Recall@5 0.688 → 0.729, MRR 0.646 → 0.675 (the lift is entirely on fact-style
+  queries; cross-language and multi-hop are unchanged). **Opt out** by setting
+  `"bm25TitleWeight": 1` in `wiki.config.json` (default `3`) — `1` restores the
+  previous flat single-field indexing.
+- **`ask` now budgets tokens pessimistically.** Page size against `askTokenBudget`
+  is now estimated at a worst-case ~2 chars/token (was ~4), so dense
+  markdown/code pages no longer silently overflow a small model context window.
+  The visible effect: on a tight budget, `ask` may load fewer whole pages before
+  trimming the lowest-ranked ones. **To include more pages**, raise
+  `"askTokenBudget"` in `wiki.config.json` (default `32000`) — pages are still
+  always sent whole, never truncated.
+
 ## 0.6.7 (2026-07-11)
 
 Contract/skill clarifications, an env-var convenience, and an explicit platform

package/bin/llm-wiki.mjs

@@ -50,6 +50,10 @@ program.command('convert')
     const r = await runConvertPlan(opts.kb)
     console.log(`converted ${r.converted.length}, failed ${r.failed.length}`)
     for (const f of r.failed) console.log(`  FAILED ${f.src}: ${f.warnings.join('; ')}`)
+    // Total failure (nothing converted, at least one error) is a hard error a
+    // pipeline must catch — exit 1. A partial success keeps exit 0 so a messy dir
+    // with a few unconvertible junk files doesn't break an otherwise-good run.
+    if (r.converted.length === 0 && r.failed.length > 0) process.exit(1)
   })
 
 program.command('index')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sdsrs/llm-wiki",
-  "version": "0.6.7",
+  "version": "0.7.1",
   "description": "Compile messy document directories into Karpathy-style llm_wiki knowledge bases — standalone Q&A CLI + Claude Code/Codex skills",
   "directories": {
     "test": "test"

package/src/ask.mjs

@@ -2,19 +2,24 @@ import fs from 'node:fs'
 import path from 'node:path'
 import { kbPaths } from './paths.mjs'
 import { loadKbConfig } from './templates.mjs'
-import { listWikiPages, isInvalidated } from './pages.mjs'
+import { listWikiPages, isInvalidated, asList } from './pages.mjs'
 import { buildBm25Index, searchBm25 } from './bm25.mjs'
-import { estimateTokens } from './scanner.mjs'
+import { worstCaseTokens } from './scanner.mjs'
 import { loadLlmConfig, makeTransport } from './llm-config.mjs'
 import { loadVectorStore, normalize, cosineTopK } from './vector.mjs'
 import { embedTexts } from './embed.mjs'
 import { fetchWithRetry } from './retry.mjs'
 
 export function retrievePages(kbRoot, question, k = 6) {
+  // Title is a field: repeat it bm25TitleWeight times in the indexed text so a
+  // title term outweighs the same term buried in the body (measured +0.04 Recall@5
+  // on the dogfood KB at ×3). Set bm25TitleWeight: 1 in wiki.config.json to restore
+  // flat single-field indexing.
+  const w = Math.max(1, Math.trunc(loadKbConfig(kbRoot).bm25TitleWeight ?? 1))
   const pages = listWikiPages(kbRoot).filter(p => !p.error && !isInvalidated(p))
   const idx = buildBm25Index(pages.map(p => ({
     id: p.relPath,
-    text: [p.data.title, p.data.description, (p.data.tags ?? []).join(' '), p.body].join('\n'),
+    text: [Array(w).fill(p.data.title ?? '').join('\n'), p.data.description, asList(p.data.tags).join(' '), p.body].join('\n'),
   })))
   return searchBm25(idx, question, k).map(h => ({ relPath: h.id, score: h.score }))
 }
@@ -79,11 +84,19 @@ export async function locatePages(kbRoot, question, { k = 6, fetchImpl, retrieva
 }
 
 export async function chatCompletion(cfg, t, messages) {
-  const res = await fetchWithRetry(t.fetchImpl, `${cfg.baseURL.replace(/\/$/, '')}/chat/completions`, {
-    method: 'POST',
-    headers: { 'content-type': 'application/json', authorization: `Bearer ${cfg.apiKey}` },
-    body: JSON.stringify({ model: cfg.model, messages }),
-  }, { dispatcher: t.dispatcher, ...(t.retry ?? {}) })
+  const url = `${cfg.baseURL.replace(/\/$/, '')}/chat/completions`
+  let res
+  try {
+    res = await fetchWithRetry(t.fetchImpl, url, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json', authorization: `Bearer ${cfg.apiKey}` },
+      body: JSON.stringify({ model: cfg.model, messages }),
+    }, { dispatcher: t.dispatcher, ...(t.retry ?? {}) })
+  } catch (err) {
+    // Node's undici surfaces a bare "fetch failed" for DNS/refused/timeout — name
+    // the endpoint so a mistyped baseURL or offline host is diagnosable, not opaque.
+    throw new Error(`could not reach the LLM endpoint ${url} — check baseURL/network in ~/.llm-wiki/config.json (${err.message})`)
+  }
   if (!res.ok) {
     let body = ''
     try { body = (await res.text()).slice(0, 200) } catch { /* body unreadable; status alone */ }
@@ -164,7 +177,9 @@ export async function askKb(kbRoot, question, { k = 6, retrieveOnly = false, fet
     // off every lower-ranked page too (no greedy backfill with smaller pages).
     if (trimmed.length > 0) { trimmed.push(h.relPath); continue }
     const text = fs.readFileSync(path.join(p.wiki, h.relPath), 'utf8')
-    const tokens = estimateTokens(text)
+    // Pessimistic estimate (worst-case ~2 chars/token) so dense pages don't overflow
+    // a small context window; raise askTokenBudget to include more pages per query.
+    const tokens = worstCaseTokens(text)
     if (loaded.length > 0 && used + tokens > kbCfg.askTokenBudget) { trimmed.push(h.relPath); continue }
     used += tokens
     loaded.push({ ...h, text })

package/src/embed.mjs

@@ -45,11 +45,19 @@ function capEmbedText(text) {
 }
 
 export async function embedTexts(cfg, t, texts) {
-  const res = await fetchWithRetry(t.fetchImpl, `${cfg.baseURL.replace(/\/$/, '')}/embeddings`, {
-    method: 'POST',
-    headers: { 'content-type': 'application/json', authorization: `Bearer ${cfg.apiKey}` },
-    body: JSON.stringify({ model: cfg.embeddingModel, input: texts }),
-  }, { dispatcher: t.dispatcher, ...(t.retry ?? {}) })
+  const url = `${cfg.baseURL.replace(/\/$/, '')}/embeddings`
+  let res
+  try {
+    res = await fetchWithRetry(t.fetchImpl, url, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json', authorization: `Bearer ${cfg.apiKey}` },
+      body: JSON.stringify({ model: cfg.embeddingModel, input: texts }),
+    }, { dispatcher: t.dispatcher, ...(t.retry ?? {}) })
+  } catch (err) {
+    // Mirror chatCompletion: turn undici's opaque "fetch failed" into a diagnosable
+    // message naming the endpoint (mistyped baseURL / offline host / DNS failure).
+    throw new Error(`could not reach the embedding endpoint ${url} — check baseURL/network in ~/.llm-wiki/config.json (${err.message})`)
+  }
   if (!res.ok) {
     let body = ''
     try { body = (await res.text()).slice(0, 200) } catch { /* status alone */ }

package/src/indexer.mjs

@@ -2,7 +2,7 @@ import fs from 'node:fs'
 import path from 'node:path'
 import { kbPaths } from './paths.mjs'
 import { loadKbConfig } from './templates.mjs'
-import { listWikiPages, isInvalidated, RELATION_CONFIDENCES } from './pages.mjs'
+import { listWikiPages, isInvalidated, asList, RELATION_CONFIDENCES } from './pages.mjs'
 
 const WIKILINK_RE = /\[\[([^\]|#]+)(?:[#|][^\]]*)?\]\]/g
 
@@ -96,7 +96,7 @@ export function buildIndex(kbRoot) {
     for (const target of extractWikilinks(pg.body)) {
       if (ids.has(target)) edges.push({ source: id, target, type: 'wikilink', confidence: 'inferred' })
     }
-    for (const src of pg.data.sources ?? []) edges.push({ source: id, target: String(src), type: 'source', confidence: 'extracted' })
+    for (const src of asList(pg.data.sources)) edges.push({ source: id, target: String(src), type: 'source', confidence: 'extracted' })
     if (pg.data.superseded_by && ids.has(String(pg.data.superseded_by))) {
       edges.push({ source: id, target: String(pg.data.superseded_by), type: 'superseded_by', confidence: 'extracted' })
     }

package/src/lint.mjs

@@ -2,7 +2,7 @@ import fs from 'node:fs'
 import path from 'node:path'
 import { kbPaths } from './paths.mjs'
 import { loadKbConfig } from './templates.mjs'
-import { listWikiPages, validatePage, isInvalidated, PAGE_STATUSES, RELATION_CONFIDENCES } from './pages.mjs'
+import { listWikiPages, validatePage, isInvalidated, asList, PAGE_STATUSES, RELATION_CONFIDENCES } from './pages.mjs'
 import { extractWikilinks, buildIndex } from './indexer.mjs'
 import { loadManifest } from './manifest.mjs'
 
@@ -27,7 +27,7 @@ export async function lintKb(kbRoot, { fix = false } = {}) {
       if (!ids.has(target)) mechanical.push({ rule: 'broken-wikilink', path: pg.relPath, detail: `-> ${target}` })
       else incoming.set(target, (incoming.get(target) ?? 0) + 1)
     }
-    for (const src of pg.data.sources ?? []) {
+    for (const src of asList(pg.data.sources)) {
       if (!fs.existsSync(path.join(kbRoot, String(src)))) mechanical.push({ rule: 'missing-raw-source', path: pg.relPath, detail: String(src) })
     }
     if (pg.data.status !== undefined && !PAGE_STATUSES.includes(pg.data.status)) {
@@ -90,8 +90,11 @@ export async function lintKb(kbRoot, { fix = false } = {}) {
   }
   const byTag = new Map()
   for (const pg of pages) {
-    if (pg.error) continue
-    for (const tag of pg.data.tags ?? []) {
+    // Invalidated pages are retired knowledge: they must drop out of the semantic
+    // worklist just like stale-scan (below) and the orphan rule already do, so the
+    // agent is never asked to reconcile a "contradiction" against a dead page.
+    if (pg.error || isInvalidated(pg)) continue
+    for (const tag of asList(pg.data.tags)) {
       if (!byTag.has(tag)) byTag.set(tag, [])
       byTag.get(tag).push(pg.relPath)
     }
@@ -115,7 +118,7 @@ export async function lintKb(kbRoot, { fix = false } = {}) {
     if (pg.error || isInvalidated(pg)) continue
     const updated = String(pg.data.updated ?? '')
     if (!updated) continue
-    for (const src of pg.data.sources ?? []) {
+    for (const src of asList(pg.data.sources)) {
       const conv = convertedAtByRaw.get(String(src))
       if (conv && conv > updated) semantic.push({ task: 'stale-scan', detail: `${pg.relPath}: ${src} reconverted ${conv}, page updated ${updated}` })
     }

package/src/pages.mjs

@@ -10,6 +10,14 @@ export const PAGE_STATUSES = ['active', 'invalidated']
 
 export const RELATION_CONFIDENCES = ['extracted', 'inferred', 'ambiguous']
 
+// A frontmatter list field (tags, sources) authored as a bare scalar — `tags: cache`
+// instead of `tags: [cache]` — is parsed by YAML as a string. That silently passes a
+// `?? []` guard and then crashes `.join()` (retrieval/embed) or iterates the string
+// character-by-character (lint noise, and indexer synthesizing a bogus graph edge per
+// char). Every consumer routes list fields through this so one malformed page cannot
+// DoS retrieval or corrupt the graph; validatePage/lint still flag the bad shape.
+export const asList = (v) => (Array.isArray(v) ? v : [])
+
 export function isInvalidated(page) {
   return page.data?.status === 'invalidated'
 }
@@ -40,5 +48,9 @@ export function validatePage(page) {
     if (page.data[k] === undefined || page.data[k] === null || page.data[k] === '') issues.push(`missing field: ${k}`)
   }
   if (!Array.isArray(page.data.sources)) issues.push('missing field: sources (evidence chain)')
+  // tags is caught as present by the REQUIRED loop above even when authored as a
+  // bare scalar (`tags: cache`); flag the wrong shape explicitly so the author fixes
+  // it rather than silently getting a page with no searchable tags.
+  if (page.data.tags !== undefined && !Array.isArray(page.data.tags)) issues.push('tags must be a YAML list')
   return issues
 }

package/src/scanner.mjs

@@ -49,6 +49,17 @@ export function estimateTokens(text) {
   return Math.round(cjk / 1.6 + (text.length - cjk) / 4)
 }
 
+// Pessimistic token estimate for budgeting model input (ask). Dense markdown/code
+// runs ~2-3.5 chars/BPE-token, so estimateTokens' chars/4 under-counts and can
+// overflow a small context window; assume ~2 chars/token for non-CJK and ~1
+// token/char for CJK. (Mirror of embed.mjs's local worstCaseEmbedTokens, which
+// stays local by its own note; this is the shared budgeting copy.)
+export function worstCaseTokens(text) {
+  let cjk = 0
+  for (const ch of text) if (/[　-鿿豈-﫿]/.test(ch)) cjk++
+  return Math.round(cjk + (text.length - cjk) / 2)
+}
+
 // Symlinked directories are not followed (loop safety) but are recorded in
 // `skippedDirs` so they surface in the scan report instead of vanishing silently.
 // Symlinked files are followed ONLY when their target resolves inside the source

package/src/status.mjs

@@ -1,7 +1,7 @@
 import fs from 'node:fs'
 import path from 'node:path'
 import { kbPaths } from './paths.mjs'
-import { listWikiPages } from './pages.mjs'
+import { listWikiPages, asList } from './pages.mjs'
 import { scanSource } from './scanner.mjs'
 import { loadManifest, diffManifest } from './manifest.mjs'
 
@@ -27,7 +27,7 @@ export async function statusKb(kbRoot, srcDir) {
   for (const pg of listWikiPages(kbRoot)) {
     if (pg.error) continue
     const id = pg.relPath.replace(/\.md$/, '')
-    for (const src of pg.data.sources ?? []) {
+    for (const src of asList(pg.data.sources)) {
       const raw = String(src)
       referenced.add(raw)
       if (!pagesByRaw.has(raw)) pagesByRaw.set(raw, [])

package/src/templates.mjs

@@ -12,6 +12,7 @@ export const DEFAULT_CONFIG = {
   linkStyle: 'wikilink',
   askTokenBudget: 32000,
   maxFileBytes: 52428800,
+  bm25TitleWeight: 3,
   vectorEnabled: false,
   relationTypes: ['implements', 'uses', 'depends_on', 'part_of', 'instance_of', 'derived_from', 'contrasts_with', 'causes'],
 }

package/src/vector.mjs

@@ -2,6 +2,7 @@ import fs from 'node:fs'
 import path from 'node:path'
 import { kbPaths } from './paths.mjs'
 import { readJsonFile } from './json.mjs'
+import { asList } from './pages.mjs'
 
 export function normalize(vec) {
   let s = 0
@@ -13,7 +14,7 @@ export function normalize(vec) {
 
 // Same fields BM25 indexes (src/ask.mjs retrievePages) so both channels see one text.
 export function pageEmbedText(pg) {
-  return [pg.data.title ?? '', pg.data.description ?? '', (pg.data.tags ?? []).join(' '), pg.body].join('\n')
+  return [pg.data.title ?? '', pg.data.description ?? '', asList(pg.data.tags).join(' '), pg.body].join('\n')
 }
 
 export function vectorStorePath(kbRoot) {