@sdsrs/llm-wiki 0.7.1 → 0.7.2

package/CHANGELOG.md

@@ -1,5 +1,36 @@
 # Changelog
 
+## 0.7.2 (2026-07-12)
+
+Three defects found by continued end-to-end dogfooding — two robustness/DoS
+guards plus one UX fix. No config schema change, no KB migration, no
+retrieval-default change. Suite 221 → 228.
+
+**Fixes:**
+
+- **Malformed numeric config can no longer crash retrieval or hang scan.**
+  `wiki.config.json` is shallow-merged as-is, so a numeric key could arrive as a
+  string/0/negative/NaN and reach a scalar consumer: `bm25TitleWeight: "high"`
+  hit `Array(NaN)` → `RangeError` on *every* `ask`/`search`/MCP query (a huge int
+  → per-page `Array(n).fill` OOM), and `batchSize: 0` made `scan`'s batch loop
+  (`i += batchSize`) spin forever. `loadKbConfig` now coerces every numeric key
+  back to a finite integer ≥ 1 (falling back to its default), and
+  `bm25TitleWeight` is additionally capped at its consumer. Legitimate overrides
+  are untouched.
+- **Derived stores are written atomically.** `buildIndex` wrote `graph.json` /
+  `index.md` / `llms.txt` / topic files with a direct `writeFileSync`, which
+  truncates the target before writing — a reader running concurrently (a
+  long-lived MCP `graph` tool, `ask`'s `llms.txt` fallback) could read a torn
+  `graph.json` and crash on `JSON.parse`. All derived stores now write via a
+  shared temp-and-rename `writeFileAtomic` (the pattern `saveManifest` /
+  `saveVectorStore` already used), so a concurrent reader only ever sees the whole
+  old or whole new file.
+- **A source that converts to an empty page is no longer silent.** A
+  scanned/image-only PDF, an empty DOCX, or a blank `.txt` extracts to no text;
+  `convert` still writes the (empty) raw page and counts it as converted, but now
+  prints `WARN <src>: converted to an empty page — no extractable text` so the
+  user isn't misled into thinking real content was ingested.
+
 ## 0.7.1 (2026-07-11)
 
 Four defects found by end-to-end dogfooding — one crash-class fix plus three

package/bin/llm-wiki.mjs

@@ -50,6 +50,9 @@ program.command('convert')
     const r = await runConvertPlan(opts.kb)
     console.log(`converted ${r.converted.length}, failed ${r.failed.length}`)
     for (const f of r.failed) console.log(`  FAILED ${f.src}: ${f.warnings.join('; ')}`)
+    // A converted-but-empty page (scanned PDF, empty DOCX) is not a failure, but the
+    // user must see it — an unflagged blank raw page reads as "ingested" when it isn't.
+    for (const c of r.converted) if (c.warnings) console.log(`  WARN ${c.src}: ${c.warnings.join('; ')}`)
     // Total failure (nothing converted, at least one error) is a hard error a
     // pipeline must catch — exit 1. A partial success keeps exit 0 so a messy dir
     // with a few unconvertible junk files doesn't break an otherwise-good run.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sdsrs/llm-wiki",
-  "version": "0.7.1",
+  "version": "0.7.2",
   "description": "Compile messy document directories into Karpathy-style llm_wiki knowledge bases — standalone Q&A CLI + Claude Code/Codex skills",
   "directories": {
     "test": "test"

package/src/ask.mjs

@@ -15,7 +15,9 @@ export function retrievePages(kbRoot, question, k = 6) {
   // title term outweighs the same term buried in the body (measured +0.04 Recall@5
   // on the dogfood KB at ×3). Set bm25TitleWeight: 1 in wiki.config.json to restore
   // flat single-field indexing.
-  const w = Math.max(1, Math.trunc(loadKbConfig(kbRoot).bm25TitleWeight ?? 1))
+  // loadKbConfig guarantees a finite integer >= 1; cap the upper bound so a large
+  // value can't materialize a giant per-page array (Array(w).fill) and OOM retrieval.
+  const w = Math.min(25, loadKbConfig(kbRoot).bm25TitleWeight)
   const pages = listWikiPages(kbRoot).filter(p => !p.error && !isInvalidated(p))
   const idx = buildBm25Index(pages.map(p => ({
     id: p.relPath,

package/src/convert-run.mjs

@@ -69,7 +69,7 @@ export async function runConvertPlan(kbRoot) {
       convertedAt: new Date().toISOString().slice(0, 10),
       ...(originalRel ? { original: originalRel } : {}),
     }
-    converted.push({ src: rel, raw: rawRel })
+    converted.push({ src: rel, raw: rawRel, ...(warnings.length ? { warnings } : {}) })
   }
   saveManifest(kbRoot, manifest)
   return { converted, failed }

package/src/convert.mjs

@@ -19,6 +19,19 @@ function titleFrom(markdown, fallback) {
   return m ? m[1].trim() : fallback
 }
 
+// A source can convert successfully yet yield no text — a scanned/image-only PDF, an
+// empty DOCX, a blank .txt. That is still a converted page (markdown '' is not null, so
+// the deliberate empty-file handling downstream is untouched), but writing a blank raw
+// page while reporting "converted" silently misleads the user. Attach a warning the CLI
+// can surface. `# heading`-prefixed HTML output never trims to empty, so this fires only
+// on genuinely contentless extractions.
+function result(markdown, title, warnings) {
+  if (markdown !== null && markdown.trim() === '') {
+    warnings.push('converted to an empty page — no extractable text (scanned/image-only PDF or empty document?)')
+  }
+  return { markdown, title, warnings }
+}
+
 export async function convertFile(srcPath) {
   const ext = path.extname(srcPath).toLowerCase()
   const base = path.basename(srcPath)
@@ -26,11 +39,11 @@ export async function convertFile(srcPath) {
   try {
     if (ext === '.md' || ext === '.markdown') {
       const md = fs.readFileSync(srcPath, 'utf8')
-      return { markdown: md, title: titleFrom(md, base), warnings }
+      return result(md, titleFrom(md, base), warnings)
     }
     if (ext === '.txt') {
       const md = fs.readFileSync(srcPath, 'utf8')
-      return { markdown: md, title: titleFrom(md, base), warnings }
+      return result(md, titleFrom(md, base), warnings)
     }
     if (ext === '.html' || ext === '.htm') {
       const { JSDOM } = await import('jsdom')
@@ -45,17 +58,17 @@ export async function convertFile(srcPath) {
       }
       const td = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' })
       const md = td.turndown(article.content)
-      return { markdown: `# ${article.title || base}\n\n${md}`, title: article.title || base, warnings }
+      return result(`# ${article.title || base}\n\n${md}`, article.title || base, warnings)
     }
     if (ext === '.pdf') {
       const { PDFParse } = await import('pdf-parse')
       const { text } = await new PDFParse({ data: fs.readFileSync(srcPath) }).getText()
-      return { markdown: text, title: titleFrom(text, base), warnings }
+      return result(text, titleFrom(text, base), warnings)
     }
     if (ext === '.docx') {
       const mammoth = await import('mammoth')
       const { value } = await mammoth.extractRawText({ path: srcPath })
-      return { markdown: value, title: titleFrom(value, base), warnings }
+      return result(value, titleFrom(value, base), warnings)
     }
     warnings.push(`unsupported extension: ${ext}`)
     return { markdown: null, title: base, warnings }

package/src/indexer.mjs

@@ -3,6 +3,7 @@ import path from 'node:path'
 import { kbPaths } from './paths.mjs'
 import { loadKbConfig } from './templates.mjs'
 import { listWikiPages, isInvalidated, asList, RELATION_CONFIDENCES } from './pages.mjs'
+import { writeFileAtomic } from './json.mjs'
 
 const WIKILINK_RE = /\[\[([^\]|#]+)(?:[#|][^\]]*)?\]\]/g
 
@@ -69,7 +70,7 @@ export function buildIndex(kbRoot) {
     const written = new Set()
     for (const [type, list] of Object.entries(byType)) {
       if (!list.length) continue
-      fs.writeFileSync(path.join(p.topics, `${type}.md`), `# ${SECTION_TITLES[type]}\n\n${list.map(line).join('\n')}\n`)
+      writeFileAtomic(path.join(p.topics, `${type}.md`), `# ${SECTION_TITLES[type]}\n\n${list.map(line).join('\n')}\n`)
       written.add(`${type}.md`)
       indexBody += `\n## ${SECTION_TITLES[type]}\nSee [[topics/${type}]] (${list.length} pages)\n`
     }
@@ -81,7 +82,7 @@ export function buildIndex(kbRoot) {
       indexBody += `\n## ${SECTION_TITLES[type]}\n${list.map(line).join('\n')}\n`
     }
   }
-  fs.writeFileSync(p.indexMd, `${indexBody}\n${pending}${tail}`)
+  writeFileAtomic(p.indexMd, `${indexBody}\n${pending}${tail}`)
 
   const ids = new Set(pages.map(pg => pg.relPath.replace(/\.md$/, '')))
   const nodes = pages.map(pg => ({
@@ -114,12 +115,12 @@ export function buildIndex(kbRoot) {
       edges.push({ source: id, target, type: String(rel.type), confidence })
     }
   }
-  fs.writeFileSync(p.graphJson, JSON.stringify({ nodes, edges }, null, 2) + '\n')
+  writeFileAtomic(p.graphJson, JSON.stringify({ nodes, edges }, null, 2) + '\n')
 
   const kbName = path.basename(path.resolve(kbRoot))
   const llms = [`# ${kbName}`, '', `> llm_wiki knowledge base. Read wiki/index.md first, then open full pages.`, '', '## Pages',
     ...pages.filter(pg => !isInvalidated(pg)).map(pg => `- [${inline(pg.data.title)}](wiki/${pg.relPath}): ${inline(pg.data.description)}`)].join('\n') + '\n'
-  fs.writeFileSync(p.llmsTxt, llms)
+  writeFileAtomic(p.llmsTxt, llms)
 
   return { pageCount: pages.length, topicsSplit }
 }

package/src/json.mjs

@@ -14,3 +14,15 @@ export function readJsonFile(file, { redactContents = false } = {}) {
     throw new Error(redactContents ? `${file}: invalid JSON` : `${file}: invalid JSON (${err.message})`)
   }
 }
+
+// Atomic file write: `fs.writeFileSync` truncates the target *before* writing, so a
+// concurrent reader (a long-lived MCP server, an `ask` fallback) can observe an empty
+// or half-written file — for JSON stores that means a `JSON.parse` crash. Write a temp
+// sibling and rename (atomic on the POSIX filesystems this tool supports) so a reader
+// only ever sees the whole old file or the whole new one. Single source of the pattern
+// that manifest/vector already used and buildIndex's derived stores previously missed.
+export function writeFileAtomic(file, data) {
+  const tmp = `${file}.tmp`
+  fs.writeFileSync(tmp, data)
+  fs.renameSync(tmp, file)
+}

package/src/manifest.mjs

@@ -1,6 +1,6 @@
 import fs from 'node:fs'
 import { kbPaths } from './paths.mjs'
-import { readJsonFile } from './json.mjs'
+import { readJsonFile, writeFileAtomic } from './json.mjs'
 
 export function loadManifest(kbRoot) {
   const p = kbPaths(kbRoot)
@@ -13,11 +13,9 @@ export function loadManifest(kbRoot) {
 
 export function saveManifest(kbRoot, manifest) {
   const p = kbPaths(kbRoot)
-  // Atomic write: manifest is non-derived state (hash->raw map). A crash mid-write
-  // must not leave a truncated/corrupt file. Write a temp sibling, then rename.
-  const tmp = `${p.manifest}.tmp`
-  fs.writeFileSync(tmp, JSON.stringify(manifest, null, 2) + '\n')
-  fs.renameSync(tmp, p.manifest)
+  // Atomic write (shared writeFileAtomic): manifest is non-derived state (hash->raw
+  // map) and a crash or concurrent reader must never see a truncated/corrupt file.
+  writeFileAtomic(p.manifest, JSON.stringify(manifest, null, 2) + '\n')
 }
 
 export function diffManifest(manifest, entries) {

package/src/templates.mjs

@@ -17,12 +17,28 @@ export const DEFAULT_CONFIG = {
   relationTypes: ['implements', 'uses', 'depends_on', 'part_of', 'instance_of', 'derived_from', 'contrasts_with', 'causes'],
 }
 
+// Keys whose default is a number are consumed as scalars — an Array(n) length
+// (bm25TitleWeight), a loop stride (batchSize), or a threshold comparison. Because
+// wiki.config.json is shallow-merged as-is (any JSON type per key), a scalar there
+// can be a string/object/NaN/0/negative, which crashes (`Array(NaN)` RangeError),
+// hangs (`i += 0` never advances the batch loop), or silently disables a gate.
+const NUMERIC_KEYS = Object.keys(DEFAULT_CONFIG).filter((k) => typeof DEFAULT_CONFIG[k] === 'number')
+
 // Single reader for wiki.config.json (defaults merged) — every consumer used
 // to inline this parse, each with its own bare-SyntaxError failure mode.
 export function loadKbConfig(kbRoot) {
   const p = kbPaths(kbRoot)
-  if (!fs.existsSync(p.config)) return DEFAULT_CONFIG
-  return { ...DEFAULT_CONFIG, ...readJsonFile(p.config) }
+  if (!fs.existsSync(p.config)) return { ...DEFAULT_CONFIG }
+  const merged = { ...DEFAULT_CONFIG, ...readJsonFile(p.config) }
+  // Coerce every numeric key back to a usable finite integer >= 1; fall back to the
+  // default on anything a user could type that isn't one. Every DEFAULT is itself a
+  // positive integer, so a legitimate override is untouched — only malformed values
+  // (which would crash/hang/mis-gate a consumer) are normalized away.
+  for (const k of NUMERIC_KEYS) {
+    const n = Math.trunc(Number(merged[k]))
+    merged[k] = Number.isFinite(n) && n >= 1 ? n : DEFAULT_CONFIG[k]
+  }
+  return merged
 }
 
 export function agentsMdTemplate(cfg) {

package/src/vector.mjs

@@ -1,7 +1,7 @@
 import fs from 'node:fs'
 import path from 'node:path'
 import { kbPaths } from './paths.mjs'
-import { readJsonFile } from './json.mjs'
+import { readJsonFile, writeFileAtomic } from './json.mjs'
 import { asList } from './pages.mjs'
 
 export function normalize(vec) {
@@ -37,13 +37,10 @@ export function loadVectorStore(kbRoot) {
 export function saveVectorStore(kbRoot, store) {
   const rounded = { ...store, pages: Object.fromEntries(Object.entries(store.pages).map(([id, e]) =>
     [id, { hash: e.hash, vec: e.vec.map(x => Number(x.toFixed(5))) }])) }
-  // Atomic write: the per-batch flush in embedKb re-saves repeatedly; a crash
-  // mid-write must not leave a truncated store (loadVectorStore fails open, but
-  // temp+rename avoids the corruption entirely).
-  const f = vectorStorePath(kbRoot)
-  const tmp = `${f}.tmp`
-  fs.writeFileSync(tmp, JSON.stringify(rounded) + '\n')
-  fs.renameSync(tmp, f)
+  // Atomic write (shared writeFileAtomic): the per-batch flush in embedKb re-saves
+  // repeatedly; a crash or concurrent reader must never see a truncated store
+  // (loadVectorStore fails open, but temp+rename avoids the corruption entirely).
+  writeFileAtomic(vectorStorePath(kbRoot), JSON.stringify(rounded) + '\n')
 }
 
 // queryVec must already be normalized; stored vecs are normalized at save time,