@sdsrs/llm-wiki 0.7.1 → 0.7.3

package/CHANGELOG.md

@@ -1,5 +1,55 @@
 # Changelog
 
+## 0.7.3 (2026-07-12)
+
+Two robustness fixes from the final targeted dogfooding sweep. No config schema
+change, no KB migration, no retrieval-default change. Suite 228 → 230.
+
+**Fixes:**
+
+- **`graph`/exports name the file on a corrupt `graph.json`.** `loadGraph` was the
+  last JSON reader still using a bare `JSON.parse`: a corrupt `wiki/graph.json`
+  threw an unqualified `Unexpected token` SyntaxError, and a valid-JSON-but-wrong-
+  shape file like `{}` crashed on `graph.nodes.map`. It now routes through the
+  shared `readJsonFile` (names the file) and guards the nodes/edges shape, so both
+  corruption modes yield a named, actionable "rerun `llm-wiki index`" error.
+- **`.scan-plan.json` is written atomically.** `scan` wrote the plan with a direct
+  `writeFileSync`, but `convert` reads it from a separate process — the same
+  truncate-then-write torn read that 0.7.2 fixed for `graph.json`. It (and
+  `.lint-report.json`) now write via the shared temp-and-rename `writeFileAtomic`,
+  so every JSON state file another process can read is crash-safe.
+
+## 0.7.2 (2026-07-12)
+
+Three defects found by continued end-to-end dogfooding — two robustness/DoS
+guards plus one UX fix. No config schema change, no KB migration, no
+retrieval-default change. Suite 221 → 228.
+
+**Fixes:**
+
+- **Malformed numeric config can no longer crash retrieval or hang scan.**
+  `wiki.config.json` is shallow-merged as-is, so a numeric key could arrive as a
+  string/0/negative/NaN and reach a scalar consumer: `bm25TitleWeight: "high"`
+  hit `Array(NaN)` → `RangeError` on *every* `ask`/`search`/MCP query (a huge int
+  → per-page `Array(n).fill` OOM), and `batchSize: 0` made `scan`'s batch loop
+  (`i += batchSize`) spin forever. `loadKbConfig` now coerces every numeric key
+  back to a finite integer ≥ 1 (falling back to its default), and
+  `bm25TitleWeight` is additionally capped at its consumer. Legitimate overrides
+  are untouched.
+- **Derived stores are written atomically.** `buildIndex` wrote `graph.json` /
+  `index.md` / `llms.txt` / topic files with a direct `writeFileSync`, which
+  truncates the target before writing — a reader running concurrently (a
+  long-lived MCP `graph` tool, `ask`'s `llms.txt` fallback) could read a torn
+  `graph.json` and crash on `JSON.parse`. All derived stores now write via a
+  shared temp-and-rename `writeFileAtomic` (the pattern `saveManifest` /
+  `saveVectorStore` already used), so a concurrent reader only ever sees the whole
+  old or whole new file.
+- **A source that converts to an empty page is no longer silent.** A
+  scanned/image-only PDF, an empty DOCX, or a blank `.txt` extracts to no text;
+  `convert` still writes the (empty) raw page and counts it as converted, but now
+  prints `WARN <src>: converted to an empty page — no extractable text` so the
+  user isn't misled into thinking real content was ingested.
+
 ## 0.7.1 (2026-07-11)
 
 Four defects found by end-to-end dogfooding — one crash-class fix plus three

package/bin/llm-wiki.mjs

@@ -50,6 +50,9 @@ program.command('convert')
     const r = await runConvertPlan(opts.kb)
     console.log(`converted ${r.converted.length}, failed ${r.failed.length}`)
     for (const f of r.failed) console.log(`  FAILED ${f.src}: ${f.warnings.join('; ')}`)
+    // A converted-but-empty page (scanned PDF, empty DOCX) is not a failure, but the
+    // user must see it — an unflagged blank raw page reads as "ingested" when it isn't.
+    for (const c of r.converted) if (c.warnings) console.log(`  WARN ${c.src}: ${c.warnings.join('; ')}`)
     // Total failure (nothing converted, at least one error) is a hard error a
     // pipeline must catch — exit 1. A partial success keeps exit 0 so a messy dir
     // with a few unconvertible junk files doesn't break an otherwise-good run.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sdsrs/llm-wiki",
-  "version": "0.7.1",
+  "version": "0.7.3",
   "description": "Compile messy document directories into Karpathy-style llm_wiki knowledge bases — standalone Q&A CLI + Claude Code/Codex skills",
   "directories": {
     "test": "test"

package/src/ask.mjs

@@ -15,7 +15,9 @@ export function retrievePages(kbRoot, question, k = 6) {
   // title term outweighs the same term buried in the body (measured +0.04 Recall@5
   // on the dogfood KB at ×3). Set bm25TitleWeight: 1 in wiki.config.json to restore
   // flat single-field indexing.
-  const w = Math.max(1, Math.trunc(loadKbConfig(kbRoot).bm25TitleWeight ?? 1))
+  // loadKbConfig guarantees a finite integer >= 1; cap the upper bound so a large
+  // value can't materialize a giant per-page array (Array(w).fill) and OOM retrieval.
+  const w = Math.min(25, loadKbConfig(kbRoot).bm25TitleWeight)
   const pages = listWikiPages(kbRoot).filter(p => !p.error && !isInvalidated(p))
   const idx = buildBm25Index(pages.map(p => ({
     id: p.relPath,

package/src/convert-run.mjs

@@ -69,7 +69,7 @@ export async function runConvertPlan(kbRoot) {
       convertedAt: new Date().toISOString().slice(0, 10),
       ...(originalRel ? { original: originalRel } : {}),
     }
-    converted.push({ src: rel, raw: rawRel })
+    converted.push({ src: rel, raw: rawRel, ...(warnings.length ? { warnings } : {}) })
   }
   saveManifest(kbRoot, manifest)
   return { converted, failed }

package/src/convert.mjs

@@ -19,6 +19,19 @@ function titleFrom(markdown, fallback) {
   return m ? m[1].trim() : fallback
 }
 
+// A source can convert successfully yet yield no text — a scanned/image-only PDF, an
+// empty DOCX, a blank .txt. That is still a converted page (markdown '' is not null, so
+// the deliberate empty-file handling downstream is untouched), but writing a blank raw
+// page while reporting "converted" silently misleads the user. Attach a warning the CLI
+// can surface. `# heading`-prefixed HTML output never trims to empty, so this fires only
+// on genuinely contentless extractions.
+function result(markdown, title, warnings) {
+  if (markdown !== null && markdown.trim() === '') {
+    warnings.push('converted to an empty page — no extractable text (scanned/image-only PDF or empty document?)')
+  }
+  return { markdown, title, warnings }
+}
+
 export async function convertFile(srcPath) {
   const ext = path.extname(srcPath).toLowerCase()
   const base = path.basename(srcPath)
@@ -26,11 +39,11 @@ export async function convertFile(srcPath) {
   try {
     if (ext === '.md' || ext === '.markdown') {
       const md = fs.readFileSync(srcPath, 'utf8')
-      return { markdown: md, title: titleFrom(md, base), warnings }
+      return result(md, titleFrom(md, base), warnings)
     }
     if (ext === '.txt') {
       const md = fs.readFileSync(srcPath, 'utf8')
-      return { markdown: md, title: titleFrom(md, base), warnings }
+      return result(md, titleFrom(md, base), warnings)
     }
     if (ext === '.html' || ext === '.htm') {
       const { JSDOM } = await import('jsdom')
@@ -45,17 +58,17 @@ export async function convertFile(srcPath) {
       }
       const td = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' })
       const md = td.turndown(article.content)
-      return { markdown: `# ${article.title || base}\n\n${md}`, title: article.title || base, warnings }
+      return result(`# ${article.title || base}\n\n${md}`, article.title || base, warnings)
     }
     if (ext === '.pdf') {
       const { PDFParse } = await import('pdf-parse')
       const { text } = await new PDFParse({ data: fs.readFileSync(srcPath) }).getText()
-      return { markdown: text, title: titleFrom(text, base), warnings }
+      return result(text, titleFrom(text, base), warnings)
     }
     if (ext === '.docx') {
       const mammoth = await import('mammoth')
       const { value } = await mammoth.extractRawText({ path: srcPath })
-      return { markdown: value, title: titleFrom(value, base), warnings }
+      return result(value, titleFrom(value, base), warnings)
     }
     warnings.push(`unsupported extension: ${ext}`)
     return { markdown: null, title: base, warnings }

package/src/export.mjs

@@ -2,6 +2,7 @@ import fs from 'node:fs'
 import path from 'node:path'
 import { kbPaths } from './paths.mjs'
 import { listWikiPages } from './pages.mjs'
+import { readJsonFile } from './json.mjs'
 
 const pkg = JSON.parse(fs.readFileSync(new URL('../package.json', import.meta.url), 'utf8'))
 
@@ -16,7 +17,18 @@ const cyEscape = (s) => String(s).replace(/\\/g, '\\\\').replace(/\n/g, '\\n').r
 export function loadGraph(kbRoot) {
   const p = kbPaths(kbRoot)
   if (!fs.existsSync(p.graphJson)) throw new Error('wiki/graph.json not found — run `llm-wiki index` first.')
-  const graph = JSON.parse(fs.readFileSync(p.graphJson, 'utf8'))
+  // graph.json is a derived store — route it through readJsonFile (names the file)
+  // rather than a bare JSON.parse whose SyntaxError leaks no path, and guard the shape
+  // so a hand-corrupted-but-valid `{}` doesn't crash on `graph.nodes.map` (undefined).
+  let graph
+  try {
+    graph = readJsonFile(p.graphJson)
+  } catch (err) {
+    throw new Error(`${err.message} — rerun \`llm-wiki index\` to rebuild the graph`)
+  }
+  if (!graph || !Array.isArray(graph.nodes) || !Array.isArray(graph.edges)) {
+    throw new Error('wiki/graph.json has an unexpected shape (needs nodes/edges arrays) — rerun `llm-wiki index`')
+  }
   const ids = new Set(graph.nodes.map(n => n.id))
   for (const e of graph.edges) {
     for (const end of [e.source, e.target]) {

package/src/indexer.mjs

@@ -3,6 +3,7 @@ import path from 'node:path'
 import { kbPaths } from './paths.mjs'
 import { loadKbConfig } from './templates.mjs'
 import { listWikiPages, isInvalidated, asList, RELATION_CONFIDENCES } from './pages.mjs'
+import { writeFileAtomic } from './json.mjs'
 
 const WIKILINK_RE = /\[\[([^\]|#]+)(?:[#|][^\]]*)?\]\]/g
 
@@ -69,7 +70,7 @@ export function buildIndex(kbRoot) {
     const written = new Set()
     for (const [type, list] of Object.entries(byType)) {
       if (!list.length) continue
-      fs.writeFileSync(path.join(p.topics, `${type}.md`), `# ${SECTION_TITLES[type]}\n\n${list.map(line).join('\n')}\n`)
+      writeFileAtomic(path.join(p.topics, `${type}.md`), `# ${SECTION_TITLES[type]}\n\n${list.map(line).join('\n')}\n`)
       written.add(`${type}.md`)
       indexBody += `\n## ${SECTION_TITLES[type]}\nSee [[topics/${type}]] (${list.length} pages)\n`
     }
@@ -81,7 +82,7 @@ export function buildIndex(kbRoot) {
       indexBody += `\n## ${SECTION_TITLES[type]}\n${list.map(line).join('\n')}\n`
     }
   }
-  fs.writeFileSync(p.indexMd, `${indexBody}\n${pending}${tail}`)
+  writeFileAtomic(p.indexMd, `${indexBody}\n${pending}${tail}`)
 
   const ids = new Set(pages.map(pg => pg.relPath.replace(/\.md$/, '')))
   const nodes = pages.map(pg => ({
@@ -114,12 +115,12 @@ export function buildIndex(kbRoot) {
       edges.push({ source: id, target, type: String(rel.type), confidence })
     }
   }
-  fs.writeFileSync(p.graphJson, JSON.stringify({ nodes, edges }, null, 2) + '\n')
+  writeFileAtomic(p.graphJson, JSON.stringify({ nodes, edges }, null, 2) + '\n')
 
   const kbName = path.basename(path.resolve(kbRoot))
   const llms = [`# ${kbName}`, '', `> llm_wiki knowledge base. Read wiki/index.md first, then open full pages.`, '', '## Pages',
     ...pages.filter(pg => !isInvalidated(pg)).map(pg => `- [${inline(pg.data.title)}](wiki/${pg.relPath}): ${inline(pg.data.description)}`)].join('\n') + '\n'
-  fs.writeFileSync(p.llmsTxt, llms)
+  writeFileAtomic(p.llmsTxt, llms)
 
   return { pageCount: pages.length, topicsSplit }
 }

package/src/json.mjs

@@ -14,3 +14,15 @@ export function readJsonFile(file, { redactContents = false } = {}) {
     throw new Error(redactContents ? `${file}: invalid JSON` : `${file}: invalid JSON (${err.message})`)
   }
 }
+
+// Atomic file write: `fs.writeFileSync` truncates the target *before* writing, so a
+// concurrent reader (a long-lived MCP server, an `ask` fallback) can observe an empty
+// or half-written file — for JSON stores that means a `JSON.parse` crash. Write a temp
+// sibling and rename (atomic on the POSIX filesystems this tool supports) so a reader
+// only ever sees the whole old file or the whole new one. Single source of the pattern
+// that manifest/vector already used and buildIndex's derived stores previously missed.
+export function writeFileAtomic(file, data) {
+  const tmp = `${file}.tmp`
+  fs.writeFileSync(tmp, data)
+  fs.renameSync(tmp, file)
+}

package/src/lint.mjs

@@ -5,6 +5,7 @@ import { loadKbConfig } from './templates.mjs'
 import { listWikiPages, validatePage, isInvalidated, asList, PAGE_STATUSES, RELATION_CONFIDENCES } from './pages.mjs'
 import { extractWikilinks, buildIndex } from './indexer.mjs'
 import { loadManifest } from './manifest.mjs'
+import { writeFileAtomic } from './json.mjs'
 
 export async function lintKb(kbRoot, { fix = false } = {}) {
   const p = kbPaths(kbRoot)
@@ -126,6 +127,8 @@ export async function lintKb(kbRoot, { fix = false } = {}) {
 
   if (fix) { buildIndex(kbRoot); autoFixed.push('index-rebuilt') }
   const report = { autoFixed, mechanical, semantic }
-  fs.writeFileSync(path.join(kbRoot, '.lint-report.json'), JSON.stringify(report, null, 2) + '\n')
+  // Atomic write for uniformity with the other JSON state files (temp+rename). The
+  // report has no in-process reader today, so this is preventive, not a live torn-read.
+  writeFileAtomic(path.join(kbRoot, '.lint-report.json'), JSON.stringify(report, null, 2) + '\n')
   return report
 }

package/src/manifest.mjs

@@ -1,6 +1,6 @@
 import fs from 'node:fs'
 import { kbPaths } from './paths.mjs'
-import { readJsonFile } from './json.mjs'
+import { readJsonFile, writeFileAtomic } from './json.mjs'
 
 export function loadManifest(kbRoot) {
   const p = kbPaths(kbRoot)
@@ -13,11 +13,9 @@ export function loadManifest(kbRoot) {
 
 export function saveManifest(kbRoot, manifest) {
   const p = kbPaths(kbRoot)
-  // Atomic write: manifest is non-derived state (hash->raw map). A crash mid-write
-  // must not leave a truncated/corrupt file. Write a temp sibling, then rename.
-  const tmp = `${p.manifest}.tmp`
-  fs.writeFileSync(tmp, JSON.stringify(manifest, null, 2) + '\n')
-  fs.renameSync(tmp, p.manifest)
+  // Atomic write (shared writeFileAtomic): manifest is non-derived state (hash->raw
+  // map) and a crash or concurrent reader must never see a truncated/corrupt file.
+  writeFileAtomic(p.manifest, JSON.stringify(manifest, null, 2) + '\n')
 }
 
 export function diffManifest(manifest, entries) {

package/src/scanner.mjs

@@ -6,6 +6,7 @@ import { SUPPORTED_EXTS } from './convert.mjs'
 import { sha256File, minhashSignature, jaccardEstimate } from './hashing.mjs'
 import { loadManifest, diffManifest } from './manifest.mjs'
 import { listWikiPages, isInvalidated } from './pages.mjs'
+import { writeFileAtomic } from './json.mjs'
 
 const TEXT_EXTS = ['.md', '.markdown', '.txt', '.html', '.htm']
 const NEAR_DUP_THRESHOLD = 0.85
@@ -169,6 +170,9 @@ export async function scanSource(srcDir, kbRoot, { exclude = [], persist = true,
     batches,
     estimate,
   }
-  if (persist) fs.writeFileSync(kbPaths(kbRoot).scanPlan, JSON.stringify(report, null, 2) + '\n')
+  // Atomic write: `convert` reads .scan-plan.json (runConvertPlan) as a separate
+  // process, so a direct writeFileSync (truncate-then-write) could hand it a torn
+  // file. Same torn-read class as buildIndex's derived stores.
+  if (persist) writeFileAtomic(kbPaths(kbRoot).scanPlan, JSON.stringify(report, null, 2) + '\n')
   return report
 }

package/src/templates.mjs

@@ -17,12 +17,28 @@ export const DEFAULT_CONFIG = {
   relationTypes: ['implements', 'uses', 'depends_on', 'part_of', 'instance_of', 'derived_from', 'contrasts_with', 'causes'],
 }
 
+// Keys whose default is a number are consumed as scalars — an Array(n) length
+// (bm25TitleWeight), a loop stride (batchSize), or a threshold comparison. Because
+// wiki.config.json is shallow-merged as-is (any JSON type per key), a scalar there
+// can be a string/object/NaN/0/negative, which crashes (`Array(NaN)` RangeError),
+// hangs (`i += 0` never advances the batch loop), or silently disables a gate.
+const NUMERIC_KEYS = Object.keys(DEFAULT_CONFIG).filter((k) => typeof DEFAULT_CONFIG[k] === 'number')
+
 // Single reader for wiki.config.json (defaults merged) — every consumer used
 // to inline this parse, each with its own bare-SyntaxError failure mode.
 export function loadKbConfig(kbRoot) {
   const p = kbPaths(kbRoot)
-  if (!fs.existsSync(p.config)) return DEFAULT_CONFIG
-  return { ...DEFAULT_CONFIG, ...readJsonFile(p.config) }
+  if (!fs.existsSync(p.config)) return { ...DEFAULT_CONFIG }
+  const merged = { ...DEFAULT_CONFIG, ...readJsonFile(p.config) }
+  // Coerce every numeric key back to a usable finite integer >= 1; fall back to the
+  // default on anything a user could type that isn't one. Every DEFAULT is itself a
+  // positive integer, so a legitimate override is untouched — only malformed values
+  // (which would crash/hang/mis-gate a consumer) are normalized away.
+  for (const k of NUMERIC_KEYS) {
+    const n = Math.trunc(Number(merged[k]))
+    merged[k] = Number.isFinite(n) && n >= 1 ? n : DEFAULT_CONFIG[k]
+  }
+  return merged
 }
 
 export function agentsMdTemplate(cfg) {

package/src/vector.mjs

@@ -1,7 +1,7 @@
 import fs from 'node:fs'
 import path from 'node:path'
 import { kbPaths } from './paths.mjs'
-import { readJsonFile } from './json.mjs'
+import { readJsonFile, writeFileAtomic } from './json.mjs'
 import { asList } from './pages.mjs'
 
 export function normalize(vec) {
@@ -37,13 +37,10 @@ export function loadVectorStore(kbRoot) {
 export function saveVectorStore(kbRoot, store) {
   const rounded = { ...store, pages: Object.fromEntries(Object.entries(store.pages).map(([id, e]) =>
     [id, { hash: e.hash, vec: e.vec.map(x => Number(x.toFixed(5))) }])) }
-  // Atomic write: the per-batch flush in embedKb re-saves repeatedly; a crash
-  // mid-write must not leave a truncated store (loadVectorStore fails open, but
-  // temp+rename avoids the corruption entirely).
-  const f = vectorStorePath(kbRoot)
-  const tmp = `${f}.tmp`
-  fs.writeFileSync(tmp, JSON.stringify(rounded) + '\n')
-  fs.renameSync(tmp, f)
+  // Atomic write (shared writeFileAtomic): the per-batch flush in embedKb re-saves
+  // repeatedly; a crash or concurrent reader must never see a truncated store
+  // (loadVectorStore fails open, but temp+rename avoids the corruption entirely).
+  writeFileAtomic(vectorStorePath(kbRoot), JSON.stringify(rounded) + '\n')
 }
 
 // queryVec must already be normalized; stored vecs are normalized at save time,