@sdsrs/llm-wiki 0.7.2 → 0.7.4

package/CHANGELOG.md

@@ -1,5 +1,48 @@
 # Changelog
 
+## 0.7.4 (2026-07-12)
+
+Two robustness fixes surfaced by an adversarial QA pass over the CLI. No config
+schema change, no KB migration, no retrieval-default change. Suite 230 → 234.
+
+**Fixes:**
+
+- **`index` warns when a page is skipped for invalid frontmatter.** `buildIndex`
+  silently excluded any page with unparseable YAML frontmatter from
+  `index.md`/`graph.json`/`llms.txt`. Because the docs route hand-editors to
+  `index` (not `lint`) after editing, a broken-YAML edit made the page vanish
+  from every query surface with no signal — `indexed N pages`, exit 0, nothing
+  else. `buildIndex` now returns the skipped pages and the CLI prints a one-line
+  stderr warning naming them and pointing at `lint`; stdout stays the clean
+  count, and `lint` still owns the detailed per-page report.
+- **`scan` near-duplicate detection uses 128 minhash permutations (was 32).**
+  Near-dup similarity was estimated from a 32-permutation minhash and compared
+  against the 0.85 threshold, but at 32 perms the estimate's standard error
+  (~0.05) let a genuine near-duplicate (true Jaccard 0.898) estimate 0.844 and
+  be reported as `near: 0`. 128 perms tightens the estimate (SD ~0.026) so real
+  near-dups clear the threshold; the cost is trivial next to the per-file
+  sha256+read scan already does, and the signature is stripped before the plan
+  is persisted (no on-disk change).
+
+## 0.7.3 (2026-07-12)
+
+Two robustness fixes from the final targeted dogfooding sweep. No config schema
+change, no KB migration, no retrieval-default change. Suite 228 → 230.
+
+**Fixes:**
+
+- **`graph`/exports name the file on a corrupt `graph.json`.** `loadGraph` was the
+  last JSON reader still using a bare `JSON.parse`: a corrupt `wiki/graph.json`
+  threw an unqualified `Unexpected token` SyntaxError, and a valid-JSON-but-wrong-
+  shape file like `{}` crashed on `graph.nodes.map`. It now routes through the
+  shared `readJsonFile` (names the file) and guards the nodes/edges shape, so both
+  corruption modes yield a named, actionable "rerun `llm-wiki index`" error.
+- **`.scan-plan.json` is written atomically.** `scan` wrote the plan with a direct
+  `writeFileSync`, but `convert` reads it from a separate process — the same
+  truncate-then-write torn read that 0.7.2 fixed for `graph.json`. It (and
+  `.lint-report.json`) now write via the shared temp-and-rename `writeFileAtomic`,
+  so every JSON state file another process can read is crash-safe.
+
 ## 0.7.2 (2026-07-12)
 
 Three defects found by continued end-to-end dogfooding — two robustness/DoS

package/bin/llm-wiki.mjs

@@ -65,6 +65,12 @@ program.command('index')
   .action((opts) => {
     const r = buildIndex(opts.kb)
     console.log(`indexed ${r.pageCount} pages${r.topicsSplit ? ' (split into topics/)' : ''}`)
+    // A page with unparseable frontmatter is excluded from index.md/graph.json/llms.txt. Surface
+    // it on stderr (stdout stays the clean count) so a hand-edit that broke the YAML doesn't
+    // silently vanish — the README points hand-editors here, not at `lint`.
+    if (r.skipped?.length) {
+      console.error(`warning: skipped ${r.skipped.length} page(s) with invalid frontmatter (run 'lint' for details): ${r.skipped.map(s => s.relPath).join(', ')}`)
+    }
   })
 
 program.command('embed')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sdsrs/llm-wiki",
-  "version": "0.7.2",
+  "version": "0.7.4",
   "description": "Compile messy document directories into Karpathy-style llm_wiki knowledge bases — standalone Q&A CLI + Claude Code/Codex skills",
   "directories": {
     "test": "test"

package/src/export.mjs

@@ -2,6 +2,7 @@ import fs from 'node:fs'
 import path from 'node:path'
 import { kbPaths } from './paths.mjs'
 import { listWikiPages } from './pages.mjs'
+import { readJsonFile } from './json.mjs'
 
 const pkg = JSON.parse(fs.readFileSync(new URL('../package.json', import.meta.url), 'utf8'))
 
@@ -16,7 +17,18 @@ const cyEscape = (s) => String(s).replace(/\\/g, '\\\\').replace(/\n/g, '\\n').r
 export function loadGraph(kbRoot) {
   const p = kbPaths(kbRoot)
   if (!fs.existsSync(p.graphJson)) throw new Error('wiki/graph.json not found — run `llm-wiki index` first.')
-  const graph = JSON.parse(fs.readFileSync(p.graphJson, 'utf8'))
+  // graph.json is a derived store — route it through readJsonFile (names the file)
+  // rather than a bare JSON.parse whose SyntaxError leaks no path, and guard the shape
+  // so a hand-corrupted-but-valid `{}` doesn't crash on `graph.nodes.map` (undefined).
+  let graph
+  try {
+    graph = readJsonFile(p.graphJson)
+  } catch (err) {
+    throw new Error(`${err.message} — rerun \`llm-wiki index\` to rebuild the graph`)
+  }
+  if (!graph || !Array.isArray(graph.nodes) || !Array.isArray(graph.edges)) {
+    throw new Error('wiki/graph.json has an unexpected shape (needs nodes/edges arrays) — rerun `llm-wiki index`')
+  }
   const ids = new Set(graph.nodes.map(n => n.id))
   for (const e of graph.edges) {
     for (const end of [e.source, e.target]) {

package/src/hashing.mjs

@@ -13,7 +13,12 @@ function fnv1a(str) {
   return h
 }
 
-export function minhashSignature(text, perms = 32) {
+// 128 permutations, not 32: the near-dup estimate's standard error is ~sqrt(J(1-J)/perms),
+// so at 32 perms a genuine near-duplicate (true Jaccard ~0.90) estimated as low as 0.84 and
+// fell below NEAR_DUP_THRESHOLD (0.85), silently missing it (QA73-002). 128 perms tightens the
+// estimate (SD ~0.026) so real near-dups clear the threshold; the cost is trivial next to the
+// per-file sha256+read scan already does, and `_sig` is stripped before the plan is persisted.
+export function minhashSignature(text, perms = 128) {
   const norm = text.toLowerCase().replace(/\s+/g, ' ')
   const shingles = new Set()
   for (let i = 0; i <= norm.length - 5; i++) shingles.add(norm.slice(i, i + 5))

package/src/indexer.mjs

@@ -26,7 +26,13 @@ const inline = (s) => String(s ?? '').replace(/[\r\n]+/g, ' ')
 export function buildIndex(kbRoot) {
   const p = kbPaths(kbRoot)
   const cfg = loadKbConfig(kbRoot)
-  const pages = listWikiPages(kbRoot).filter(pg => !pg.error)
+  const allPages = listWikiPages(kbRoot)
+  const pages = allPages.filter(pg => !pg.error)
+  // Pages with unparseable/missing frontmatter are excluded from every derived file. Return
+  // them so the CLI can warn instead of silently dropping a page — the README routes
+  // hand-editors to `index` (not `lint`), so a bad edit would otherwise vanish with no signal
+  // (QA73-001). lint still owns the detailed per-page report.
+  const skipped = allPages.filter(pg => pg.error).map(pg => ({ relPath: pg.relPath, error: pg.error }))
 
   // Preserve the pending section, bounded at the next `## ` heading — a greedy
   // match to EOF would swallow user-added sections into "pending" forever.
@@ -122,5 +128,5 @@ export function buildIndex(kbRoot) {
     ...pages.filter(pg => !isInvalidated(pg)).map(pg => `- [${inline(pg.data.title)}](wiki/${pg.relPath}): ${inline(pg.data.description)}`)].join('\n') + '\n'
   writeFileAtomic(p.llmsTxt, llms)
 
-  return { pageCount: pages.length, topicsSplit }
+  return { pageCount: pages.length, topicsSplit, skipped }
 }

package/src/lint.mjs

@@ -5,6 +5,7 @@ import { loadKbConfig } from './templates.mjs'
 import { listWikiPages, validatePage, isInvalidated, asList, PAGE_STATUSES, RELATION_CONFIDENCES } from './pages.mjs'
 import { extractWikilinks, buildIndex } from './indexer.mjs'
 import { loadManifest } from './manifest.mjs'
+import { writeFileAtomic } from './json.mjs'
 
 export async function lintKb(kbRoot, { fix = false } = {}) {
   const p = kbPaths(kbRoot)
@@ -126,6 +127,8 @@ export async function lintKb(kbRoot, { fix = false } = {}) {
 
   if (fix) { buildIndex(kbRoot); autoFixed.push('index-rebuilt') }
   const report = { autoFixed, mechanical, semantic }
-  fs.writeFileSync(path.join(kbRoot, '.lint-report.json'), JSON.stringify(report, null, 2) + '\n')
+  // Atomic write for uniformity with the other JSON state files (temp+rename). The
+  // report has no in-process reader today, so this is preventive, not a live torn-read.
+  writeFileAtomic(path.join(kbRoot, '.lint-report.json'), JSON.stringify(report, null, 2) + '\n')
   return report
 }

package/src/scanner.mjs

@@ -6,6 +6,7 @@ import { SUPPORTED_EXTS } from './convert.mjs'
 import { sha256File, minhashSignature, jaccardEstimate } from './hashing.mjs'
 import { loadManifest, diffManifest } from './manifest.mjs'
 import { listWikiPages, isInvalidated } from './pages.mjs'
+import { writeFileAtomic } from './json.mjs'
 
 const TEXT_EXTS = ['.md', '.markdown', '.txt', '.html', '.htm']
 const NEAR_DUP_THRESHOLD = 0.85
@@ -169,6 +170,9 @@ export async function scanSource(srcDir, kbRoot, { exclude = [], persist = true,
     batches,
     estimate,
   }
-  if (persist) fs.writeFileSync(kbPaths(kbRoot).scanPlan, JSON.stringify(report, null, 2) + '\n')
+  // Atomic write: `convert` reads .scan-plan.json (runConvertPlan) as a separate
+  // process, so a direct writeFileSync (truncate-then-write) could hand it a torn
+  // file. Same torn-read class as buildIndex's derived stores.
+  if (persist) writeFileAtomic(kbPaths(kbRoot).scanPlan, JSON.stringify(report, null, 2) + '\n')
   return report
 }