@sdsrs/llm-wiki 0.7.3 → 0.7.4

package/CHANGELOG.md

@@ -1,5 +1,29 @@
 # Changelog
 
+## 0.7.4 (2026-07-12)
+
+Two robustness fixes surfaced by an adversarial QA pass over the CLI. No config
+schema change, no KB migration, no retrieval-default change. Suite 230 → 234.
+
+**Fixes:**
+
+- **`index` warns when a page is skipped for invalid frontmatter.** `buildIndex`
+  silently excluded any page with unparseable YAML frontmatter from
+  `index.md`/`graph.json`/`llms.txt`. Because the docs route hand-editors to
+  `index` (not `lint`) after editing, a broken-YAML edit made the page vanish
+  from every query surface with no signal — `indexed N pages`, exit 0, nothing
+  else. `buildIndex` now returns the skipped pages and the CLI prints a one-line
+  stderr warning naming them and pointing at `lint`; stdout stays the clean
+  count, and `lint` still owns the detailed per-page report.
+- **`scan` near-duplicate detection uses 128 minhash permutations (was 32).**
+  Near-dup similarity was estimated from a 32-permutation minhash and compared
+  against the 0.85 threshold, but at 32 perms the estimate's standard error
+  (~0.05) let a genuine near-duplicate (true Jaccard 0.898) estimate 0.844 and
+  be reported as `near: 0`. 128 perms tightens the estimate (SD ~0.026) so real
+  near-dups clear the threshold; the cost is trivial next to the per-file
+  sha256+read scan already does, and the signature is stripped before the plan
+  is persisted (no on-disk change).
+
 ## 0.7.3 (2026-07-12)
 
 Two robustness fixes from the final targeted dogfooding sweep. No config schema

package/bin/llm-wiki.mjs

@@ -65,6 +65,12 @@ program.command('index')
   .action((opts) => {
     const r = buildIndex(opts.kb)
     console.log(`indexed ${r.pageCount} pages${r.topicsSplit ? ' (split into topics/)' : ''}`)
+    // A page with unparseable frontmatter is excluded from index.md/graph.json/llms.txt. Surface
+    // it on stderr (stdout stays the clean count) so a hand-edit that broke the YAML doesn't
+    // silently vanish — the README points hand-editors here, not at `lint`.
+    if (r.skipped?.length) {
+      console.error(`warning: skipped ${r.skipped.length} page(s) with invalid frontmatter (run 'lint' for details): ${r.skipped.map(s => s.relPath).join(', ')}`)
+    }
   })
 
 program.command('embed')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sdsrs/llm-wiki",
-  "version": "0.7.3",
+  "version": "0.7.4",
   "description": "Compile messy document directories into Karpathy-style llm_wiki knowledge bases — standalone Q&A CLI + Claude Code/Codex skills",
   "directories": {
     "test": "test"

package/src/hashing.mjs

@@ -13,7 +13,12 @@ function fnv1a(str) {
   return h
 }
 
-export function minhashSignature(text, perms = 32) {
+// 128 permutations, not 32: the near-dup estimate's standard error is ~sqrt(J(1-J)/perms),
+// so at 32 perms a genuine near-duplicate (true Jaccard ~0.90) estimated as low as 0.84 and
+// fell below NEAR_DUP_THRESHOLD (0.85), silently missing it (QA73-002). 128 perms tightens the
+// estimate (SD ~0.026) so real near-dups clear the threshold; the cost is trivial next to the
+// per-file sha256+read scan already does, and `_sig` is stripped before the plan is persisted.
+export function minhashSignature(text, perms = 128) {
   const norm = text.toLowerCase().replace(/\s+/g, ' ')
   const shingles = new Set()
   for (let i = 0; i <= norm.length - 5; i++) shingles.add(norm.slice(i, i + 5))

package/src/indexer.mjs

@@ -26,7 +26,13 @@ const inline = (s) => String(s ?? '').replace(/[\r\n]+/g, ' ')
 export function buildIndex(kbRoot) {
   const p = kbPaths(kbRoot)
   const cfg = loadKbConfig(kbRoot)
-  const pages = listWikiPages(kbRoot).filter(pg => !pg.error)
+  const allPages = listWikiPages(kbRoot)
+  const pages = allPages.filter(pg => !pg.error)
+  // Pages with unparseable/missing frontmatter are excluded from every derived file. Return
+  // them so the CLI can warn instead of silently dropping a page — the README routes
+  // hand-editors to `index` (not `lint`), so a bad edit would otherwise vanish with no signal
+  // (QA73-001). lint still owns the detailed per-page report.
+  const skipped = allPages.filter(pg => pg.error).map(pg => ({ relPath: pg.relPath, error: pg.error }))
 
   // Preserve the pending section, bounded at the next `## ` heading — a greedy
   // match to EOF would swallow user-added sections into "pending" forever.
@@ -122,5 +128,5 @@ export function buildIndex(kbRoot) {
     ...pages.filter(pg => !isInvalidated(pg)).map(pg => `- [${inline(pg.data.title)}](wiki/${pg.relPath}): ${inline(pg.data.description)}`)].join('\n') + '\n'
   writeFileAtomic(p.llmsTxt, llms)
 
-  return { pageCount: pages.length, topicsSplit }
+  return { pageCount: pages.length, topicsSplit, skipped }
 }