@sdsrs/llm-wiki 0.9.1 → 0.9.2

package/CHANGELOG.md

@@ -1,5 +1,45 @@
 # Changelog
 
+## 0.9.2 (2026-07-12)
+
+Robustness and UX fixes from an end-to-end usage audit; no change to core retrieval or
+convert behavior. Suite 271 → 279.
+
+**Fixed:**
+
+- **Concurrent writes to a derived file no longer crash.** `writeFileAtomic` used a fixed
+  temp name, so two writers to the same file (concurrent `index`, or the MCP server and a
+  CLI command touching the same derived file) collided — the second renamed a temp the
+  first had already consumed and died with ENOENT. Each write now uses a unique temp; the
+  rename onto the target stays atomic, so readers still never see a torn write.
+- **`graph` accepts `--kb` on the parent command** (`graph --kb ./kb hubs`), like every
+  other command — it previously errored "unknown option --kb" unless `--kb` came after the
+  subcommand.
+- **`ask` rejects an empty/whitespace question** instead of running a no-op search.
+- **`ask --retrieve-only` tells an empty wiki apart from a no-match query** — it no longer
+  tells a user with a populated KB to go build pages they already have.
+- **`connect` reports a clear error for a missing/non-directory project path** instead of
+  leaking a raw ENOENT for an internal file.
+- **`scan` no longer reports a phantom "+1 to convert" for an exact-duplicate file** — the
+  incremental count now agrees with the compile plan (duplicates are reported separately).
+- **Clearer frontmatter error**: a page that opens `---` but never closes it is reported as
+  `unterminated-frontmatter`, not `missing-frontmatter`.
+- **Clearer `embed` error** when no embedding model is configured (names the field to set,
+  for both remote and local).
+
+**Added (lint):**
+
+- `type-dir-mismatch` — flags a page whose `type` disagrees with its directory (e.g.
+  `type: source` in `concepts/`), which the indexer would otherwise silently miscategorize.
+- `self-supersede` — flags a page whose `superseded_by` points at itself.
+
+**Changed (internal):**
+
+- `loadEmbedConfig` reads the global config once per call and resolves a provider-level
+  embedding model in priority order, consistent with chat provider selection.
+- Local embedding no longer builds a network transport (no undici import / proxy agent),
+  since it makes no network calls.
+
 ## 0.9.1 (2026-07-12)
 
 Correctness and diagnostics for the 0.9.0 local-embedding path; no change to the

package/bin/llm-wiki.mjs

@@ -89,10 +89,16 @@ program.command('ask <question>')
   .action(async (question, opts) => {
     const k = Number.parseInt(opts.k, 10)
     if (!Number.isFinite(k) || k < 1) { console.error(`invalid -k value: ${opts.k} (expected a positive integer)`); process.exit(1) }
+    if (!question.trim()) { console.error('empty question — provide something to ask'); process.exit(1) }
     const r = await askKb(opts.kb, question, { k, retrieveOnly: opts.retrieveOnly })
     if (opts.retrieveOnly) {
       if (r.pages.length === 0) {
-        console.error('no pages located — the KB has no matching wiki pages (build them with the wiki-build skill, or check that wiki/ contains pages)')
+        // Distinguish "wiki is empty" (build pages) from "wiki has pages but none
+        // matched" (rephrase) — the old fixed message told users with a full KB to go
+        // build pages they already have.
+        console.error(r.kbEmpty
+          ? 'no pages located — the wiki has no pages yet (build them with the wiki-build skill, or check that wiki/ contains pages)'
+          : 'no pages located — no wiki page matched your query (the wiki has pages; try different or fewer search terms)')
         return
       }
       for (const h of r.pages) console.log(`${h.score.toFixed(3)}  ${h.relPath}  [${(h.sources ?? ['bm25']).join('+')}]`)
@@ -164,12 +170,17 @@ program.command('export')
   })
 
 const graphCmd = program.command('graph').description('query wiki/graph.json: path | neighbors | hubs (zero-LLM traversal)')
+  // Accept --kb on the parent too, so `graph --kb ./kb hubs` works like every other
+  // command (`ask --kb`, `lint --kb`). Subcommands keep their own --kb (no default) and
+  // fall back to the parent's, so both placements resolve and neither silently wins '.'.
+  .option('--kb <dir>', 'knowledge base root', '.')
+const graphKb = (opts) => opts.kb ?? graphCmd.opts().kb
 
 graphCmd.command('path <from> <to>')
   .description('shortest link chain between two page ids (either link direction)')
-  .option('--kb <dir>', 'knowledge base root', '.')
+  .option('--kb <dir>', 'knowledge base root')
   .action((from, to, opts) => {
-    const r = shortestPath(loadGraph(opts.kb), from, to)
+    const r = shortestPath(loadGraph(graphKb(opts)), from, to)
     if (!r) { console.log(`no path between ${from} and ${to}`); return }
     console.log(r.nodes[0])
     for (const h of r.hops) console.log(`  ${h.dir === 'out' ? `-[${h.type}]->` : `<-[${h.type}]-`} ${h.to}${h.confidence ? `  (${h.confidence})` : ''}${h.status === 'invalidated' ? '  ⚠ invalidated' : ''}`)
@@ -178,11 +189,11 @@ graphCmd.command('path <from> <to>')
 graphCmd.command('neighbors <id>')
   .description('pages within N hops (links counted in both directions)')
   .option('-d, --depth <n>', 'expansion depth', '1')
-  .option('--kb <dir>', 'knowledge base root', '.')
+  .option('--kb <dir>', 'knowledge base root')
   .action((id, opts) => {
     const depth = Number.parseInt(opts.depth, 10)
     if (!Number.isFinite(depth) || depth < 1) { console.error(`invalid depth: ${opts.depth} (expected a positive integer)`); process.exit(1) }
-    const r = neighborhood(loadGraph(opts.kb), id, depth)
+    const r = neighborhood(loadGraph(graphKb(opts)), id, depth)
     if (!r.length) { console.log(`${id} has no linked neighbors`); return }
     for (const n of r) console.log(`d=${n.distance}  ${n.id}  [${n.type}${n.confidence ? '/' + n.confidence : ''} ${n.dir}]${n.status === 'invalidated' ? '  ⚠ invalidated' : ''}`)
   })
@@ -190,11 +201,11 @@ graphCmd.command('neighbors <id>')
 graphCmd.command('hubs')
   .description('most-connected pages (degree ranking; raw files excluded)')
   .option('--top <n>', 'how many to show', '10')
-  .option('--kb <dir>', 'knowledge base root', '.')
+  .option('--kb <dir>', 'knowledge base root')
   .action((opts) => {
     const top = Number.parseInt(opts.top, 10)
     if (!Number.isFinite(top) || top < 1) { console.error(`invalid --top: ${opts.top} (expected a positive integer)`); process.exit(1) }
-    for (const h of hubs(loadGraph(opts.kb), { top })) {
+    for (const h of hubs(loadGraph(graphKb(opts)), { top })) {
       console.log(`${String(h.degree).padStart(3)}  ${h.id}  (in ${h.in} / out ${h.out})  ${h.title}${h.status === 'invalidated' ? '  ⚠ invalidated' : ''}`)
     }
   })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sdsrs/llm-wiki",
-  "version": "0.9.1",
+  "version": "0.9.2",
   "description": "Compile messy document directories into Karpathy-style llm_wiki knowledge bases — standalone Q&A CLI + Claude Code/Codex skills",
   "directories": {
     "test": "test"

package/src/ask.mjs

@@ -8,6 +8,7 @@ import { worstCaseTokens } from './scanner.mjs'
 import { loadLlmConfig, loadEmbedConfig, makeTransport } from './llm-config.mjs'
 import { loadVectorStore, normalize, cosineTopK } from './vector.mjs'
 import { embedTexts } from './embed.mjs'
+import { isLocalModel } from './local-embed.mjs'
 import { fetchWithRetry } from './retry.mjs'
 
 // Two per-KB caches, both keyed by a cheap page-set freshness token so neither can
@@ -149,7 +150,11 @@ export async function locatePages(kbRoot, question, { k = 6, fetchImpl, retrieva
     const injected = fetchImpl || pipelineFactory
     const t = injected
       ? { fetchImpl, dispatcher: undefined, retry, pipelineFactory }
-      : { ...(await makeTransport()), retry }
+      // A local query embedding makes no network call (embedLocal ignores the
+      // transport), so skip makeTransport — no undici import / proxy agent for nothing.
+      : isLocalModel(cfg.embeddingModel)
+        ? { retry }
+        : { ...(await makeTransport()), retry }
     const [qv] = await embedTexts(cfg, t, [question], { role: 'query' })
     const qn = normalize(qv)
     const vecHits = qn ? cosineTopK(qn, store, k).map(v => ({ relPath: v.id, score: v.score })) : []
@@ -232,7 +237,9 @@ async function pickPagesFromListing(p, question, k, cfg, t, validIds) {
 export async function askKb(kbRoot, question, { k = 6, retrieveOnly = false, fetchImpl, retrieval = 'auto', retry, pipelineFactory } = {}) {
   const p = kbPaths(kbRoot)
   let { hits } = await locatePages(kbRoot, question, { k, fetchImpl, retrieval, retry, pipelineFactory })
-  if (retrieveOnly) return { pages: hits, answer: null }
+  // kbEmpty lets the CLI tell "no pages built yet" apart from "pages exist but none
+  // matched". Only pay the livePages listing when there were zero hits to explain.
+  if (retrieveOnly) return { pages: hits, answer: null, kbEmpty: hits.length === 0 && livePages(kbRoot).length === 0 }
   let validIds
   if (hits.length === 0) {
     validIds = new Set(livePages(kbRoot).map(pg => pg.relPath.replace(/\.md$/, '')))

package/src/connect.mjs

@@ -15,6 +15,10 @@ function renderBlock(kbs) {
 }
 
 export function connectProject(projectDir, { kb, role = 'project', remove = false }) {
+  // Fail with a clear message instead of leaking a raw ENOENT for an internal file
+  // (.llm-wiki.json) when the user points connect at a missing/typo'd project path.
+  if (!fs.existsSync(projectDir)) throw new Error(`project directory not found: ${projectDir}`)
+  if (!fs.statSync(projectDir).isDirectory()) throw new Error(`not a directory: ${projectDir}`)
   const regFile = path.join(projectDir, '.llm-wiki.json')
   const regExists = fs.existsSync(regFile)
   const registry = regExists ? readJsonFile(regFile) : { kbs: [] }

package/src/embed.mjs

@@ -120,7 +120,11 @@ export async function embedKb(kbRoot, { fetchImpl, retry, pipelineFactory } = {}
     const injected = fetchImpl || pipelineFactory
     const t = injected
       ? { fetchImpl, dispatcher: undefined, retry, pipelineFactory }
-      : { ...(await makeTransport()), retry }
+      // A local model makes no network call (embedLocal ignores the transport), so
+      // skip makeTransport — it would import undici / build a proxy agent for nothing.
+      : local
+        ? { retry }
+        : { ...(await makeTransport()), retry }
     for (let i = 0; i < jobs.length; i += BATCH) {
       const batch = jobs.slice(i, i + BATCH)
       const vecs = await embedTexts(cfg, t, batch.map(j => j.text))

package/src/frontmatter.mjs

@@ -1,10 +1,17 @@
 import YAML from 'yaml'
 
 const FM_RE = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?/
+const FM_OPEN_RE = /^---\r?\n/
 
 export function parseFrontmatter(text) {
   const m = text.match(FM_RE)
-  if (!m) return null
+  if (!m) {
+    // Opens with a `---` delimiter but never closes it — an unterminated block, not
+    // an absent one. A forgotten closing `---` is a common authoring slip; report it
+    // distinctly so the fix is "close the frontmatter", not "add frontmatter".
+    if (FM_OPEN_RE.test(text)) return { error: 'unterminated-frontmatter' }
+    return null
+  }
   let data
   try { data = YAML.parse(m[1]) } catch { return { error: 'invalid-yaml' } }
   if (data === null || typeof data !== 'object') return { error: 'invalid-yaml' }

package/src/json.mjs

@@ -1,4 +1,9 @@
 import fs from 'node:fs'
+import { randomBytes } from 'node:crypto'
+
+// Per-write temp counter: combined with the pid it keeps concurrent writers to the
+// same target on distinct temp files even within one process (see writeFileAtomic).
+let atomicSeq = 0
 
 // JSON.parse with the offending file named in the error — a bare SyntaxError
 // from a corrupt state file gives the user nothing to act on.
@@ -22,7 +27,19 @@ export function readJsonFile(file, { redactContents = false } = {}) {
 // only ever sees the whole old file or the whole new one. Single source of the pattern
 // that manifest/vector already used and buildIndex's derived stores previously missed.
 export function writeFileAtomic(file, data) {
-  const tmp = `${file}.tmp`
-  fs.writeFileSync(tmp, data)
-  fs.renameSync(tmp, file)
+  // Unique temp name per write. A fixed `${file}.tmp` makes two concurrent writers to
+  // the same target share one temp file: the first rename consumes it, the second then
+  // renames a now-missing file and crashes with ENOENT (concurrent `index`, or MCP +
+  // CLI touching the same derived file). pid + seq + random keeps temps independent.
+  // The rename onto `file` stays atomic, so readers still never see a torn write.
+  const tmp = `${file}.${process.pid}.${(atomicSeq++).toString(36)}.${randomBytes(4).toString('hex')}.tmp`
+  try {
+    fs.writeFileSync(tmp, data)
+    fs.renameSync(tmp, file)
+  } catch (err) {
+    // Unique temps don't self-overwrite on the next run, so a failed write must clean
+    // up after itself or it leaks a stray .tmp beside the target.
+    try { fs.unlinkSync(tmp) } catch { /* best-effort */ }
+    throw err
+  }
 }

package/src/lint.mjs

@@ -7,6 +7,13 @@ import { extractWikilinks, buildIndex } from './indexer.mjs'
 import { loadManifest } from './manifest.mjs'
 import { writeFileAtomic } from './json.mjs'
 
+// The AGENTS.md contract binds each page type to its directory (source↔sources/, …).
+// A page whose `type` disagrees with its directory is silently miscategorized: the
+// indexer groups index.md sections by `type`, but ids / wikilinks / graph nodes use
+// the directory — so e.g. a `type: source` page in concepts/ is filed under "Sources"
+// yet linked as `concepts/…`.
+const DIR_TYPE = { sources: 'source', entities: 'entity', concepts: 'concept', comparisons: 'comparison' }
+
 export async function lintKb(kbRoot, { fix = false } = {}) {
   const p = kbPaths(kbRoot)
   const cfg = loadKbConfig(kbRoot)
@@ -20,6 +27,12 @@ export async function lintKb(kbRoot, { fix = false } = {}) {
   for (const pg of pages) {
     if (pg.error) { mechanical.push({ rule: 'invalid-frontmatter', path: pg.relPath, detail: pg.error }); continue }
     for (const issue of validatePage(pg)) mechanical.push({ rule: 'missing-field', path: pg.relPath, detail: issue })
+    // type must match the directory (AGENTS.md contract) — a mismatch is miscategorized
+    // by the indexer. Skip when type is absent (already flagged as a missing field).
+    const dir = pg.relPath.split('/')[0]
+    if (DIR_TYPE[dir] && pg.data.type !== undefined && pg.data.type !== DIR_TYPE[dir]) {
+      mechanical.push({ rule: 'type-dir-mismatch', path: pg.relPath, detail: `type "${pg.data.type}" in ${dir}/ (expected "${DIR_TYPE[dir]}")` })
+    }
     for (const target of extractWikilinks(pg.body)) {
       if (target.startsWith('raw/')) {
         if (!fs.existsSync(path.join(kbRoot, target)) && !fs.existsSync(path.join(kbRoot, target + '.md'))) mechanical.push({ rule: 'broken-raw-link', path: pg.relPath, detail: `-> ${target}` })
@@ -34,8 +47,15 @@ export async function lintKb(kbRoot, { fix = false } = {}) {
     if (pg.data.status !== undefined && !PAGE_STATUSES.includes(pg.data.status)) {
       mechanical.push({ rule: 'invalid-status', path: pg.relPath, detail: `status "${pg.data.status}" (expected ${PAGE_STATUSES.join(' | ')})` })
     }
-    if (pg.data.superseded_by !== undefined && !ids.has(String(pg.data.superseded_by))) {
-      mechanical.push({ rule: 'superseded-target-missing', path: pg.relPath, detail: `superseded_by -> ${pg.data.superseded_by}` })
+    if (pg.data.superseded_by !== undefined) {
+      const supTarget = String(pg.data.superseded_by)
+      // A page cannot be its own replacement: superseded_by == self invalidates the page
+      // with no valid successor (checked before target-missing, since the self id exists).
+      if (supTarget === pg.relPath.replace(/\.md$/, '')) {
+        mechanical.push({ rule: 'self-supersede', path: pg.relPath, detail: 'superseded_by points at itself' })
+      } else if (!ids.has(supTarget)) {
+        mechanical.push({ rule: 'superseded-target-missing', path: pg.relPath, detail: `superseded_by -> ${pg.data.superseded_by}` })
+      }
     }
     if (pg.data.relations !== undefined && !Array.isArray(pg.data.relations)) {
       mechanical.push({ rule: 'invalid-relation-entry', path: pg.relPath, detail: 'relations must be a YAML list' })

package/src/llm-config.mjs

@@ -23,12 +23,36 @@ function resolveProviders(fileCfg) {
   return null
 }
 
-export function loadLlmConfig(kbRoot) {
+// The embeddingModel a credential-less (local) config would use: the flat top-level
+// form wins, else the first provider IN PRIORITY ORDER that declares one — the same
+// ordering resolveProviders uses, so the chat and embed paths can't disagree on which
+// provider wins. Only file-configured providers are scanned (BUILTIN declares none).
+function resolveEmbeddingModel(fileCfg) {
+  if (typeof fileCfg.embeddingModel === 'string') return fileCfg.embeddingModel
+  const { priority, providers } = fileCfg
+  if (!providers) return null
+  for (const name of priority ?? Object.keys(providers)) {
+    const em = providers[name]?.embeddingModel
+    if (em) return em
+  }
+  return null
+}
+
+// Read ~/.llm-wiki/config.json once per entry point. redactContents: this file holds
+// API keys — a corrupt-JSON error must not echo a fragment of it to the terminal.
+function readGlobalConfig() {
   const dir = process.env.LLM_WIKI_CONFIG_DIR ?? path.join(os.homedir(), '.llm-wiki')
   const globalFile = path.join(dir, 'config.json')
-  // redactContents: this file holds API keys — a corrupt-JSON error must not
-  // echo a fragment of it to the terminal.
-  const fileCfg = fs.existsSync(globalFile) ? readJsonFile(globalFile, { redactContents: true }) : {}
+  return fs.existsSync(globalFile) ? readJsonFile(globalFile, { redactContents: true }) : {}
+}
+
+export function loadLlmConfig(kbRoot) {
+  return loadChatConfig(kbRoot, readGlobalConfig())
+}
+
+// Resolve a chat-complete config from an already-read global fileCfg (+ the KB's
+// optional llm override). Returns null unless baseURL/apiKey/model are all present.
+function loadChatConfig(kbRoot, fileCfg) {
   // flat form: explicit custom endpoint wins outright
   let cfg = (fileCfg.baseURL && fileCfg.apiKey && fileCfg.model)
     ? { baseURL: fileCfg.baseURL, apiKey: fileCfg.apiKey, model: fileCfg.model, ...(fileCfg.embeddingModel ? { embeddingModel: fileCfg.embeddingModel } : {}) }
@@ -72,14 +96,10 @@ export function loadLlmConfig(kbRoot) {
 // call the API, so it resolves only when loadLlmConfig does. loadLlmConfig's own
 // (chat-complete-or-null) contract is unchanged.
 export function loadEmbedConfig(kbRoot) {
-  const chat = loadLlmConfig(kbRoot)
+  const fileCfg = readGlobalConfig()
+  const chat = loadChatConfig(kbRoot, fileCfg)
   if (chat?.embeddingModel) return chat
-  const dir = process.env.LLM_WIKI_CONFIG_DIR ?? path.join(os.homedir(), '.llm-wiki')
-  const globalFile = path.join(dir, 'config.json')
-  if (!fs.existsSync(globalFile)) return null
-  const fileCfg = readJsonFile(globalFile, { redactContents: true })
-  const em = fileCfg.embeddingModel
-    ?? Object.values(fileCfg.providers ?? {}).map(p => p?.embeddingModel).find(Boolean)
+  const em = resolveEmbeddingModel(fileCfg)
   return (typeof em === 'string' && em.startsWith('local:')) ? { embeddingModel: em } : null
 }
 

package/src/scanner.mjs

@@ -150,6 +150,12 @@ export async function scanSource(srcDir, kbRoot, { exclude = [], persist = true,
   const diff = diffManifest(loadManifest(kbRoot), files.map(f => ({ rel: f.rel, hash: f.hash })))
   const toCompileSet = new Set([...diff.added, ...diff.changed].map(e => e.rel))
   const toCompile = files.filter(f => toCompileSet.has(f.rel) && !exactDups.has(f.rel))
+  // Exact duplicates are excluded from the compile batches (toCompile), so they must not
+  // inflate the reported added/changed counts either — a deduped file is never written to
+  // the manifest, so it would otherwise read as a permanent "+1 to convert" that `convert`
+  // never clears and that contradicts the "0 batches" plan. Dups stay in `duplicates.exact`.
+  const incAdded = diff.added.filter(e => !exactDups.has(e.rel)).length
+  const incChanged = diff.changed.filter(e => !exactDups.has(e.rel)).length
   const batches = []
   for (let i = 0; i < toCompile.length; i += cfg.batchSize) {
     batches.push(toCompile.slice(i, i + cfg.batchSize).map(f => f.rel))
@@ -165,7 +171,7 @@ export async function scanSource(srcDir, kbRoot, { exclude = [], persist = true,
     files: files.map(({ _sig, ...f }) => f),
     skipped,
     duplicates: { exact, near },
-    incremental: { added: diff.added.length, changed: diff.changed.length, removed: diff.removed.length, unchanged: diff.unchanged.length },
+    incremental: { added: incAdded, changed: incChanged, removed: diff.removed.length, unchanged: diff.unchanged.length },
     domainMixture: detectDomainMixture(files, kbRoot),
     batches,
     estimate,