@sdsrs/llm-wiki 0.8.2 → 0.9.0

package/CHANGELOG.md

@@ -1,5 +1,43 @@
 # Changelog
 
+## 0.9.0 (2026-07-12)
+
+**Added:**
+
+- **Local / offline embedding via transformers.js.** Set `"embeddingModel":
+  "local:Xenova/multilingual-e5-small"` in `~/.llm-wiki/config.json` and install the
+  optional `@huggingface/transformers` to generate and query vectors with no embedding
+  API. With only a local model configured, `embed` / `search` / `ask --retrieve-only`
+  run fully offline (answering with `ask` still needs a chat provider). The default
+  model is multilingual, so zh↔en cross-language retrieval is preserved. The dependency
+  is optional — the base install is unchanged.
+
+## 0.8.3 (2026-07-12)
+
+Performance and test-hardening cleanup; no shipped behavior change. Suite 261 → 263.
+
+**Changed:**
+
+- **Retrieval parses the wiki page set once per query, not up to three times.** The
+  BM25 build, the vector-hit valid set, and the index-listing fallback's id set now
+  share one page cache keyed by the same mtime + size freshness token as the BM25
+  index (added in 0.8.2), so any add / edit / removal / invalidation still busts it
+  and retired knowledge can never resurface. On a long-lived MCP server a vector
+  (`auto`/`hybrid`) query drops its second full page parse; `locatePages` also reads
+  `wiki.config.json` once instead of twice.
+- **The `mcp` command loads the MCP SDK + zod lazily.** They were imported at CLI
+  startup for every command; only `mcp` needs them. `scan` / `convert` / `ask` /
+  `init` and friends now skip ~200–250 ms of import cost.
+
+**Internal (not shipped in the npm package):**
+
+- Atomic-write regression guards: `saveManifest`, `saveVectorStore`, and the `scan`
+  plan write now assert the temp + rename path via a `fs.renameSync` spy — the old
+  `.tmp`-absence check alone also passed for a direct truncating write (mem #10097).
+- The full eval report (`scripts/eval/run-all.mjs`) judges the shipped default
+  retrieval mode: answer-level head-to-head now defaults to `bm25,auto` (was the
+  hardcoded `bm25,hybrid`) and takes an `--answer-arms` flag.
+
 ## 0.8.2 (2026-07-12)
 
 Performance, no behavior change. Suite 260 → 261.

package/README.md

@@ -144,6 +144,22 @@ similarity (RRF); `--retrieve-only` labels each hit `[bm25]`, `[vector]` or
 `[bm25+vector]`. Vectors only *locate* pages — context is still whole pages,
 never chunks. Any vector-path failure falls back to BM25 with a warning.
 
+### Local / offline embedding
+
+Vector retrieval normally calls an embedding API. To run it locally with no API key:
+
+1. Install the optional model runtime: `npm i @huggingface/transformers`
+2. In `~/.llm-wiki/config.json`, set a `local:` embedding model:
+   ```json
+   { "embeddingModel": "local:Xenova/multilingual-e5-small" }
+   ```
+3. `llm-wiki embed --kb ./kb` — the first run downloads the model (~120 MB).
+
+The default `Xenova/multilingual-e5-small` is multilingual, so cross-language
+(zh↔en) retrieval keeps working. With only a local embedding model configured,
+`embed`, `search`, and `ask --retrieve-only` run fully offline; generating an
+answer with `ask` still needs a chat provider.
+
 ### Graph queries
 
 `graph.json` (rebuilt by `index`) is a link graph over the pages — wikilinks,

package/bin/llm-wiki.mjs

@@ -14,7 +14,6 @@ import { statusKb } from '../src/status.mjs'
 import { connectProject, installSkills } from '../src/connect.mjs'
 import { exportGraph, loadGraph, exportMarkdownPages } from '../src/export.mjs'
 import { shortestPath, neighborhood, hubs } from '../src/graph.mjs'
-import { runMcpServer } from '../src/mcp.mjs'
 
 const pkg = JSON.parse(fs.readFileSync(new URL('../package.json', import.meta.url), 'utf8'))
 
@@ -203,6 +202,9 @@ program.command('mcp')
   .description('run a read-only MCP server (stdio) over the knowledge base')
   .option('--kb <dir>', 'knowledge base root', '.')
   .action(async (opts) => {
+    // Lazy import: mcp.mjs pulls in the MCP SDK + zod, which no other command needs.
+    // Static-importing it would load them on every scan/convert/ask invocation.
+    const { runMcpServer } = await import('../src/mcp.mjs')
     // stdout belongs to the MCP transport from here on — no console.log.
     await runMcpServer(opts.kb)
   })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sdsrs/llm-wiki",
-  "version": "0.8.2",
+  "version": "0.9.0",
   "description": "Compile messy document directories into Karpathy-style llm_wiki knowledge bases — standalone Q&A CLI + Claude Code/Codex skills",
   "directories": {
     "test": "test"
@@ -51,5 +51,13 @@
     "undici": "^8.7.0",
     "yaml": "^2.9.0",
     "zod": "^4.4.3"
+  },
+  "peerDependencies": {
+    "@huggingface/transformers": "^3.0.0"
+  },
+  "peerDependenciesMeta": {
+    "@huggingface/transformers": {
+      "optional": true
+    }
   }
 }

package/src/ask.mjs

@@ -5,29 +5,26 @@ import { loadKbConfig } from './templates.mjs'
 import { listWikiPages, isInvalidated, asList, PAGE_DIRS } from './pages.mjs'
 import { buildBm25Index, searchBm25 } from './bm25.mjs'
 import { worstCaseTokens } from './scanner.mjs'
-import { loadLlmConfig, makeTransport } from './llm-config.mjs'
+import { loadLlmConfig, loadEmbedConfig, makeTransport } from './llm-config.mjs'
 import { loadVectorStore, normalize, cosineTopK } from './vector.mjs'
 import { embedTexts } from './embed.mjs'
 import { fetchWithRetry } from './retry.mjs'
 
-// One built BM25 index per KB root, reused across queries while the wiki pages and
-// title-weight config are unchanged. The cost being cached is reading + tokenizing
-// every page: a CLI process builds it once and exits (cache is transparent there), but
-// a long-lived MCP server otherwise rebuilds from disk on every wiki_search. Bounded so
-// a process serving many KBs can't grow it without limit.
-const bm25Cache = new Map()
-const BM25_CACHE_MAX = 8
+// Two per-KB caches, both keyed by a cheap page-set freshness token so neither can
+// serve a stale view. The token is each live page file's mtime+size: any page
+// add / remove / in-place edit / invalidation (a frontmatter status change IS a
+// content change) flips it and forces a rebuild. Statting the page set is far cheaper
+// than reading + parsing it; MCP is read-only and never mutates pages while serving, so
+// within one server's lifetime pages change only via a separate build process (which
+// rewrites files with new mtimes). Both caches are bounded so a process serving many
+// KBs can't grow them without limit.
+const bm25Cache = new Map()   // kbRoot -> { token: `${pagesToken}|w=${w}`, idx }
+const pagesCache = new Map()  // kbRoot -> { token: pagesToken, pages }
+const CACHE_MAX = 8
 
-// Cheap freshness token: each live page file's mtime+size plus the title weight. Any
-// page add / remove / in-place edit / invalidation (a frontmatter status change IS a
-// content change) or config change flips the token and forces a rebuild — so the cache
-// can never serve a stale index. Statting the page set is far cheaper than reading and
-// tokenizing it; MCP is read-only and never mutates pages while serving, so within one
-// server's lifetime pages change only via a separate build process (which rewrites
-// files with new mtimes).
-function pageSetToken(kbRoot, w) {
+function pagesToken(kbRoot) {
   const wiki = kbPaths(kbRoot).wiki
-  const parts = [`w=${w}`]
+  const parts = []
   for (const dir of PAGE_DIRS) {
     let entries
     try { entries = fs.readdirSync(path.join(wiki, dir)) } catch { continue }
@@ -42,6 +39,23 @@ function pageSetToken(kbRoot, w) {
   return parts.join('|')
 }
 
+// The parsed, filtered live-page list — the dominant per-query cost (listWikiPages
+// reads + frontmatter-parses every page), shared by the BM25 build, the vector-hit
+// valid set, and the listing fallback's validIds. Excluding invalidated pages here is
+// the query-time exclusion every consumer needs, done once. Callers on the same query
+// pass the token already computed for the BM25 cache; standalone callers let it default.
+function livePages(kbRoot, token = pagesToken(kbRoot)) {
+  let entry = pagesCache.get(kbRoot)
+  if (!entry || entry.token !== token) {
+    const pages = listWikiPages(kbRoot).filter(p => !p.error && !isInvalidated(p))
+    pagesCache.delete(kbRoot) // re-insert so this KB becomes most-recent for LRU eviction
+    pagesCache.set(kbRoot, { token, pages })
+    while (pagesCache.size > CACHE_MAX) pagesCache.delete(pagesCache.keys().next().value)
+    entry = pagesCache.get(kbRoot)
+  }
+  return entry.pages
+}
+
 export function retrievePages(kbRoot, question, k = 6) {
   // Title is a field: repeat it bm25TitleWeight times in the indexed text so a
   // title term outweighs the same term buried in the body (measured +0.04 Recall@5
@@ -50,17 +64,17 @@ export function retrievePages(kbRoot, question, k = 6) {
   // loadKbConfig guarantees a finite integer >= 1; cap the upper bound so a large
   // value can't materialize a giant per-page array (Array(w).fill) and OOM retrieval.
   const w = Math.min(25, loadKbConfig(kbRoot).bm25TitleWeight)
-  const token = pageSetToken(kbRoot, w)
+  const token = pagesToken(kbRoot)
+  const bmKey = `${token}|w=${w}` // title weight changes the indexed text but not the page list
   let entry = bm25Cache.get(kbRoot)
-  if (!entry || entry.token !== token) {
-    const pages = listWikiPages(kbRoot).filter(p => !p.error && !isInvalidated(p))
-    const idx = buildBm25Index(pages.map(p => ({
+  if (!entry || entry.token !== bmKey) {
+    const idx = buildBm25Index(livePages(kbRoot, token).map(p => ({
       id: p.relPath,
       text: [Array(w).fill(p.data.title ?? '').join('\n'), p.data.description, asList(p.data.tags).join(' '), p.body].join('\n'),
     })))
     bm25Cache.delete(kbRoot) // re-insert so this KB becomes most-recent for LRU eviction
-    bm25Cache.set(kbRoot, { token, idx })
-    while (bm25Cache.size > BM25_CACHE_MAX) bm25Cache.delete(bm25Cache.keys().next().value)
+    bm25Cache.set(kbRoot, { token: bmKey, idx })
+    while (bm25Cache.size > CACHE_MAX) bm25Cache.delete(bm25Cache.keys().next().value)
     entry = bm25Cache.get(kbRoot)
   }
   return searchBm25(entry.idx, question, k).map(h => ({ relPath: h.id, score: h.score }))
@@ -110,7 +124,7 @@ export function fuseChannels({ bm25, vector }, k, { lexicalGuard = true } = {})
 // 'bm25': lexical only, returns before any vector access.
 // 'hybrid': explicit BM25+vector fusion — every missing prerequisite (and any
 // vector error) throws instead of degrading, and vectorEnabled is ignored.
-export async function locatePages(kbRoot, question, { k = 6, fetchImpl, retrieval = 'auto', retry } = {}) {
+export async function locatePages(kbRoot, question, { k = 6, fetchImpl, retrieval = 'auto', retry, pipelineFactory } = {}) {
   if (!['auto', 'bm25', 'hybrid'].includes(retrieval)) throw new Error(`unknown retrieval mode: ${retrieval}`)
   const bm25 = retrievePages(kbRoot, question, k)
   const asBm25 = () => ({ hits: bm25.map(h => ({ ...h, sources: ['bm25'] })), usedVector: false, guardApplied: false })
@@ -121,27 +135,32 @@ export async function locatePages(kbRoot, question, { k = 6, fetchImpl, retrieva
     if (retrieval === 'hybrid') throw new Error(`retrieval 'hybrid' unavailable: ${what}`)
     return asBm25()
   }
-  if (retrieval === 'auto' && !loadKbConfig(kbRoot).vectorEnabled) return asBm25()
+  // One config read for both the vectorEnabled gate and the lexicalGuard flag below.
+  const kbCfg = loadKbConfig(kbRoot)
+  if (retrieval === 'auto' && !kbCfg.vectorEnabled) return asBm25()
   const store = loadVectorStore(kbRoot)
   if (!store) return unavailable('no wiki/.vectors.json — run `llm-wiki embed` first')
-  const cfg = loadLlmConfig(kbRoot)
+  const cfg = loadEmbedConfig(kbRoot)
   if (!cfg?.embeddingModel) return unavailable('no embeddingModel in ~/.llm-wiki/config.json')
   // A store built by a different embeddingModel lives in a foreign vector space:
   // fusing it produces silent garbage, so treat it as missing (not a failure).
   if (store.model !== cfg.embeddingModel) return unavailable(`vector store was built with "${store.model}" but config says "${cfg.embeddingModel}" — re-run \`llm-wiki embed\``)
   try {
-    const t = fetchImpl ? { fetchImpl, dispatcher: undefined, retry } : { ...(await makeTransport()), retry }
-    const [qv] = await embedTexts(cfg, t, [question])
+    const injected = fetchImpl || pipelineFactory
+    const t = injected
+      ? { fetchImpl, dispatcher: undefined, retry, pipelineFactory }
+      : { ...(await makeTransport()), retry }
+    const [qv] = await embedTexts(cfg, t, [question], { role: 'query' })
     const qn = normalize(qv)
     const vecHits = qn ? cosineTopK(qn, store, k).map(v => ({ relPath: v.id, score: v.score })) : []
     // The store is a snapshot from the last embed; pages invalidated, deleted, or
     // renamed since then still have vectors. Keep only hits that map to a live,
     // non-invalidated page so retired knowledge cannot resurface (and askKb never
     // ENOENTs reading a vanished file).
-    const valid = new Set(listWikiPages(kbRoot).filter(pg => !pg.error && !isInvalidated(pg)).map(pg => pg.relPath))
+    const valid = new Set(livePages(kbRoot).map(pg => pg.relPath))
     const vecHitsValid = vecHits.filter(h => valid.has(h.relPath))
     const { hits, guardApplied } = fuseChannels({ bm25, vector: vecHitsValid }, k,
-      { lexicalGuard: retrieval === 'auto' && loadKbConfig(kbRoot).lexicalGuard })
+      { lexicalGuard: retrieval === 'auto' && kbCfg.lexicalGuard })
     return { hits, usedVector: true, guardApplied }
   } catch (err) {
     if (retrieval === 'hybrid') throw err
@@ -210,15 +229,13 @@ async function pickPagesFromListing(p, question, k, cfg, t, validIds) {
   return picked.map(id => ({ relPath: `${id}.md`, score: 0 }))
 }
 
-export async function askKb(kbRoot, question, { k = 6, retrieveOnly = false, fetchImpl, retrieval = 'auto', retry } = {}) {
+export async function askKb(kbRoot, question, { k = 6, retrieveOnly = false, fetchImpl, retrieval = 'auto', retry, pipelineFactory } = {}) {
   const p = kbPaths(kbRoot)
-  let { hits } = await locatePages(kbRoot, question, { k, fetchImpl, retrieval, retry })
+  let { hits } = await locatePages(kbRoot, question, { k, fetchImpl, retrieval, retry, pipelineFactory })
   if (retrieveOnly) return { pages: hits, answer: null }
   let validIds
   if (hits.length === 0) {
-    validIds = new Set(listWikiPages(kbRoot)
-      .filter(pg => !pg.error && !isInvalidated(pg))
-      .map(pg => pg.relPath.replace(/\.md$/, '')))
+    validIds = new Set(livePages(kbRoot).map(pg => pg.relPath.replace(/\.md$/, '')))
     if (validIds.size === 0) throw new Error('No relevant pages found — the knowledge base has no valid pages.')
   }
   const cfg = loadLlmConfig(kbRoot)

package/src/embed.mjs

@@ -1,8 +1,9 @@
 import { listWikiPages, isInvalidated } from './pages.mjs'
-import { loadLlmConfig, makeTransport } from './llm-config.mjs'
+import { loadEmbedConfig, makeTransport } from './llm-config.mjs'
 import { sha256Text } from './hashing.mjs'
 import { normalize, pageEmbedText, loadVectorStore, saveVectorStore } from './vector.mjs'
 import { fetchWithRetry } from './retry.mjs'
+import { embedLocal, isLocalModel, stripLocalPrefix } from './local-embed.mjs'
 
 const BATCH = 64
 // Safety margin under the 8192-token input limit of common embedding models
@@ -44,7 +45,10 @@ function capEmbedText(text) {
   return text.slice(0, lo)
 }
 
-export async function embedTexts(cfg, t, texts) {
+export async function embedTexts(cfg, t, texts, { role = 'passage' } = {}) {
+  if (isLocalModel(cfg.embeddingModel)) {
+    return embedLocal(stripLocalPrefix(cfg.embeddingModel), texts, { role, pipelineFactory: t?.pipelineFactory })
+  }
   const url = `${cfg.baseURL.replace(/\/$/, '')}/embeddings`
   let res
   try {
@@ -70,10 +74,9 @@ export async function embedTexts(cfg, t, texts) {
   return [...data.data].sort((a, b) => a.index - b.index).map(d => d.embedding)
 }
 
-export async function embedKb(kbRoot, { fetchImpl, retry } = {}) {
-  const cfg = loadLlmConfig(kbRoot)
-  if (!cfg) throw new Error('No LLM configured. Create ~/.llm-wiki/config.json (OpenAI-compatible).')
-  if (!cfg.embeddingModel) throw new Error('No embedding model configured. Add "embeddingModel" to your provider (or flat config) in ~/.llm-wiki/config.json, e.g. "text-embedding-3-small".')
+export async function embedKb(kbRoot, { fetchImpl, retry, pipelineFactory } = {}) {
+  const cfg = loadEmbedConfig(kbRoot)
+  if (!cfg?.embeddingModel) throw new Error('No usable embedding config. A remote embeddingModel needs chat credentials (baseURL/apiKey) in ~/.llm-wiki/config.json; a local one needs none — set "embeddingModel": "local:Xenova/multilingual-e5-small".')
   const pages = listWikiPages(kbRoot).filter(p => !p.error && !isInvalidated(p))
   const prev = loadVectorStore(kbRoot)
   const reuse = (prev && prev.model === cfg.embeddingModel) ? prev.pages : {}
@@ -92,7 +95,10 @@ export async function embedKb(kbRoot, { fetchImpl, retry } = {}) {
   let dim = (prev && prev.model === cfg.embeddingModel) ? prev.dim : null
   let embedded = 0
   if (jobs.length > 0) {
-    const t = fetchImpl ? { fetchImpl, dispatcher: undefined, retry } : { ...(await makeTransport()), retry }
+    const injected = fetchImpl || pipelineFactory
+    const t = injected
+      ? { fetchImpl, dispatcher: undefined, retry, pipelineFactory }
+      : { ...(await makeTransport()), retry }
     for (let i = 0; i < jobs.length; i += BATCH) {
       const batch = jobs.slice(i, i + BATCH)
       const vecs = await embedTexts(cfg, t, batch.map(j => j.text))

package/src/llm-config.mjs

@@ -65,6 +65,24 @@ export function loadLlmConfig(kbRoot) {
   return cfg
 }
 
+// The embedding path (embedKb, locatePages' vector branch) needs an embeddingModel
+// but not necessarily chat credentials: a LOCAL model runs with no baseURL/apiKey at
+// all, so a fully-offline KB (local embeddings, no chat provider) can still embed +
+// search. A REMOTE embeddingModel still needs the chat provider's baseURL/apiKey to
+// call the API, so it resolves only when loadLlmConfig does. loadLlmConfig's own
+// (chat-complete-or-null) contract is unchanged.
+export function loadEmbedConfig(kbRoot) {
+  const chat = loadLlmConfig(kbRoot)
+  if (chat?.embeddingModel) return chat
+  const dir = process.env.LLM_WIKI_CONFIG_DIR ?? path.join(os.homedir(), '.llm-wiki')
+  const globalFile = path.join(dir, 'config.json')
+  if (!fs.existsSync(globalFile)) return null
+  const fileCfg = readJsonFile(globalFile, { redactContents: true })
+  const em = fileCfg.embeddingModel
+    ?? Object.values(fileCfg.providers ?? {}).map(p => p?.embeddingModel).find(Boolean)
+  return (typeof em === 'string' && em.startsWith('local:')) ? { embeddingModel: em } : null
+}
+
 // Node's built-in fetch rejects the npm undici package's dispatcher
 // ("invalid onRequestStart method": handler interface mismatch between
 // undici 8.x and Node's bundled undici), so when a proxy is configured we

package/src/local-embed.mjs

@@ -0,0 +1,72 @@
+// Local/offline embedding via transformers.js. The heavy dependency
+// (@huggingface/transformers, ~253 MB installed) is an optional peer dependency,
+// imported ONLY here and ONLY when a `local:` embeddingModel is used — the base
+// install stays lean. (Separate from the ~120 MB first-run model download noted in
+// the README — that's the model weights, not this package.) The pipeline factory is
+// injectable so tests never touch the real dependency.
+
+const LOCAL_PREFIX = 'local:'
+
+export function isLocalModel(m) {
+  return typeof m === 'string' && m.startsWith(LOCAL_PREFIX)
+}
+
+export function stripLocalPrefix(m) {
+  return m.slice(LOCAL_PREFIX.length)
+}
+
+// A missing optional dependency surfaces as ERR_MODULE_NOT_FOUND — turn it into an
+// actionable instruction instead of an opaque stack. Other errors pass through.
+export function friendlyImportError(err) {
+  if (err?.code === 'ERR_MODULE_NOT_FOUND') {
+    return new Error('local embedding needs @huggingface/transformers; run: npm i @huggingface/transformers')
+  }
+  return err
+}
+
+// e5-family models are trained with asymmetric "query:" / "passage:" instruction
+// prefixes; retrieval quality drops materially without them. Other families
+// (e.g. paraphrase-multilingual) encode symmetrically and must NOT get a prefix.
+function applyRolePrefix(model, texts, role) {
+  if (!/e5/i.test(model)) return texts
+  const tag = role === 'query' ? 'query: ' : 'passage: '
+  return texts.map(t => tag + t)
+}
+
+// Default factory: dynamically import transformers.js and build a feature-extraction
+// pipeline. Mean-pooled, UN-normalized — the shared normalize() in embed/ask applies
+// the unit-length step, identical to the API path.
+async function defaultPipelineFactory(model) {
+  let transformers
+  try {
+    transformers = await import('@huggingface/transformers')
+  } catch (err) {
+    throw friendlyImportError(err)
+  }
+  const extractor = await transformers.pipeline('feature-extraction', model)
+  return async (texts) => {
+    const out = await extractor(texts, { pooling: 'mean', normalize: false })
+    return out.tolist() // number[][], one row per input, in order
+  }
+}
+
+// Cache the real (expensive) per-model load across a batch run / long-lived MCP
+// server. An injected factory (tests) bypasses the cache so tests stay isolated.
+const pipelineCache = new Map() // model -> Promise<embedFn>
+
+export async function embedLocal(model, texts, { role = 'passage', pipelineFactory } = {}) {
+  const prepared = applyRolePrefix(model, texts, role)
+  if (pipelineFactory) {
+    const embedFn = await pipelineFactory(model)
+    return embedFn(prepared)
+  }
+  let entry = pipelineCache.get(model)
+  if (!entry) {
+    entry = defaultPipelineFactory(model)
+    pipelineCache.set(model, entry)
+  }
+  let embedFn
+  try { embedFn = await entry }
+  catch (err) { pipelineCache.delete(model); throw err } // don't cache a failed load
+  return embedFn(prepared)
+}