@sdsrs/llm-wiki 0.8.3 → 0.9.1

package/CHANGELOG.md

@@ -1,5 +1,37 @@
 # Changelog
 
+## 0.9.1 (2026-07-12)
+
+Correctness and diagnostics for the 0.9.0 local-embedding path; no change to the
+vectors produced. Suite 271 → 272.
+
+**Fixed:**
+
+- **Long pages no longer truncate silently on local models.** A local model
+  (e.g. `local:Xenova/multilingual-e5-small`) has a ~512-token input window and its
+  pipeline silently truncates longer pages to a head-only vector. `embed` now warns per
+  page and reports a `truncated N` count in its summary, so the head-only truncation is
+  visible instead of silent. The vector produced is unchanged.
+- **A broken transitive dependency is no longer masked as "install
+  @huggingface/transformers".** The missing-dependency hint now fires only when
+  transformers.js itself is absent; when transformers.js is installed but one of its
+  (native) deps fails to load, the original error naming the real culprit passes through.
+- **Clearer `embed` error when no embedding model is configured** — it now names the
+  actual fix (set `embeddingModel`) for both remote and local, instead of telling a user
+  who already has chat credentials to add credentials.
+
+## 0.9.0 (2026-07-12)
+
+**Added:**
+
+- **Local / offline embedding via transformers.js.** Set `"embeddingModel":
+  "local:Xenova/multilingual-e5-small"` in `~/.llm-wiki/config.json` and install the
+  optional `@huggingface/transformers` to generate and query vectors with no embedding
+  API. With only a local model configured, `embed` / `search` / `ask --retrieve-only`
+  run fully offline (answering with `ask` still needs a chat provider). The default
+  model is multilingual, so zh↔en cross-language retrieval is preserved. The dependency
+  is optional — the base install is unchanged.
+
 ## 0.8.3 (2026-07-12)
 
 Performance and test-hardening cleanup; no shipped behavior change. Suite 261 → 263.

package/README.md

@@ -144,6 +144,22 @@ similarity (RRF); `--retrieve-only` labels each hit `[bm25]`, `[vector]` or
 `[bm25+vector]`. Vectors only *locate* pages — context is still whole pages,
 never chunks. Any vector-path failure falls back to BM25 with a warning.
 
+### Local / offline embedding
+
+Vector retrieval normally calls an embedding API. To run it locally with no API key:
+
+1. Install the optional model runtime: `npm i @huggingface/transformers`
+2. In `~/.llm-wiki/config.json`, set a `local:` embedding model:
+   ```json
+   { "embeddingModel": "local:Xenova/multilingual-e5-small" }
+   ```
+3. `llm-wiki embed --kb ./kb` — the first run downloads the model (~120 MB).
+
+The default `Xenova/multilingual-e5-small` is multilingual, so cross-language
+(zh↔en) retrieval keeps working. With only a local embedding model configured,
+`embed`, `search`, and `ask --retrieve-only` run fully offline; generating an
+answer with `ask` still needs a chat provider.
+
 ### Graph queries
 
 `graph.json` (rebuilt by `index`) is a link graph over the pages — wikilinks,

package/bin/llm-wiki.mjs

@@ -77,7 +77,8 @@ program.command('embed')
   .option('--kb <dir>', 'knowledge base root', '.')
   .action(async (opts) => {
     const r = await embedKb(opts.kb)
-    console.log(`embedded ${r.embedded}, reused ${r.reused}, pruned ${r.pruned} (model ${r.model}, dim ${r.dim})`)
+    const trunc = r.truncated ? `, truncated ${r.truncated}` : ''
+    console.log(`embedded ${r.embedded}, reused ${r.reused}, pruned ${r.pruned}${trunc} (model ${r.model}, dim ${r.dim})`)
   })
 
 program.command('ask <question>')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sdsrs/llm-wiki",
-  "version": "0.8.3",
+  "version": "0.9.1",
   "description": "Compile messy document directories into Karpathy-style llm_wiki knowledge bases — standalone Q&A CLI + Claude Code/Codex skills",
   "directories": {
     "test": "test"
@@ -51,5 +51,13 @@
     "undici": "^8.7.0",
     "yaml": "^2.9.0",
     "zod": "^4.4.3"
+  },
+  "peerDependencies": {
+    "@huggingface/transformers": "^3.0.0"
+  },
+  "peerDependenciesMeta": {
+    "@huggingface/transformers": {
+      "optional": true
+    }
   }
 }

package/src/ask.mjs

@@ -5,7 +5,7 @@ import { loadKbConfig } from './templates.mjs'
 import { listWikiPages, isInvalidated, asList, PAGE_DIRS } from './pages.mjs'
 import { buildBm25Index, searchBm25 } from './bm25.mjs'
 import { worstCaseTokens } from './scanner.mjs'
-import { loadLlmConfig, makeTransport } from './llm-config.mjs'
+import { loadLlmConfig, loadEmbedConfig, makeTransport } from './llm-config.mjs'
 import { loadVectorStore, normalize, cosineTopK } from './vector.mjs'
 import { embedTexts } from './embed.mjs'
 import { fetchWithRetry } from './retry.mjs'
@@ -124,7 +124,7 @@ export function fuseChannels({ bm25, vector }, k, { lexicalGuard = true } = {})
 // 'bm25': lexical only, returns before any vector access.
 // 'hybrid': explicit BM25+vector fusion — every missing prerequisite (and any
 // vector error) throws instead of degrading, and vectorEnabled is ignored.
-export async function locatePages(kbRoot, question, { k = 6, fetchImpl, retrieval = 'auto', retry } = {}) {
+export async function locatePages(kbRoot, question, { k = 6, fetchImpl, retrieval = 'auto', retry, pipelineFactory } = {}) {
   if (!['auto', 'bm25', 'hybrid'].includes(retrieval)) throw new Error(`unknown retrieval mode: ${retrieval}`)
   const bm25 = retrievePages(kbRoot, question, k)
   const asBm25 = () => ({ hits: bm25.map(h => ({ ...h, sources: ['bm25'] })), usedVector: false, guardApplied: false })
@@ -140,14 +140,17 @@ export async function locatePages(kbRoot, question, { k = 6, fetchImpl, retrieva
   if (retrieval === 'auto' && !kbCfg.vectorEnabled) return asBm25()
   const store = loadVectorStore(kbRoot)
   if (!store) return unavailable('no wiki/.vectors.json — run `llm-wiki embed` first')
-  const cfg = loadLlmConfig(kbRoot)
+  const cfg = loadEmbedConfig(kbRoot)
   if (!cfg?.embeddingModel) return unavailable('no embeddingModel in ~/.llm-wiki/config.json')
   // A store built by a different embeddingModel lives in a foreign vector space:
   // fusing it produces silent garbage, so treat it as missing (not a failure).
   if (store.model !== cfg.embeddingModel) return unavailable(`vector store was built with "${store.model}" but config says "${cfg.embeddingModel}" — re-run \`llm-wiki embed\``)
   try {
-    const t = fetchImpl ? { fetchImpl, dispatcher: undefined, retry } : { ...(await makeTransport()), retry }
-    const [qv] = await embedTexts(cfg, t, [question])
+    const injected = fetchImpl || pipelineFactory
+    const t = injected
+      ? { fetchImpl, dispatcher: undefined, retry, pipelineFactory }
+      : { ...(await makeTransport()), retry }
+    const [qv] = await embedTexts(cfg, t, [question], { role: 'query' })
     const qn = normalize(qv)
     const vecHits = qn ? cosineTopK(qn, store, k).map(v => ({ relPath: v.id, score: v.score })) : []
     // The store is a snapshot from the last embed; pages invalidated, deleted, or
@@ -226,9 +229,9 @@ async function pickPagesFromListing(p, question, k, cfg, t, validIds) {
   return picked.map(id => ({ relPath: `${id}.md`, score: 0 }))
 }
 
-export async function askKb(kbRoot, question, { k = 6, retrieveOnly = false, fetchImpl, retrieval = 'auto', retry } = {}) {
+export async function askKb(kbRoot, question, { k = 6, retrieveOnly = false, fetchImpl, retrieval = 'auto', retry, pipelineFactory } = {}) {
   const p = kbPaths(kbRoot)
-  let { hits } = await locatePages(kbRoot, question, { k, fetchImpl, retrieval, retry })
+  let { hits } = await locatePages(kbRoot, question, { k, fetchImpl, retrieval, retry, pipelineFactory })
   if (retrieveOnly) return { pages: hits, answer: null }
   let validIds
   if (hits.length === 0) {

package/src/embed.mjs

@@ -1,8 +1,10 @@
 import { listWikiPages, isInvalidated } from './pages.mjs'
-import { loadLlmConfig, makeTransport } from './llm-config.mjs'
+import { loadEmbedConfig, makeTransport } from './llm-config.mjs'
 import { sha256Text } from './hashing.mjs'
 import { normalize, pageEmbedText, loadVectorStore, saveVectorStore } from './vector.mjs'
 import { fetchWithRetry } from './retry.mjs'
+import { estimateTokens } from './scanner.mjs'
+import { embedLocal, isLocalModel, stripLocalPrefix } from './local-embed.mjs'
 
 const BATCH = 64
 // Safety margin under the 8192-token input limit of common embedding models
@@ -10,6 +12,15 @@ const BATCH = 64
 // whole-page retrieval is unaffected — the page file and its BM25 text are never
 // touched, only the vector's input is truncated so one huge page can't abort the run.
 const EMBED_TOKEN_CAP = 8000
+// Local (transformers.js) models don't reject over-limit input the way an API does —
+// the feature-extraction pipeline silently truncates at the model's own window, so a
+// page over it yields a head-only vector with NO error. e5-small and most Xenova
+// sentence-transformers are 512 tokens; we keep the 8000 cap on the text we send (it
+// bounds compute and stays well above the window so the pipeline still fills it) and
+// warn separately, via a realistic token estimate, whenever a page crosses this window
+// so the head-only truncation is visible instead of silent. A model with a different
+// window would extend this into a per-model lookup.
+const LOCAL_TOKEN_WINDOW = 512
 
 // Worst-case token count for the embedding API — deliberately NOT scanner's
 // estimateTokens (chars/4), which underestimates dense markdown/code where real
@@ -44,7 +55,10 @@ function capEmbedText(text) {
   return text.slice(0, lo)
 }
 
-export async function embedTexts(cfg, t, texts) {
+export async function embedTexts(cfg, t, texts, { role = 'passage' } = {}) {
+  if (isLocalModel(cfg.embeddingModel)) {
+    return embedLocal(stripLocalPrefix(cfg.embeddingModel), texts, { role, pipelineFactory: t?.pipelineFactory })
+  }
   const url = `${cfg.baseURL.replace(/\/$/, '')}/embeddings`
   let res
   try {
@@ -70,29 +84,43 @@ export async function embedTexts(cfg, t, texts) {
   return [...data.data].sort((a, b) => a.index - b.index).map(d => d.embedding)
 }
 
-export async function embedKb(kbRoot, { fetchImpl, retry } = {}) {
-  const cfg = loadLlmConfig(kbRoot)
-  if (!cfg) throw new Error('No LLM configured. Create ~/.llm-wiki/config.json (OpenAI-compatible).')
-  if (!cfg.embeddingModel) throw new Error('No embedding model configured. Add "embeddingModel" to your provider (or flat config) in ~/.llm-wiki/config.json, e.g. "text-embedding-3-small".')
+export async function embedKb(kbRoot, { fetchImpl, retry, pipelineFactory } = {}) {
+  const cfg = loadEmbedConfig(kbRoot)
+  if (!cfg?.embeddingModel) throw new Error('No embedding model configured. Set "embeddingModel" in ~/.llm-wiki/config.json — a remote one (e.g. "text-embedding-3-small") also needs the provider baseURL/apiKey; a local one (e.g. "local:Xenova/multilingual-e5-small") needs no credentials.')
+  const local = isLocalModel(cfg.embeddingModel)
   const pages = listWikiPages(kbRoot).filter(p => !p.error && !isInvalidated(p))
   const prev = loadVectorStore(kbRoot)
   const reuse = (prev && prev.model === cfg.embeddingModel) ? prev.pages : {}
   const jobs = []
   const nextPages = {}
   let reused = 0
+  let truncatedCount = 0
   for (const pg of pages) {
     const raw = pageEmbedText(pg)
     const text = capEmbedText(raw)
-    const truncated = text !== raw
+    // API path: `truncated` means our worst-case cap actually cut the text (the API
+    // would otherwise reject it). Local path: the cap rarely fires (window ≪ 8000), so
+    // detect the model's OWN silent truncation with a realistic estimate against its
+    // window — that's what produces a head-only vector the user should know about.
+    const overWindow = local && estimateTokens(raw) > LOCAL_TOKEN_WINDOW
+    const truncated = overWindow || (!local && text !== raw)
     const hash = sha256Text(text)
     if (reuse[pg.relPath]?.hash === hash) { nextPages[pg.relPath] = reuse[pg.relPath]; reused++; continue }
-    if (truncated) process.stderr.write(`warning: ${pg.relPath} embed text truncated to ~${EMBED_TOKEN_CAP} worst-case tokens (page exceeds embedding model input limit)\n`)
+    if (truncated) {
+      truncatedCount++
+      process.stderr.write(overWindow
+        ? `warning: ${pg.relPath} exceeds the local embedding model's ~${LOCAL_TOKEN_WINDOW}-token window — its vector covers only the page head\n`
+        : `warning: ${pg.relPath} embed text truncated to ~${EMBED_TOKEN_CAP} worst-case tokens (page exceeds embedding model input limit)\n`)
+    }
     jobs.push({ relPath: pg.relPath, text, hash })
   }
   let dim = (prev && prev.model === cfg.embeddingModel) ? prev.dim : null
   let embedded = 0
   if (jobs.length > 0) {
-    const t = fetchImpl ? { fetchImpl, dispatcher: undefined, retry } : { ...(await makeTransport()), retry }
+    const injected = fetchImpl || pipelineFactory
+    const t = injected
+      ? { fetchImpl, dispatcher: undefined, retry, pipelineFactory }
+      : { ...(await makeTransport()), retry }
     for (let i = 0; i < jobs.length; i += BATCH) {
       const batch = jobs.slice(i, i + BATCH)
       const vecs = await embedTexts(cfg, t, batch.map(j => j.text))
@@ -113,5 +141,5 @@ export async function embedKb(kbRoot, { fetchImpl, retry } = {}) {
   const pruned = prev ? Object.keys(prev.pages).filter(id => !(id in nextPages)).length : 0
   const store = { model: cfg.embeddingModel, dim: dim ?? 0, pages: nextPages }
   saveVectorStore(kbRoot, store)
-  return { embedded, reused, pruned, model: cfg.embeddingModel, dim: store.dim }
+  return { embedded, reused, pruned, truncated: truncatedCount, model: cfg.embeddingModel, dim: store.dim }
 }

package/src/llm-config.mjs

@@ -65,6 +65,24 @@ export function loadLlmConfig(kbRoot) {
   return cfg
 }
 
+// The embedding path (embedKb, locatePages' vector branch) needs an embeddingModel
+// but not necessarily chat credentials: a LOCAL model runs with no baseURL/apiKey at
+// all, so a fully-offline KB (local embeddings, no chat provider) can still embed +
+// search. A REMOTE embeddingModel still needs the chat provider's baseURL/apiKey to
+// call the API, so it resolves only when loadLlmConfig does. loadLlmConfig's own
+// (chat-complete-or-null) contract is unchanged.
+export function loadEmbedConfig(kbRoot) {
+  const chat = loadLlmConfig(kbRoot)
+  if (chat?.embeddingModel) return chat
+  const dir = process.env.LLM_WIKI_CONFIG_DIR ?? path.join(os.homedir(), '.llm-wiki')
+  const globalFile = path.join(dir, 'config.json')
+  if (!fs.existsSync(globalFile)) return null
+  const fileCfg = readJsonFile(globalFile, { redactContents: true })
+  const em = fileCfg.embeddingModel
+    ?? Object.values(fileCfg.providers ?? {}).map(p => p?.embeddingModel).find(Boolean)
+  return (typeof em === 'string' && em.startsWith('local:')) ? { embeddingModel: em } : null
+}
+
 // Node's built-in fetch rejects the npm undici package's dispatcher
 // ("invalid onRequestStart method": handler interface mismatch between
 // undici 8.x and Node's bundled undici), so when a proxy is configured we

package/src/local-embed.mjs

@@ -0,0 +1,83 @@
+// Local/offline embedding via transformers.js. The heavy dependency
+// (@huggingface/transformers, ~253 MB installed) is an optional peer dependency,
+// imported ONLY here and ONLY when a `local:` embeddingModel is used — the base
+// install stays lean. (Separate from the ~120 MB first-run model download noted in
+// the README — that's the model weights, not this package.) The pipeline factory is
+// injectable so tests never touch the real dependency.
+
+const LOCAL_PREFIX = 'local:'
+
+export function isLocalModel(m) {
+  return typeof m === 'string' && m.startsWith(LOCAL_PREFIX)
+}
+
+export function stripLocalPrefix(m) {
+  return m.slice(LOCAL_PREFIX.length)
+}
+
+// A missing optional dependency surfaces as ERR_MODULE_NOT_FOUND — turn it into an
+// actionable instruction instead of an opaque stack. Only map it when the ABSENT
+// module is transformers.js itself: the same code fires when transformers.js IS
+// installed but one of its (transitive/native) deps fails to resolve, and telling the
+// user to re-install transformers then hides the real missing module. Pass those
+// through so their message names the actual culprit.
+export function friendlyImportError(err) {
+  if (err?.code !== 'ERR_MODULE_NOT_FOUND') return err
+  // Gate on the MISSING package — the quoted name after "Cannot find package/module" —
+  // NOT a substring of the whole message. A transitive failure names its real culprit
+  // in the quotes (e.g. 'onnxruntime-node') yet still mentions "@huggingface/transformers"
+  // in the "imported from <path>" tail, so a whole-message test mis-maps it. Rewrite only
+  // when the absent module IS transformers.js (the package root or a subpath import of it).
+  const missing = /Cannot find (?:package|module) '([^']+)'/.exec(err.message ?? '')?.[1]
+  if (missing && /^@huggingface\/transformers(\/|$)/.test(missing)) {
+    return new Error('local embedding needs @huggingface/transformers; run: npm i @huggingface/transformers')
+  }
+  return err
+}
+
+// e5-family models are trained with asymmetric "query:" / "passage:" instruction
+// prefixes; retrieval quality drops materially without them. Other families
+// (e.g. paraphrase-multilingual) encode symmetrically and must NOT get a prefix.
+function applyRolePrefix(model, texts, role) {
+  if (!/e5/i.test(model)) return texts
+  const tag = role === 'query' ? 'query: ' : 'passage: '
+  return texts.map(t => tag + t)
+}
+
+// Default factory: dynamically import transformers.js and build a feature-extraction
+// pipeline. Mean-pooled, UN-normalized — the shared normalize() in embed/ask applies
+// the unit-length step, identical to the API path.
+async function defaultPipelineFactory(model) {
+  let transformers
+  try {
+    transformers = await import('@huggingface/transformers')
+  } catch (err) {
+    throw friendlyImportError(err)
+  }
+  const extractor = await transformers.pipeline('feature-extraction', model)
+  return async (texts) => {
+    const out = await extractor(texts, { pooling: 'mean', normalize: false })
+    return out.tolist() // number[][], one row per input, in order
+  }
+}
+
+// Cache the real (expensive) per-model load across a batch run / long-lived MCP
+// server. An injected factory (tests) bypasses the cache so tests stay isolated.
+const pipelineCache = new Map() // model -> Promise<embedFn>
+
+export async function embedLocal(model, texts, { role = 'passage', pipelineFactory } = {}) {
+  const prepared = applyRolePrefix(model, texts, role)
+  if (pipelineFactory) {
+    const embedFn = await pipelineFactory(model)
+    return embedFn(prepared)
+  }
+  let entry = pipelineCache.get(model)
+  if (!entry) {
+    entry = defaultPipelineFactory(model)
+    pipelineCache.set(model, entry)
+  }
+  let embedFn
+  try { embedFn = await entry }
+  catch (err) { pipelineCache.delete(model); throw err } // don't cache a failed load
+  return embedFn(prepared)
+}