@sdsrs/llm-wiki 0.9.0 → 0.9.1

package/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## 0.9.1 (2026-07-12)
+
+Correctness and diagnostics for the 0.9.0 local-embedding path; no change to the
+vectors produced. Suite 271 → 272.
+
+**Fixed:**
+
+- **Long pages no longer truncate silently on local models.** A local model
+  (e.g. `local:Xenova/multilingual-e5-small`) has a ~512-token input window and its
+  pipeline silently truncates longer pages to a head-only vector. `embed` now warns per
+  page and reports a `truncated N` count in its summary, so the head-only truncation is
+  visible instead of silent. The vector produced is unchanged.
+- **A broken transitive dependency is no longer masked as "install
+  @huggingface/transformers".** The missing-dependency hint now fires only when
+  transformers.js itself is absent; when transformers.js is installed but one of its
+  (native) deps fails to load, the original error naming the real culprit passes through.
+- **Clearer `embed` error when no embedding model is configured** — it now names the
+  actual fix (set `embeddingModel`) for both remote and local, instead of telling a user
+  who already has chat credentials to add credentials.
+
 ## 0.9.0 (2026-07-12)
 
 **Added:**

package/bin/llm-wiki.mjs

@@ -77,7 +77,8 @@ program.command('embed')
   .option('--kb <dir>', 'knowledge base root', '.')
   .action(async (opts) => {
     const r = await embedKb(opts.kb)
-    console.log(`embedded ${r.embedded}, reused ${r.reused}, pruned ${r.pruned} (model ${r.model}, dim ${r.dim})`)
+    const trunc = r.truncated ? `, truncated ${r.truncated}` : ''
+    console.log(`embedded ${r.embedded}, reused ${r.reused}, pruned ${r.pruned}${trunc} (model ${r.model}, dim ${r.dim})`)
   })
 
 program.command('ask <question>')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sdsrs/llm-wiki",
-  "version": "0.9.0",
+  "version": "0.9.1",
   "description": "Compile messy document directories into Karpathy-style llm_wiki knowledge bases — standalone Q&A CLI + Claude Code/Codex skills",
   "directories": {
     "test": "test"

package/src/embed.mjs

@@ -3,6 +3,7 @@ import { loadEmbedConfig, makeTransport } from './llm-config.mjs'
 import { sha256Text } from './hashing.mjs'
 import { normalize, pageEmbedText, loadVectorStore, saveVectorStore } from './vector.mjs'
 import { fetchWithRetry } from './retry.mjs'
+import { estimateTokens } from './scanner.mjs'
 import { embedLocal, isLocalModel, stripLocalPrefix } from './local-embed.mjs'
 
 const BATCH = 64
@@ -11,6 +12,15 @@ const BATCH = 64
 // whole-page retrieval is unaffected — the page file and its BM25 text are never
 // touched, only the vector's input is truncated so one huge page can't abort the run.
 const EMBED_TOKEN_CAP = 8000
+// Local (transformers.js) models don't reject over-limit input the way an API does —
+// the feature-extraction pipeline silently truncates at the model's own window, so a
+// page over it yields a head-only vector with NO error. e5-small and most Xenova
+// sentence-transformers are 512 tokens; we keep the 8000 cap on the text we send (it
+// bounds compute and stays well above the window so the pipeline still fills it) and
+// warn separately, via a realistic token estimate, whenever a page crosses this window
+// so the head-only truncation is visible instead of silent. A model with a different
+// window would extend this into a per-model lookup.
+const LOCAL_TOKEN_WINDOW = 512
 
 // Worst-case token count for the embedding API — deliberately NOT scanner's
 // estimateTokens (chars/4), which underestimates dense markdown/code where real
@@ -76,20 +86,32 @@ export async function embedTexts(cfg, t, texts, { role = 'passage' } = {}) {
 
 export async function embedKb(kbRoot, { fetchImpl, retry, pipelineFactory } = {}) {
   const cfg = loadEmbedConfig(kbRoot)
-  if (!cfg?.embeddingModel) throw new Error('No usable embedding config. A remote embeddingModel needs chat credentials (baseURL/apiKey) in ~/.llm-wiki/config.json; a local one needs none — set "embeddingModel": "local:Xenova/multilingual-e5-small".')
+  if (!cfg?.embeddingModel) throw new Error('No embedding model configured. Set "embeddingModel" in ~/.llm-wiki/config.json — a remote one (e.g. "text-embedding-3-small") also needs the provider baseURL/apiKey; a local one (e.g. "local:Xenova/multilingual-e5-small") needs no credentials.')
+  const local = isLocalModel(cfg.embeddingModel)
   const pages = listWikiPages(kbRoot).filter(p => !p.error && !isInvalidated(p))
   const prev = loadVectorStore(kbRoot)
   const reuse = (prev && prev.model === cfg.embeddingModel) ? prev.pages : {}
   const jobs = []
   const nextPages = {}
   let reused = 0
+  let truncatedCount = 0
   for (const pg of pages) {
     const raw = pageEmbedText(pg)
     const text = capEmbedText(raw)
-    const truncated = text !== raw
+    // API path: `truncated` means our worst-case cap actually cut the text (the API
+    // would otherwise reject it). Local path: the cap rarely fires (window ≪ 8000), so
+    // detect the model's OWN silent truncation with a realistic estimate against its
+    // window — that's what produces a head-only vector the user should know about.
+    const overWindow = local && estimateTokens(raw) > LOCAL_TOKEN_WINDOW
+    const truncated = overWindow || (!local && text !== raw)
     const hash = sha256Text(text)
     if (reuse[pg.relPath]?.hash === hash) { nextPages[pg.relPath] = reuse[pg.relPath]; reused++; continue }
-    if (truncated) process.stderr.write(`warning: ${pg.relPath} embed text truncated to ~${EMBED_TOKEN_CAP} worst-case tokens (page exceeds embedding model input limit)\n`)
+    if (truncated) {
+      truncatedCount++
+      process.stderr.write(overWindow
+        ? `warning: ${pg.relPath} exceeds the local embedding model's ~${LOCAL_TOKEN_WINDOW}-token window — its vector covers only the page head\n`
+        : `warning: ${pg.relPath} embed text truncated to ~${EMBED_TOKEN_CAP} worst-case tokens (page exceeds embedding model input limit)\n`)
+    }
     jobs.push({ relPath: pg.relPath, text, hash })
   }
   let dim = (prev && prev.model === cfg.embeddingModel) ? prev.dim : null
@@ -119,5 +141,5 @@ export async function embedKb(kbRoot, { fetchImpl, retry, pipelineFactory } = {}
   const pruned = prev ? Object.keys(prev.pages).filter(id => !(id in nextPages)).length : 0
   const store = { model: cfg.embeddingModel, dim: dim ?? 0, pages: nextPages }
   saveVectorStore(kbRoot, store)
-  return { embedded, reused, pruned, model: cfg.embeddingModel, dim: store.dim }
+  return { embedded, reused, pruned, truncated: truncatedCount, model: cfg.embeddingModel, dim: store.dim }
 }

package/src/local-embed.mjs

@@ -16,9 +16,20 @@ export function stripLocalPrefix(m) {
 }
 
 // A missing optional dependency surfaces as ERR_MODULE_NOT_FOUND — turn it into an
-// actionable instruction instead of an opaque stack. Other errors pass through.
+// actionable instruction instead of an opaque stack. Only map it when the ABSENT
+// module is transformers.js itself: the same code fires when transformers.js IS
+// installed but one of its (transitive/native) deps fails to resolve, and telling the
+// user to re-install transformers then hides the real missing module. Pass those
+// through so their message names the actual culprit.
 export function friendlyImportError(err) {
-  if (err?.code === 'ERR_MODULE_NOT_FOUND') {
+  if (err?.code !== 'ERR_MODULE_NOT_FOUND') return err
+  // Gate on the MISSING package — the quoted name after "Cannot find package/module" —
+  // NOT a substring of the whole message. A transitive failure names its real culprit
+  // in the quotes (e.g. 'onnxruntime-node') yet still mentions "@huggingface/transformers"
+  // in the "imported from <path>" tail, so a whole-message test mis-maps it. Rewrite only
+  // when the absent module IS transformers.js (the package root or a subpath import of it).
+  const missing = /Cannot find (?:package|module) '([^']+)'/.exec(err.message ?? '')?.[1]
+  if (missing && /^@huggingface\/transformers(\/|$)/.test(missing)) {
     return new Error('local embedding needs @huggingface/transformers; run: npm i @huggingface/transformers')
   }
   return err