@sdsrs/llm-wiki 0.4.1 → 0.5.0

package/CHANGELOG.md

@@ -1,5 +1,37 @@
 # Changelog
 
+## 0.5.0 (2026-07-11)
+
+No breaking changes, no KB migration, defaults unchanged. One behavior fix you
+may notice: `embed` no longer aborts the whole run when a page exceeds the
+embedding model's input limit — the text sent to the API is capped (~8000
+worst-case tokens, with a per-page stderr warning); the page file itself is
+never modified. Pin `@sdsrs/llm-wiki@0.4.1` to keep the previous (aborting)
+behavior.
+
+- feat(ask): `locatePages`/`askKb` accept `retrieval: 'auto' | 'bm25' | 'hybrid'`
+  (`auto` = existing behavior exactly; `bm25` forces lexical-only; `hybrid`
+  forces fusion and errors when vector prerequisites are missing instead of
+  degrading). Programmatic surface only — CLI behavior unchanged. `chatCompletion`
+  is now an exported helper.
+- fix(embed): oversized pages embed their head instead of killing the run —
+  input capped at ~8000 worst-case tokens (binary-search cut, lone-surrogate
+  trim at the boundary), per-page warning, incremental reuse unaffected for
+  pages under the cap.
+- Eval harness completed (dev, in-repo `scripts/eval/`): four probe classes
+  (fact / multihop / xlang / none), answer-level eval (abstention honesty +
+  3-dimension LLM-as-judge head-to-head with position-swap debiasing),
+  50/150-page scale corpora generated from local markdown (zero network),
+  experimental graph-degree RRF arm (measured negative on ./kb — recorded in
+  `scripts/eval/README.md`, not promoted to `ask`), and `run-all.mjs`
+  producing a single markdown report.
+- Measured 2026-07-11 (./kb 35pp + tiers, k=5): abstention on unanswerable
+  probes 100% for both bm25 and hybrid (n=4, zero fabrication); answer-level
+  head-to-head hybrid vs bm25 (n=24, judge gpt-5.1) — correctness 7-1-16,
+  citations 11-3-10, completeness 7-3-14 (wins-losses-ties); retrieval
+  Recall@5 bm25→hybrid: 35pp .688→.979, 50pp .600→1.000, 150pp .667→1.000
+  (no BM25-favored regime found in the 35–150 range).
+
 ## 0.4.1 (2026-07-11)
 
 Patch: all fixes, no new features, no deps, no KB migration. Defaults unchanged.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sdsrs/llm-wiki",
-  "version": "0.4.1",
+  "version": "0.5.0",
   "description": "Compile messy document directories into Karpathy-style llm_wiki knowledge bases — standalone Q&A CLI + Claude Code/Codex skills",
   "directories": {
     "test": "test"

package/src/ask.mjs

@@ -36,17 +36,25 @@ export function rrfFuse(lists, k) {
 // BM25 always; vector channel only when opted in (vectorEnabled) AND the
 // sidecar exists AND an embeddingModel is configured. Fail-open: any vector
 // error degrades to BM25 with a single stderr warning.
-export async function locatePages(kbRoot, question, { k = 6, fetchImpl } = {}) {
+export async function locatePages(kbRoot, question, { k = 6, fetchImpl, retrieval = 'auto' } = {}) {
+  if (!['auto', 'bm25', 'hybrid'].includes(retrieval)) throw new Error(`unknown retrieval mode: ${retrieval}`)
   const bm25 = retrievePages(kbRoot, question, k)
   const asBm25 = () => ({ hits: bm25.map(h => ({ ...h, sources: ['bm25'] })), usedVector: false })
-  if (!loadKbConfig(kbRoot).vectorEnabled) return asBm25()
+  if (retrieval === 'bm25') return asBm25()
+  // 'hybrid' is an explicit request: prerequisites failing is an error, not a
+  // silent degrade. 'auto' keeps the opt-in + fail-open contract unchanged.
+  const unavailable = (what) => {
+    if (retrieval === 'hybrid') throw new Error(`retrieval 'hybrid' unavailable: ${what}`)
+    return asBm25()
+  }
+  if (retrieval === 'auto' && !loadKbConfig(kbRoot).vectorEnabled) return asBm25()
   const store = loadVectorStore(kbRoot)
-  if (!store) return asBm25()
+  if (!store) return unavailable('no wiki/.vectors.json — run `llm-wiki embed` first')
   const cfg = loadLlmConfig(kbRoot)
-  if (!cfg?.embeddingModel) return asBm25()
+  if (!cfg?.embeddingModel) return unavailable('no embeddingModel in ~/.llm-wiki/config.json')
   // A store built by a different embeddingModel lives in a foreign vector space:
   // fusing it produces silent garbage, so treat it as missing (not a failure).
-  if (store.model !== cfg.embeddingModel) return asBm25()
+  if (store.model !== cfg.embeddingModel) return unavailable(`vector store was built with "${store.model}" but config says "${cfg.embeddingModel}" — re-run \`llm-wiki embed\``)
   try {
     const t = fetchImpl ? { fetchImpl, dispatcher: undefined } : await makeTransport()
     const [qv] = await embedTexts(cfg, t, [question])
@@ -60,12 +68,13 @@ export async function locatePages(kbRoot, question, { k = 6, fetchImpl } = {}) {
     const vecHitsValid = vecHits.filter(h => valid.has(h.relPath))
     return { hits: rrfFuse([{ source: 'bm25', hits: bm25 }, { source: 'vector', hits: vecHitsValid }], k), usedVector: true }
   } catch (err) {
+    if (retrieval === 'hybrid') throw err
     process.stderr.write(`warning: vector retrieval unavailable (${err.message}); falling back to BM25\n`)
     return asBm25()
   }
 }
 
-async function chatCompletion(cfg, t, messages) {
+export async function chatCompletion(cfg, t, messages) {
   const res = await t.fetchImpl(`${cfg.baseURL.replace(/\/$/, '')}/chat/completions`, {
     method: 'POST',
     headers: { 'content-type': 'application/json', authorization: `Bearer ${cfg.apiKey}` },
@@ -118,9 +127,9 @@ async function pickPagesFromListing(p, question, k, cfg, t, validIds) {
   return picked.map(id => ({ relPath: `${id}.md`, score: 0 }))
 }
 
-export async function askKb(kbRoot, question, { k = 6, retrieveOnly = false, fetchImpl } = {}) {
+export async function askKb(kbRoot, question, { k = 6, retrieveOnly = false, fetchImpl, retrieval = 'auto' } = {}) {
   const p = kbPaths(kbRoot)
-  let { hits } = await locatePages(kbRoot, question, { k, fetchImpl })
+  let { hits } = await locatePages(kbRoot, question, { k, fetchImpl, retrieval })
   if (retrieveOnly) return { pages: hits, answer: null }
   let validIds
   if (hits.length === 0) {

package/src/embed.mjs

@@ -4,6 +4,44 @@ import { sha256Text } from './hashing.mjs'
 import { normalize, pageEmbedText, loadVectorStore, saveVectorStore } from './vector.mjs'
 
 const BATCH = 64
+// Safety margin under the 8192-token input limit of common embedding models
+// (e.g. text-embedding-3-small). We cap ONLY the text sent to the embedding API;
+// whole-page retrieval is unaffected — the page file and its BM25 text are never
+// touched, only the vector's input is truncated so one huge page can't abort the run.
+const EMBED_TOKEN_CAP = 8000
+
+// Worst-case token count for the embedding API — deliberately NOT scanner's
+// estimateTokens (chars/4), which underestimates dense markdown/code where real
+// BPE runs ~2-3.5 chars/token and lets 8192-token pages slip through. This assumes
+// the pessimistic ~2 chars/token for non-CJK and 1 token/char for CJK, halving
+// usable capacity for English prose — the accepted tradeoff for a deterministic,
+// provider-independent cap. A location vector doesn't need the page's tail.
+// (CJK char class copied from scanner.mjs estimateTokens — kept local on purpose.)
+function worstCaseEmbedTokens(text) {
+  let cjk = 0
+  for (const ch of text) if (/[　-鿿豈-﫿]/.test(ch)) cjk++
+  return cjk + (text.length - cjk) / 2
+}
+
+// Largest prefix of `text` whose worstCaseEmbedTokens is within EMBED_TOKEN_CAP.
+// worstCaseEmbedTokens is monotonic non-decreasing in prefix length, so a binary
+// search over the string finds the cut with no dependencies.
+function capEmbedText(text) {
+  if (worstCaseEmbedTokens(text) <= EMBED_TOKEN_CAP) return text
+  let lo = 0, hi = text.length
+  while (lo < hi) {
+    const mid = Math.ceil((lo + hi) / 2)
+    if (worstCaseEmbedTokens(text.slice(0, mid)) <= EMBED_TOKEN_CAP) lo = mid
+    else hi = mid - 1
+  }
+  // Slicing by UTF-16 code unit can split an astral char (e.g. emoji, CJK ext-B),
+  // leaving a lone high surrogate (0xD800–0xDBFF) at the boundary. JSON.stringify
+  // escapes it as a literal \uD83D and a strict provider can reject the request —
+  // the exact abort this cap exists to prevent. Drop a dangling high surrogate.
+  const last = text.charCodeAt(lo - 1)
+  if (lo > 0 && last >= 0xD800 && last <= 0xDBFF) lo--
+  return text.slice(0, lo)
+}
 
 export async function embedTexts(cfg, t, texts) {
   const res = await t.fetchImpl(`${cfg.baseURL.replace(/\/$/, '')}/embeddings`, {
@@ -35,9 +73,12 @@ export async function embedKb(kbRoot, { fetchImpl } = {}) {
   const nextPages = {}
   let reused = 0
   for (const pg of pages) {
-    const text = pageEmbedText(pg)
+    const raw = pageEmbedText(pg)
+    const text = capEmbedText(raw)
+    const truncated = text !== raw
     const hash = sha256Text(text)
     if (reuse[pg.relPath]?.hash === hash) { nextPages[pg.relPath] = reuse[pg.relPath]; reused++; continue }
+    if (truncated) process.stderr.write(`warning: ${pg.relPath} embed text truncated to ~${EMBED_TOKEN_CAP} worst-case tokens (page exceeds embedding model input limit)\n`)
     jobs.push({ relPath: pg.relPath, text, hash })
   }
   let dim = (prev && prev.model === cfg.embeddingModel) ? prev.dim : null