@modusensus/dsh-mneme 0.4.6 → 0.5.0

package/src/api.js

@@ -103,7 +103,7 @@ export function createApi(ctx, service, settings, commands, embedder, semantic =
       try {
         const url = new URL(req.url, "http://localhost");
         const q = url.searchParams.get("q") ?? "";
-        const limit = Number(url.searchParams.get("limit") ?? 20);
+        const limit = Number(url.searchParams.get("topK") ?? url.searchParams.get("limit") ?? 20);
         // mode selects the recall strategy (defaults to auto):
         //   auto    (default) keyword first, vector fills remaining slots
         //   hybrid  vector first, keyword fills remaining slots; scores of
@@ -306,6 +306,122 @@ export function createApi(ctx, service, settings, commands, embedder, semantic =
     }
   });
 
+  // --- ego graph: 1-2 hop neighborhood of one entity (graph panel P1) ---
+  // Read-only like list/search/semantic, so it stays open when apiToken is set.
+  // BFS from the root entity over entity_relations (both directions; the
+  // idx_relations_from/to indexes keep a 2-hop walk in the tens of ms even
+  // for a few thousand nodes). `distance` on each node is the hop count from
+  // the root so the UI can shade the frontier. The API is graph-traversal
+  // only — nodes carry no attr payload; hover summaries come from
+  // /semantic/graph/entity-attrs.
+  register({
+    kind: "exact",
+    path: "/api/dsh-mneme/semantic/graph/ego",
+    handler(req, res) {
+      try {
+        const url = new URL(req.url, "http://localhost");
+        const name = (url.searchParams.get("entity") ?? "").trim();
+        if (!name) {
+          sendJson(res, 400, { error: "missing-entity" });
+          return;
+        }
+        const root = service.findEntityByName?.(name);
+        if (!root) {
+          sendJson(res, 404, { error: "entity-not-found" });
+          return;
+        }
+        const depth = Math.max(1, Math.min(2, Number(url.searchParams.get("depth") ?? 1) || 1));
+        const limit = Math.max(1, Math.min(100, Number(url.searchParams.get("limit") ?? 40) || 40));
+
+        const nodes = new Map([[root.id, { ...root, distance: 0 }]]);
+        let frontier = [root.id];
+        for (let d = 1; d <= depth && nodes.size < limit; d++) {
+          const next = [];
+          for (const id of frontier) {
+            for (const rel of service.getRelations?.(id) ?? []) {
+              const other = rel.from_entity === id ? rel.to_entity : rel.from_entity;
+              if (nodes.has(other) || nodes.size >= limit) continue;
+              const entity = service.findEntityById?.(other);
+              if (!entity) continue;
+              nodes.set(other, { ...entity, distance: d });
+              next.push(other);
+            }
+          }
+          frontier = next;
+        }
+
+        // Collect every relation whose endpoints both survived the limit cut;
+        // each edge is visited twice (once per endpoint) so dedupe by id.
+        const edgeMap = new Map();
+        for (const id of nodes.keys()) {
+          for (const rel of service.getRelations?.(id) ?? []) {
+            if (nodes.has(rel.from_entity) && nodes.has(rel.to_entity)) {
+              edgeMap.set(rel.id, rel);
+            }
+          }
+        }
+
+        sendJson(res, 200, {
+          root: { id: root.id, name: root.name, type: root.type ?? null, mention_count: root.mention_count ?? 1 },
+          nodes: [...nodes.values()].map((n) => ({
+            id: n.id,
+            name: n.name,
+            type: n.type ?? null,
+            mention_count: n.mention_count ?? 1,
+            distance: n.distance
+          })),
+          edges: [...edgeMap.values()].map((e) => ({
+            id: e.id,
+            from: e.from_entity,
+            to: e.to_entity,
+            relation_type: e.relation_type,
+            memory_id: e.memory_id ?? null,
+            created_at: e.created_at
+          }))
+        });
+      } catch {
+        sendJson(res, 500, { error: "internal" });
+      }
+    }
+  });
+
+  // --- entity attrs: current valid attrs for one entity (graph hover panel) ---
+  // Read-only; mirrors getCurrentAttrs (valid_until IS NULL). Also used as the
+  // graph panel's fallback list when the ego graph is too sparse to draw.
+  register({
+    kind: "exact",
+    path: "/api/dsh-mneme/semantic/graph/entity-attrs",
+    handler(req, res) {
+      try {
+        const url = new URL(req.url, "http://localhost");
+        const name = (url.searchParams.get("entity") ?? "").trim();
+        if (!name) {
+          sendJson(res, 400, { error: "missing-entity" });
+          return;
+        }
+        const entity = service.findEntityByName?.(name);
+        if (!entity) {
+          sendJson(res, 404, { error: "entity-not-found" });
+          return;
+        }
+        const attrs = service.getCurrentAttrs?.(entity.id) ?? [];
+        sendJson(res, 200, {
+          entity: { id: entity.id, name: entity.name, type: entity.type ?? null, mention_count: entity.mention_count ?? 1 },
+          attrs: Array.isArray(attrs)
+            ? attrs.map((a) => ({
+                key: a.attr_key,
+                value: a.attr_value,
+                confidence: a.confidence ?? null,
+                valid_from: a.valid_from ?? null
+              }))
+            : []
+        });
+      } catch {
+        sendJson(res, 500, { error: "internal" });
+      }
+    }
+  });
+
   // --- health: mirror sync state (F-NEW-03 / v0.3.6) ---
   // Auth-gated; only returns a sanitized error code (never raw last_error which
   // may leak paths/token-like strings/internal hosts). On state read failure it

package/src/config.js

@@ -17,7 +17,7 @@ export const Config = z.object({
   dreamDelayMs: z.natural().min(0).max(60000).default(2000),
   dreamProvider: z.string(),
   dreamModel: z.string(),
-  dreamMaxTokens: z.natural().min(256).max(131072).default(4096),
+  dreamMaxTokens: z.natural().min(256).max(131072).default(8192),
   // Pass-through reasoning effort for dream's LLM calls. 'none' (default)
   // omits the field so the provider's own default applies; low/medium/high
   // are forwarded verbatim. Useful to cap reasoning spend on thinking-type
@@ -92,6 +92,35 @@ export const Config = z.object({
   // rule-based pick to fill/dedupe. Empty query / no vector → legacy behavior.
   hybridInject: z.boolean().default(true),
 
+  // --- recall optimization (v0.5.0) ----------------------------------------
+  // BM25 third recall path beside vector + LIKE keyword (1.1): per-token IDF
+  // scoring recalls rows whose query terms are scattered — identifiers, code
+  // fragments, mixed CJK/ASCII — where substring LIKE cannot match.
+  bm25SearchEnabled: z.boolean().default(true),
+  // Query-aware vector cutoff (1.2) replacing the fixed 0.65: entity:/attr:
+  // prefixes loosen to 0.5, short queries tighten to 0.7, long queries loosen
+  // to 0.6, and a decisive top-1/top-5 score gap loosens to 0.5 so the tail
+  // still reaches the reranker. Off = legacy fixed threshold behavior.
+  adaptiveThresholdEnabled: z.boolean().default(true),
+  // Session-scoped hot memory (1.3): the latest N dialogue rounds rendered
+  // ahead of the long-term recall block — short-term context that never
+  // enters the memory store.
+  hotMemoryEnabled: z.boolean().default(true),
+  hotMemoryRounds: z.natural().min(1).max(50).default(5),
+  hotMemoryMaxTokens: z.natural().min(200).max(32000).default(2000),
+  // Topic-ranked injection (2.2): when a query vector is available the whole
+  // injection candidate list is re-ordered by similarity to the current
+  // query instead of keeping the rule-based order.
+  selectiveInjectEnabled: z.boolean().default(true),
+  // Search-time semantic dedup (2.3): greedy pass over the merged candidate
+  // list dropping rows whose embedding cosine-similarity to an already-kept
+  // row exceeds the threshold — duplicates are filtered at recall time
+  // instead of waiting for a dream consolidation. Opt-in aggressive mode:
+  // small embedding models can collapse legitimately distinct rows, so the
+  // default keeps every recalled row.
+  searchSemanticDedup: z.boolean().default(false),
+  searchSemanticDedupThreshold: z.number().min(0.5).max(1).default(0.95),
+
   // --- semantic: rerank layer (v0.2) --------------------------------------
   // Opt-in by default (item ⑥): the local cross-encoder pulls in onnxruntime
   // (transformers.js) at init, so a bare install must not load it. Only an

package/src/dream.js

@@ -3,6 +3,43 @@ import { clusterMemories, findPotentialConflicts } from "./dream/clustering.js";
 import { createHash, randomUUID } from "node:crypto";
 export { validateDecisions, applyDecisions };
 
+
+// Extract the first JSON array from LLM output, tolerating markdown fences,
+// leading/trailing prose, and common wrapper noise. Returns an array or null.
+function extractJsonArray(text) {
+  if (typeof text !== "string" || text.trim().length === 0) return null;
+
+  // 1. Strip markdown code fences (```json ... ``` or ``` ... ```).
+  let cleaned = text.replace(/```(?:json)?\s*([\s\S]*?)```/gi, "$1");
+  cleaned = cleaned.trim();
+
+  // 2. Find the first '[' and the matching last ']' that yields valid JSON.
+  const start = cleaned.indexOf("[");
+  if (start === -1) return null;
+  for (let end = cleaned.lastIndexOf("]"); end > start; end = cleaned.lastIndexOf("]", end - 1)) {
+    const candidate = cleaned.slice(start, end + 1);
+    try {
+      return JSON.parse(candidate);
+    } catch {
+      // Light repair: remove trailing commas before ] or }.
+      try {
+        const repaired = candidate.replace(/,(\s*[}\]])/g, "$1");
+        return JSON.parse(repaired);
+      } catch {
+        // keep searching backwards
+      }
+    }
+  }
+
+  // 3. Fallback: a broader regex extraction.
+  try {
+    const match = cleaned.match(/\[[\s\S]*\]/);
+    if (match) return JSON.parse(match[0]);
+  } catch {
+    // fall through
+  }
+  return null;
+}
 const SUMMARY_PROMPT = `你是记忆库摘要助手。根据整理后的记忆，生成一段 150-200 字的记忆库总览，覆盖：用户偏好、活跃项目、关键决策。之后作为会话上下文注入。只输出摘要文本，不要其他内容。`;
 
 const CONSOLIDATION_PROMPT = `你是记忆库整理助手。下面是全部记忆条目（id、类型、标题、内容、重要性、更新时间）。
@@ -579,18 +616,10 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
       return finish({ ok: false, error: "llm failed", summary: false });
     }
 
-    let decisions;
-    try {
-      const start = decisionText.indexOf("[");
-      const end = decisionText.lastIndexOf("]");
-      if (start === -1 || end <= start) {
-        logger?.warn?.("dsh-mneme dream: no json array in llm output");
-        return finish({ ok: false, error: "no json array in llm output", summary: false });
-      }
-      decisions = JSON.parse(decisionText.slice(start, end + 1));
-    } catch {
-      logger?.warn?.("dsh-mneme dream: invalid decisions json");
-      return finish({ ok: false, error: "invalid decisions json", summary: false });
+    const decisions = extractJsonArray(decisionText);
+    if (!Array.isArray(decisions)) {
+      logger?.warn?.(`dsh-mneme dream: no json array in llm output (raw length ${decisionText?.length ?? 0})`);
+      return finish({ ok: false, error: "no json array in llm output", summary: false });
     }
     const { ok, errors } = validateDecisions(decisions, snapshot, {
       maxUpdatePerRun: config.reflectionUpdateMaxPerRun,

package/src/hot-memory.js

@@ -0,0 +1,46 @@
+// Session-scoped hot memory (v0.5.0 召回率优化 1.3): a short-term buffer of
+// the latest dialogue rounds, kept strictly apart from the long-term memory
+// store. The injector renders it ahead of the long-term recall block so the
+// agent sees "what we were just talking about" without those rounds ever
+// being persisted as memories. Bounded two ways: maxRounds (count) and
+// maxTokens (budget) — whichever evicts first.
+
+// CJK-aware token estimate: one Chinese character ≈ 0.6 tokens (clustering
+// behavior of mainstream tokenizers), one ASCII char ≈ 0.25.
+export function estimateTokens(text) {
+  const s = String(text ?? "");
+  let cjk = 0;
+  for (const ch of s) if (ch >= "\u4e00" && ch <= "\u9fff") cjk++;
+  return Math.ceil(cjk * 0.6 + (s.length - cjk) * 0.25);
+}
+
+/**
+ * @param {{maxRounds?: number, maxTokens?: number}} opts
+ * @returns {{add(round: {query: string, response?: string}): void,
+ *            getContext(): string,
+ *            rounds(): Array, clear(): void}}
+ */
+export function createHotMemory({ maxRounds = 5, maxTokens = 2000 } = {}) {
+  const buffer = [];
+
+  function totalTokens() {
+    return buffer.reduce(
+      (sum, r) => sum + estimateTokens(`Q: ${r.query}\nA: ${r.response ?? ""}`),
+      0
+    );
+  }
+
+  return {
+    add(round) {
+      if (!round?.query) return;
+      buffer.push({ query: String(round.query), response: String(round.response ?? "") });
+      while (buffer.length > maxRounds) buffer.shift();
+      while (buffer.length > 1 && totalTokens() > maxTokens) buffer.shift();
+    },
+    getContext() {
+      return buffer.map((r) => `Q: ${r.query}\nA: ${r.response ?? ""}`).join("\n\n");
+    },
+    rounds: () => [...buffer],
+    clear() { buffer.length = 0; }
+  };
+}

package/src/inject.js

@@ -1,3 +1,5 @@
+import { createHotMemory } from "./hot-memory.js";
+
 // Best-effort extraction of the current user's latest message text from the
 // live session, for semantic-first injection (Bug4). The system-prompt
 // interpolator renders synchronously, so this walks the already-materialized
@@ -25,6 +27,50 @@ function lastUserQuery(ctx) {
   return "";
 }
 
+// Hot-memory round extraction (v0.5.0 1.3): pairs each user/message with the
+// next assistant reply from the materialized session log. Tolerates shapes
+// where assistant events carry a different type tag — anything whose payload
+// has content parts and is not a user message counts as a reply. Best-effort:
+// returns [] on any failure, and the hot block simply does not render.
+function extractRounds(ctx, maxRounds) {
+  try {
+    const events = ctx?.agent?.session?.events;
+    if (!Array.isArray(events) || events.length === 0) return [];
+    const rounds = [];
+    let pendingQuery = null;
+    const textOf = (event) => {
+      const parts = event?.data?.content;
+      if (!Array.isArray(parts)) return "";
+      return parts
+        .map((p) => (typeof p === "string" ? p : p?.text ?? ""))
+        .filter(Boolean)
+        .join("\n")
+        .trim();
+    };
+    for (const event of events) {
+      const kind = event?.data?.source?.kind;
+      const isUser = event?.type === "user/message" && (kind === undefined || kind === "user");
+      if (isUser) {
+        if (pendingQuery) rounds.push({ query: pendingQuery, response: "" });
+        pendingQuery = textOf(event).slice(0, 500);
+        continue;
+      }
+      // Only assistant-originated events close a round; tool/system events
+      // carrying text must not be mistaken for the model's reply.
+      const isAssistant = typeof event?.type === "string" && event.type.includes("assistant")
+        || kind === "assistant";
+      const body = isAssistant ? textOf(event) : "";
+      if (!body || !pendingQuery) continue;
+      rounds.push({ query: pendingQuery, response: body.slice(0, 800) });
+      pendingQuery = null;
+    }
+    if (pendingQuery) rounds.push({ query: pendingQuery, response: "" });
+    return rounds.slice(-maxRounds);
+  } catch {
+    return [];
+  }
+}
+
 export function createInjector(ctx, service, settings, config) {
   const maxItems = config.maxInjectedItems ?? 5;
   const threshold = config.importanceThreshold ?? 3;
@@ -36,6 +82,35 @@ export function createInjector(ctx, service, settings, config) {
   const MAX_CONTENT = 300;
   const MAX_BLOCK = 1500;
 
+  // Compressed injection (v0.5.0 2.1): a sleep-demoted row already carries its
+  // summary in `content` with the original parked in `_full_content` — inject
+  // the summary verbatim instead of re-truncating the (already short) text.
+  // Regular long rows keep the hard truncate.
+  function injectMemory(m, maxLength = MAX_CONTENT) {
+    if (m?._full_content) return String(m.content ?? "");
+    const text = String(m?.content ?? "");
+    return text.length <= maxLength ? text : `${text.slice(0, maxLength)}…`;
+  }
+
+  // Hot memory (v0.5.0 1.3): the latest rounds of THIS session, rebuilt from
+  // the materialized event log on every render — stateless, so it survives
+  // session switches and never persists anywhere.
+  const hot = createHotMemory({
+    maxRounds: config.hotMemoryRounds ?? 5,
+    maxTokens: config.hotMemoryMaxTokens ?? 2000
+  });
+
+  function renderHotContext(ctx) {
+    if (config.hotMemoryEnabled === false) return "";
+    const rounds = extractRounds(ctx, config.hotMemoryRounds ?? 5);
+    if (!rounds.length) return "";
+    hot.clear();
+    for (const r of rounds) hot.add(r);
+    const body = hot.getContext();
+    if (!body) return "";
+    return `[短期上下文] 最近对话（共 ${rounds.length} 轮）：\n${body}`;
+  }
+
   function render(candidates) {
     if (!candidates.length) return "";
     const header = "[记忆库] 来自 dsh-mneme 的跨会话记忆（用户偏好与高优先级项目/决策）：";
@@ -48,8 +123,7 @@ export function createInjector(ctx, service, settings, config) {
         ? "[verified] "
         : "";
       const title = `${m.title}（重要性 ${m.importance}）`;
-      let content = String(m.content ?? "");
-      if (content.length > MAX_CONTENT) content = `${content.slice(0, MAX_CONTENT)}…`;
+      const content = injectMemory(m);
       const full = `- [${m.type}] ${verified}${title}：${content}`;
       if (budget - full.length >= 0) {
         lines.push(full);
@@ -108,7 +182,14 @@ export function createInjector(ctx, service, settings, config) {
         if (query) prefetchQueryVector(query);
         const queryVector = queryVectorCache.get(query);
         const candidates = service.injectCandidates({ query, queryVector, maxItems, threshold });
-        return render(candidates);
+        // Hot memory (v0.5.0 1.3) leads the single memory block: the agent
+        // sees the short-term rounds first, then the cross-session recall —
+        // the documented injection order 1→2. Folding it here (instead of a
+        // separate context) keeps the prompt assembly stable at two blocks.
+        const hotText = renderHotContext(ctx);
+        const body = render(candidates);
+        if (!hotText) return body;
+        return body ? `${hotText}\n\n${body}` : hotText;
       }
     }),
     ctx.systemPrompt.context({

package/src/local-embedder.js

@@ -22,7 +22,13 @@ function modelHash(model) {
 
 /** Lazy default loader: dynamic import keeps module load cheap. */
 async function defaultPipelineLoader(task, model, options) {
-  const { pipeline } = await import("@huggingface/transformers");
+  const { env, pipeline } = await import("@huggingface/transformers");
+  // issue #13: transformers.js's get_tokenizer_files() drops the caller's
+  // cache_dir when it pre-checks tokenizer_config.json metadata, so the HEAD
+  // request falls back to env.cacheDir and hits the network even when the
+  // model is fully cached locally. Mirroring the cache_dir onto env.cacheDir
+  // makes that pre-check resolve locally too — fully offline loading.
+  if (options?.cache_dir) env.cacheDir = options.cache_dir;
   return pipeline(task, model, options);
 }
 

package/src/search/adaptive.js

@@ -0,0 +1,22 @@
+// Adaptive vector threshold (v0.5.0 召回率优化 1.2): replaces the fixed
+// vectorSearchThreshold=0.65 with a query-aware cutoff.
+//   entity:/attr: prefixes → 0.5  (entity recall is name-driven; loosen)
+//   very short queries      → 0.7  (<5 chars match almost anything; tighten)
+//   very long queries       → 0.6  (semantically specific; loosen a little)
+//   head-gap rule           → when the top-1 vs top-5 candidate gap exceeds
+//                             0.3 the head is decisive — loosen to 0.5 so
+//                             the tail still reaches the reranker
+//   otherwise               → 0.65 (the legacy default)
+// Pure and total: same inputs, same cutoff, no store access.
+export function adaptiveThreshold(query, candidates = []) {
+  const q = String(query ?? "");
+  if (q.startsWith("entity:") || q.startsWith("attr:")) return 0.5;
+  if (q.length > 0 && q.length < 5) return 0.7;
+  if (q.length > 50) return 0.6;
+  const scores = (Array.isArray(candidates) ? candidates : [])
+    .map((c) => (typeof c?._score === "number" ? c._score : typeof c?.score === "number" ? c.score : 0))
+    .filter((s) => s > 0)
+    .sort((a, b) => b - a);
+  if (scores.length >= 5 && scores[0] - scores[4] > 0.3) return 0.5;
+  return 0.65;
+}

package/src/search/bm25.js

@@ -0,0 +1,96 @@
+// BM25 sparse retrieval (v0.5.0 召回率优化 1.1): the third recall path beside
+// vector search and the LIKE keyword scan. The LIKE path only matches full
+// substrings, so a multi-term query ("rust 异步 tokio") misses rows whose
+// terms are scattered. BM25 scores per-token overlap with IDF weighting,
+// which is exactly the gap: identifiers, code fragments and mixed CJK/ASCII
+// queries recall rows the substring scan cannot see.
+
+// Tokenizer: ASCII words keep their shape (identifiers like "dsh-mneme" or
+// "ZFS_4421" survive as whole tokens); CJK runs become sliding bigrams
+// (unigram only for single characters), the standard workaround for BM25's
+// whitespace tokenization on Chinese.
+export function tokenize(text) {
+  const raw = String(text ?? "").toLowerCase();
+  const tokens = [];
+  const ascii = raw.match(/[a-z0-9_]+/g) ?? [];
+  tokens.push(...ascii);
+  const cjkRuns = raw.match(/[\u4e00-\u9fff]+/g) ?? [];
+  for (const run of cjkRuns) {
+    if (run.length === 1) { tokens.push(run); continue; }
+    for (let i = 0; i < run.length - 1; i++) tokens.push(run.slice(i, i + 2));
+  }
+  return tokens;
+}
+
+const K1 = 1.5; // term-frequency saturation
+const B = 0.75;  // length normalization
+
+/**
+ * Build a BM25 index over documents: [{id, title, content}].
+ * Returns { score, search }:
+ *   score(query, doc) — per-spec ad-hoc scoring (re-tokenizes the doc)
+ *   search(query, {limit}) — precomputed-tf ranking, scores normalized to
+ *     [0,1] by the max so BM25 hits can weight-blend with vector/keyword
+ *     scores on one scale. Rows the query does not touch at all are dropped.
+ */
+export function createBM25Index(documents) {
+  const docs = Array.isArray(documents) ? documents.filter(Boolean) : [];
+  const N = docs.length;
+  const df = new Map();
+  const prepared = docs.map((doc) => {
+    const tokens = tokenize(`${doc.title ?? ""} ${doc.content ?? ""}`);
+    const tf = new Map();
+    for (const t of tokens) tf.set(t, (tf.get(t) ?? 0) + 1);
+    for (const t of tf.keys()) df.set(t, (df.get(t) ?? 0) + 1);
+    return { doc, tf, len: tokens.length };
+  });
+  const avgLen = N ? prepared.reduce((s, p) => s + p.len, 0) / N : 0 || 1;
+
+  const idf = (t) => {
+    const n = df.get(t) ?? 0;
+    return Math.log((N - n + 0.5) / (n + 0.5) + 1);
+  };
+
+  function scorePrepared(queryTokens, p) {
+    let score = 0;
+    for (const t of queryTokens) {
+      const f = p.tf.get(t);
+      if (!f) continue;
+      const norm = p.len ? K1 * (1 - B + B * (p.len / avgLen)) : K1;
+      score += idf(t) * ((f * (K1 + 1)) / (f + norm));
+    }
+    return score;
+  }
+
+  return {
+    score(query, doc) {
+      const tokens = tokenize(`${doc?.title ?? ""} ${doc?.content ?? ""}`);
+      const tf = new Map();
+      for (const t of tokens) tf.set(t, (tf.get(t) ?? 0) + 1);
+      const len = tokens.length;
+      // Ad-hoc scoring can't see corpus df; fall back to tf-only saturation
+      // (df is approximated as 1 so idf ≈ log(N - 0.5 + 1) is constant).
+      let score = 0;
+      for (const t of tokenize(query)) {
+        const f = tf.get(t);
+        if (!f) continue;
+        const norm = len ? K1 * (1 - B + B * (len / avgLen)) : K1;
+        score += idf(t) * ((f * (K1 + 1)) / (f + norm));
+      }
+      return score;
+    },
+    search(query, { limit = 20 } = {}) {
+      const qTokens = tokenize(query);
+      if (!qTokens.length || !N) return [];
+      const scored = [];
+      for (const p of prepared) {
+        const s = scorePrepared(qTokens, p);
+        if (s > 0) scored.push({ row: p.doc, raw: s });
+      }
+      scored.sort((a, b) => b.raw - a.raw);
+      const top = scored.slice(0, limit);
+      const max = top[0]?.raw || 1;
+      return top.map(({ row, raw }) => ({ ...row, score: max ? raw / max : 0 }));
+    }
+  };
+}