@modusensus/dsh-mneme 0.6.8 → 0.6.10

package/src/search/bm25.js

@@ -1,96 +1,96 @@
-// BM25 sparse retrieval (v0.5.0 召回率优化 1.1): the third recall path beside
-// vector search and the LIKE keyword scan. The LIKE path only matches full
-// substrings, so a multi-term query ("rust 异步 tokio") misses rows whose
-// terms are scattered. BM25 scores per-token overlap with IDF weighting,
-// which is exactly the gap: identifiers, code fragments and mixed CJK/ASCII
-// queries recall rows the substring scan cannot see.
-
-// Tokenizer: ASCII words keep their shape (identifiers like "dsh-mneme" or
-// "ZFS_4421" survive as whole tokens); CJK runs become sliding bigrams
-// (unigram only for single characters), the standard workaround for BM25's
-// whitespace tokenization on Chinese.
-export function tokenize(text) {
-  const raw = String(text ?? "").toLowerCase();
-  const tokens = [];
-  const ascii = raw.match(/[a-z0-9_]+/g) ?? [];
-  tokens.push(...ascii);
-  const cjkRuns = raw.match(/[\u4e00-\u9fff]+/g) ?? [];
-  for (const run of cjkRuns) {
-    if (run.length === 1) { tokens.push(run); continue; }
-    for (let i = 0; i < run.length - 1; i++) tokens.push(run.slice(i, i + 2));
-  }
-  return tokens;
-}
-
-const K1 = 1.5; // term-frequency saturation
-const B = 0.75;  // length normalization
-
-/**
- * Build a BM25 index over documents: [{id, title, content}].
- * Returns { score, search }:
- *   score(query, doc) — per-spec ad-hoc scoring (re-tokenizes the doc)
- *   search(query, {limit}) — precomputed-tf ranking, scores normalized to
- *     [0,1] by the max so BM25 hits can weight-blend with vector/keyword
- *     scores on one scale. Rows the query does not touch at all are dropped.
- */
-export function createBM25Index(documents) {
-  const docs = Array.isArray(documents) ? documents.filter(Boolean) : [];
-  const N = docs.length;
-  const df = new Map();
-  const prepared = docs.map((doc) => {
-    const tokens = tokenize(`${doc.title ?? ""} ${doc.content ?? ""}`);
-    const tf = new Map();
-    for (const t of tokens) tf.set(t, (tf.get(t) ?? 0) + 1);
-    for (const t of tf.keys()) df.set(t, (df.get(t) ?? 0) + 1);
-    return { doc, tf, len: tokens.length };
-  });
-  const avgLen = N ? prepared.reduce((s, p) => s + p.len, 0) / N : 0 || 1;
-
-  const idf = (t) => {
-    const n = df.get(t) ?? 0;
-    return Math.log((N - n + 0.5) / (n + 0.5) + 1);
-  };
-
-  function scorePrepared(queryTokens, p) {
-    let score = 0;
-    for (const t of queryTokens) {
-      const f = p.tf.get(t);
-      if (!f) continue;
-      const norm = p.len ? K1 * (1 - B + B * (p.len / avgLen)) : K1;
-      score += idf(t) * ((f * (K1 + 1)) / (f + norm));
-    }
-    return score;
-  }
-
-  return {
-    score(query, doc) {
-      const tokens = tokenize(`${doc?.title ?? ""} ${doc?.content ?? ""}`);
-      const tf = new Map();
-      for (const t of tokens) tf.set(t, (tf.get(t) ?? 0) + 1);
-      const len = tokens.length;
-      // Ad-hoc scoring can't see corpus df; fall back to tf-only saturation
-      // (df is approximated as 1 so idf ≈ log(N - 0.5 + 1) is constant).
-      let score = 0;
-      for (const t of tokenize(query)) {
-        const f = tf.get(t);
-        if (!f) continue;
-        const norm = len ? K1 * (1 - B + B * (len / avgLen)) : K1;
-        score += idf(t) * ((f * (K1 + 1)) / (f + norm));
-      }
-      return score;
-    },
-    search(query, { limit = 20 } = {}) {
-      const qTokens = tokenize(query);
-      if (!qTokens.length || !N) return [];
-      const scored = [];
-      for (const p of prepared) {
-        const s = scorePrepared(qTokens, p);
-        if (s > 0) scored.push({ row: p.doc, raw: s });
-      }
-      scored.sort((a, b) => b.raw - a.raw);
-      const top = scored.slice(0, limit);
-      const max = top[0]?.raw || 1;
-      return top.map(({ row, raw }) => ({ ...row, score: max ? raw / max : 0 }));
-    }
-  };
-}
+// BM25 sparse retrieval (v0.5.0 召回率优化 1.1): the third recall path beside
+// vector search and the LIKE keyword scan. The LIKE path only matches full
+// substrings, so a multi-term query ("rust 异步 tokio") misses rows whose
+// terms are scattered. BM25 scores per-token overlap with IDF weighting,
+// which is exactly the gap: identifiers, code fragments and mixed CJK/ASCII
+// queries recall rows the substring scan cannot see.
+
+// Tokenizer: ASCII words keep their shape (identifiers like "dsh-mneme" or
+// "ZFS_4421" survive as whole tokens); CJK runs become sliding bigrams
+// (unigram only for single characters), the standard workaround for BM25's
+// whitespace tokenization on Chinese.
+export function tokenize(text) {
+  const raw = String(text ?? "").toLowerCase();
+  const tokens = [];
+  const ascii = raw.match(/[a-z0-9_]+/g) ?? [];
+  tokens.push(...ascii);
+  const cjkRuns = raw.match(/[\u4e00-\u9fff]+/g) ?? [];
+  for (const run of cjkRuns) {
+    if (run.length === 1) { tokens.push(run); continue; }
+    for (let i = 0; i < run.length - 1; i++) tokens.push(run.slice(i, i + 2));
+  }
+  return tokens;
+}
+
+const K1 = 1.5; // term-frequency saturation
+const B = 0.75;  // length normalization
+
+/**
+ * Build a BM25 index over documents: [{id, title, content}].
+ * Returns { score, search }:
+ *   score(query, doc) — per-spec ad-hoc scoring (re-tokenizes the doc)
+ *   search(query, {limit}) — precomputed-tf ranking, scores normalized to
+ *     [0,1] by the max so BM25 hits can weight-blend with vector/keyword
+ *     scores on one scale. Rows the query does not touch at all are dropped.
+ */
+export function createBM25Index(documents) {
+  const docs = Array.isArray(documents) ? documents.filter(Boolean) : [];
+  const N = docs.length;
+  const df = new Map();
+  const prepared = docs.map((doc) => {
+    const tokens = tokenize(`${doc.title ?? ""} ${doc.content ?? ""}`);
+    const tf = new Map();
+    for (const t of tokens) tf.set(t, (tf.get(t) ?? 0) + 1);
+    for (const t of tf.keys()) df.set(t, (df.get(t) ?? 0) + 1);
+    return { doc, tf, len: tokens.length };
+  });
+  const avgLen = N ? prepared.reduce((s, p) => s + p.len, 0) / N : 0 || 1;
+
+  const idf = (t) => {
+    const n = df.get(t) ?? 0;
+    return Math.log((N - n + 0.5) / (n + 0.5) + 1);
+  };
+
+  function scorePrepared(queryTokens, p) {
+    let score = 0;
+    for (const t of queryTokens) {
+      const f = p.tf.get(t);
+      if (!f) continue;
+      const norm = p.len ? K1 * (1 - B + B * (p.len / avgLen)) : K1;
+      score += idf(t) * ((f * (K1 + 1)) / (f + norm));
+    }
+    return score;
+  }
+
+  return {
+    score(query, doc) {
+      const tokens = tokenize(`${doc?.title ?? ""} ${doc?.content ?? ""}`);
+      const tf = new Map();
+      for (const t of tokens) tf.set(t, (tf.get(t) ?? 0) + 1);
+      const len = tokens.length;
+      // Ad-hoc scoring can't see corpus df; fall back to tf-only saturation
+      // (df is approximated as 1 so idf ≈ log(N - 0.5 + 1) is constant).
+      let score = 0;
+      for (const t of tokenize(query)) {
+        const f = tf.get(t);
+        if (!f) continue;
+        const norm = len ? K1 * (1 - B + B * (len / avgLen)) : K1;
+        score += idf(t) * ((f * (K1 + 1)) / (f + norm));
+      }
+      return score;
+    },
+    search(query, { limit = 20 } = {}) {
+      const qTokens = tokenize(query);
+      if (!qTokens.length || !N) return [];
+      const scored = [];
+      for (const p of prepared) {
+        const s = scorePrepared(qTokens, p);
+        if (s > 0) scored.push({ row: p.doc, raw: s });
+      }
+      scored.sort((a, b) => b.raw - a.raw);
+      const top = scored.slice(0, limit);
+      const max = top[0]?.raw || 1;
+      return top.map(({ row, raw }) => ({ ...row, score: max ? raw / max : 0 }));
+    }
+  };
+}

package/src/search/tag-boost.js

@@ -1,61 +1,61 @@
-// v0.6.4 Tag-weighted re-rank helpers.
-// Query tags come from `#hashtags` in the query plus any known tags the query
-// mentions; candidates whose tags overlap the query tags or the session's
-// hot-memory tags get their score boosted before the final top-K cut.
-
-/**
- * Extract tags from a query: explicit `#hashtags` (CJK allowed) plus known
- * tags whose lowercase form appears in the query. Deduped, order-preserving.
- */
-export function extractQueryTags(query, knownTags = []) {
-  if (typeof query !== "string" || !query) return [];
-  const set = new Set();
-  const re = /#([a-zA-Z0-9_一-龥-]+)/g;
-  let m;
-  while ((m = re.exec(query)) !== null) set.add(m[1]);
-  const q = query.toLowerCase();
-  for (const t of knownTags) {
-    if (typeof t === "string" && q.includes(t.toLowerCase())) set.add(t);
-  }
-  return Array.from(set);
-}
-
-function hasOverlap(a, b) {
-  if (!Array.isArray(a) || !Array.isArray(b)) return false;
-  const setB = new Set(b);
-  return a.some((x) => setB.has(x));
-}
-
-/**
- * Boost candidate scores by tag overlap with the query and/or session tags.
- * Each candidate is expected to carry a `tags` array (enriched by the caller).
- * Returns a new array sorted by boosted score descending; candidates that were
- * boosted are marked `tagBoost: true`. Unchanged candidates keep their
- * relative order (stable).
- */
-export function applyTagBoost(
-  candidates,
-  { queryTags = [], sessionTags = [], factor = 1.15, sessionFactor = 1.08 }
-) {
-  const boosted = candidates.map((c) => {
-    const tags = c.tags ?? [];
-    let score = typeof c.score === "number" ? c.score : 0;
-    let didBoost = false;
-
-    if (queryTags.length && hasOverlap(tags, queryTags)) {
-      score = Math.min(1, score * factor);
-      didBoost = true;
-    }
-    if (sessionTags.length && hasOverlap(tags, sessionTags)) {
-      score = Math.min(1, score * sessionFactor);
-      didBoost = true;
-    }
-
-    const out = { ...c, score };
-    if (didBoost) out.tagBoost = true;
-    return out;
-  });
-
-  boosted.sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
-  return boosted;
-}
+// v0.6.4 Tag-weighted re-rank helpers.
+// Query tags come from `#hashtags` in the query plus any known tags the query
+// mentions; candidates whose tags overlap the query tags or the session's
+// hot-memory tags get their score boosted before the final top-K cut.
+
+/**
+ * Extract tags from a query: explicit `#hashtags` (CJK allowed) plus known
+ * tags whose lowercase form appears in the query. Deduped, order-preserving.
+ */
+export function extractQueryTags(query, knownTags = []) {
+  if (typeof query !== "string" || !query) return [];
+  const set = new Set();
+  const re = /#([a-zA-Z0-9_一-龥-]+)/g;
+  let m;
+  while ((m = re.exec(query)) !== null) set.add(m[1]);
+  const q = query.toLowerCase();
+  for (const t of knownTags) {
+    if (typeof t === "string" && q.includes(t.toLowerCase())) set.add(t);
+  }
+  return Array.from(set);
+}
+
+function hasOverlap(a, b) {
+  if (!Array.isArray(a) || !Array.isArray(b)) return false;
+  const setB = new Set(b);
+  return a.some((x) => setB.has(x));
+}
+
+/**
+ * Boost candidate scores by tag overlap with the query and/or session tags.
+ * Each candidate is expected to carry a `tags` array (enriched by the caller).
+ * Returns a new array sorted by boosted score descending; candidates that were
+ * boosted are marked `tagBoost: true`. Unchanged candidates keep their
+ * relative order (stable).
+ */
+export function applyTagBoost(
+  candidates,
+  { queryTags = [], sessionTags = [], factor = 1.15, sessionFactor = 1.08 }
+) {
+  const boosted = candidates.map((c) => {
+    const tags = c.tags ?? [];
+    let score = typeof c.score === "number" ? c.score : 0;
+    let didBoost = false;
+
+    if (queryTags.length && hasOverlap(tags, queryTags)) {
+      score = Math.min(1, score * factor);
+      didBoost = true;
+    }
+    if (sessionTags.length && hasOverlap(tags, sessionTags)) {
+      score = Math.min(1, score * sessionFactor);
+      didBoost = true;
+    }
+
+    const out = { ...c, score };
+    if (didBoost) out.tagBoost = true;
+    return out;
+  });
+
+  boosted.sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
+  return boosted;
+}