mumpix 1.0.19 → 1.1.3

package/src/core/recall.js

@@ -1,176 +1,290 @@
-'use strict';
-
-/**
- * MumpixRecall — hybrid retrieval engine
- *
- * Strategy (in order):
- *   1. Exact substring match (zero-latency)
- *   2. TF-IDF cosine similarity (local semantic approximation, no API needed)
- *   3. Token overlap fallback (always produces a result)
- *
- * Optional: pass embedFn to use your own embeddings (OpenAI, Cohere, etc.)
- */
-
-// ── Stopwords ────────────────────────────────────
+"use strict";
+
+const DEFAULT_MIN_SCORE = 0.15;
+
 const STOPWORDS = new Set([
-  'a','an','the','is','are','was','were','be','been','being',
-  'have','has','had','do','does','did','will','would','could',
-  'should','may','might','i','you','he','she','it','we','they',
-  'my','your','his','her','its','our','their','what','which',
-  'who','whom','that','this','these','those','and','but','or',
-  'nor','for','so','yet','in','on','at','to','of','up','by',
-  'with','about','into','through','during','before','after',
-  'above','below','from','out','off','over','under','again',
-  'then','once','here','there','when','where','why','how','all',
-  'both','each','few','more','most','other','some','such','no',
-  'not','only','own','same','than','too','very','just','can',
-  'me','him','us','them','am','get','got','put','set','let',
-  'if','as','also','even','still','already','now', 'do', 'did',
+  "a",
+  "an",
+  "the",
+  "is",
+  "are",
+  "was",
+  "were",
+  "be",
+  "been",
+  "being",
+  "have",
+  "has",
+  "had",
+  "do",
+  "does",
+  "did",
+  "will",
+  "would",
+  "could",
+  "should",
+  "may",
+  "might",
+  "i",
+  "you",
+  "he",
+  "she",
+  "it",
+  "we",
+  "they",
+  "my",
+  "your",
+  "his",
+  "her",
+  "its",
+  "our",
+  "their",
+  "what",
+  "which",
+  "who",
+  "whom",
+  "that",
+  "this",
+  "these",
+  "those",
+  "and",
+  "but",
+  "or",
+  "nor",
+  "for",
+  "so",
+  "yet",
+  "in",
+  "on",
+  "at",
+  "to",
+  "of",
+  "up",
+  "by",
+  "with",
+  "about",
+  "into",
+  "through",
+  "during",
+  "before",
+  "after",
+  "above",
+  "below",
+  "from",
+  "out",
+  "off",
+  "over",
+  "under",
+  "again",
+  "then",
+  "once",
+  "here",
+  "there",
+  "when",
+  "where",
+  "why",
+  "how",
+  "all",
+  "both",
+  "each",
+  "few",
+  "more",
+  "most",
+  "other",
+  "some",
+  "such",
+  "no",
+  "not",
+  "only",
+  "own",
+  "same",
+  "than",
+  "too",
+  "very",
+  "just",
+  "can",
+  "me",
+  "him",
+  "us",
+  "them",
+  "am",
+  "get",
+  "got",
+  "put",
+  "set",
+  "let",
+  "if",
+  "as",
+  "also",
+  "even",
+  "still",
+  "already",
+  "now",
 ]);
 
-// ── TF-IDF utilities ─────────────────────────────
-
 function tokenize(text) {
-  return text
+  return String(text || "")
     .toLowerCase()
-    .replace(/[^a-z0-9\s'-]/g, ' ')
+    .replace(/[^a-z0-9\s'-]/g, " ")
     .split(/\s+/)
-    .filter(t => t.length > 1 && !STOPWORDS.has(t));
+    .filter((token) => token.length > 1 && !STOPWORDS.has(token));
 }
 
 function tf(tokens) {
   const freq = {};
-  for (const t of tokens) freq[t] = (freq[t] || 0) + 1;
+  for (const token of tokens) freq[token] = (freq[token] || 0) + 1;
+
   const len = tokens.length || 1;
   const out = {};
-  for (const [t, c] of Object.entries(freq)) out[t] = c / len;
+  for (const [token, count] of Object.entries(freq)) out[token] = count / len;
   return out;
 }
 
 function buildIDF(corpus) {
   const df = {};
-  const N  = corpus.length;
+  const count = corpus.length;
+
   for (const doc of corpus) {
-    const seen = new Set(doc);
-    for (const t of seen) df[t] = (df[t] || 0) + 1;
+    for (const token of new Set(doc)) df[token] = (df[token] || 0) + 1;
   }
+
   const idf = {};
-  for (const [t, c] of Object.entries(df)) {
-    idf[t] = Math.log((N + 1) / (c + 1)) + 1;
+  for (const [token, docCount] of Object.entries(df)) {
+    idf[token] = Math.log((count + 1) / (docCount + 1)) + 1;
   }
   return idf;
 }
 
 function tfidfVec(tfMap, idf) {
   const vec = {};
-  for (const [t, w] of Object.entries(tfMap)) {
-    vec[t] = w * (idf[t] || 1);
+  for (const [token, weight] of Object.entries(tfMap)) {
+    vec[token] = weight * (idf[token] || 1);
   }
   return vec;
 }
 
 function cosine(a, b) {
-  let dot = 0, normA = 0, normB = 0;
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
   const keys = new Set([...Object.keys(a), ...Object.keys(b)]);
-  for (const k of keys) {
-    const va = a[k] || 0;
-    const vb = b[k] || 0;
-    dot   += va * vb;
-    normA += va * va;
-    normB += vb * vb;
+
+  for (const key of keys) {
+    const av = a[key] || 0;
+    const bv = b[key] || 0;
+    dot += av * bv;
+    normA += av * av;
+    normB += bv * bv;
   }
+
   const denom = Math.sqrt(normA) * Math.sqrt(normB);
   return denom === 0 ? 0 : dot / denom;
 }
 
-// ── Token overlap (tie-breaker / fallback) ───────
+function cosineArrays(a, b) {
+  if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return 0;
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+
+  const denom = Math.sqrt(normA) * Math.sqrt(normB);
+  return denom === 0 ? 0 : dot / denom;
+}
 
 function tokenOverlap(queryTokens, docTokens) {
   if (!queryTokens.length) return 0;
   const docSet = new Set(docTokens);
-  const hits   = queryTokens.filter(t => docSet.has(t)).length;
-  return hits / queryTokens.length;
+  return queryTokens.filter((token) => docSet.has(token)).length / queryTokens.length;
+}
+
+function minScore(opts) {
+  if (opts.minScore === null || opts.minScore === false) return -Infinity;
+  return Number.isFinite(opts.minScore) ? Number(opts.minScore) : DEFAULT_MIN_SCORE;
+}
+
+function sortScored(a, b) {
+  if (b.score !== a.score) return b.score - a.score;
+  const bTs = Number.isFinite(b.r && b.r.ts) ? b.r.ts : 0;
+  const aTs = Number.isFinite(a.r && a.r.ts) ? a.r.ts : 0;
+  if (bTs !== aTs) return bTs - aTs;
+  const aId = Number.isFinite(a.r && a.r.id) ? a.r.id : 0;
+  const bId = Number.isFinite(b.r && b.r.id) ? b.r.id : 0;
+  return aId - bId;
 }
 
-// ── Main recall function ──────────────────────────
-
-/**
- * recall(query, records, opts) → Record | null
- *
- * opts.k        — number of results to return (default 1)
- * opts.embedFn  — async fn(texts[]) → number[][] for custom embeddings
- * opts.filter   — fn(record) → bool for pre-filtering
- * opts.since    — timestamp: only consider records newer than this
- * opts.mode     — "exact" | "semantic" | "hybrid" (default "hybrid")
- */
 async function recall(query, records, opts = {}) {
   const results = await recallMany(query, records, { ...opts, k: opts.k || 1 });
   return results.length ? results[0] : null;
 }
 
 async function recallMany(query, records, opts = {}) {
-  const k      = opts.k      || 5;
-  const mode   = opts.mode   || 'hybrid';
+  const k = opts.k || 5;
+  const mode = opts.mode || "hybrid";
   const filter = opts.filter || null;
-  const since  = opts.since  || null;
+  const since = opts.since || null;
+  const threshold = minScore(opts);
 
   let pool = records;
   if (filter) pool = pool.filter(filter);
-  if (since)  pool = pool.filter(r => r.ts >= since);
+  if (since) pool = pool.filter((record) => record.ts >= since);
   if (!pool.length) return [];
 
-  // 1. Exact match shortcut
-  const queryLower = query.toLowerCase();
-  if (mode !== 'semantic') {
-    const exact = pool.filter(r => r.content.toLowerCase().includes(queryLower));
-    if (exact.length >= k && mode === 'exact') return exact.slice(0, k);
-    if (exact.length && mode === 'exact') return exact;
+  const queryLower = String(query || "").toLowerCase();
+  if (mode !== "semantic") {
+    const exact = pool
+      .filter((record) => String(record.content || "").toLowerCase().includes(queryLower))
+      .map((record) => ({ ...record, _score: 1 }));
+
+    if (mode === "exact") return exact.slice(0, k);
+    if (exact.length >= k) return exact.slice(0, k);
   }
 
-  // 2. Custom embeddings
-  if (opts.embedFn && mode !== 'exact') {
+  if (opts.embedFn && mode !== "exact") {
     try {
-      const texts    = [query, ...pool.map(r => r.content)];
-      const vectors  = await opts.embedFn(texts);
-      const qVec     = vectors[0];
-      const scored   = pool.map((r, i) => ({ r, score: cosineArrays(qVec, vectors[i + 1]) }));
-      scored.sort((a, b) => b.score - a.score);
-      return scored.slice(0, k).map(s => ({ ...s.r, _score: s.score }));
-    } catch (_) { /* fall through to TF-IDF */ }
+      const texts = [query, ...pool.map((record) => record.content)];
+      const vectors = await opts.embedFn(texts);
+      const qVec = vectors[0];
+      const scored = pool
+        .map((record, index) => ({ r: record, score: cosineArrays(qVec, vectors[index + 1]) }))
+        .filter((item) => item.score >= threshold);
+
+      scored.sort(sortScored);
+      return scored.slice(0, k).map((item) => ({ ...item.r, _score: item.score }));
+    } catch (_) {
+      // Fall through to local TF-IDF.
+    }
   }
 
-  // 3. TF-IDF semantic
-  const qTokens   = tokenize(query);
-  const docTokens = pool.map(r => tokenize(r.content));
-  const corpus    = [qTokens, ...docTokens];
-  const idf       = buildIDF(corpus);
-
-  const qTF  = tf(qTokens);
-  const qVec = tfidfVec(qTF, idf);
-
-  const scored = pool.map((r, i) => {
-    const dVec  = tfidfVec(tf(docTokens[i]), idf);
-    const sem   = cosine(qVec, dVec);
-    const over  = tokenOverlap(qTokens, docTokens[i]);
-    // Blend: 70% semantic + 30% overlap, with recency boost
-    const recency = Math.exp(-(Date.now() - r.ts) / (1000 * 60 * 60 * 24 * 7)); // 7-day half-life
-    const score = (sem * 0.70) + (over * 0.20) + (recency * 0.10);
-    return { r, score, _debug: { sem, over, recency } };
-  });
-
-  scored.sort((a, b) => b.score - a.score);
-  return scored.slice(0, k).map(s => ({ ...s.r, _score: s.score }));
-}
+  const qTokens = tokenize(query);
+  if (!qTokens.length) return [];
 
-function cosineArrays(a, b) {
-  let dot = 0, normA = 0, normB = 0;
-  for (let i = 0; i < a.length; i++) {
-    dot   += a[i] * b[i];
-    normA += a[i] * a[i];
-    normB += b[i] * b[i];
-  }
-  const denom = Math.sqrt(normA) * Math.sqrt(normB);
-  return denom === 0 ? 0 : dot / denom;
+  const docTokens = pool.map((record) => tokenize(record.content));
+  const idf = buildIDF([qTokens, ...docTokens]);
+  const qVec = tfidfVec(tf(qTokens), idf);
+
+  const scored = pool
+    .map((record, index) => {
+      const dVec = tfidfVec(tf(docTokens[index]), idf);
+      const semantic = cosine(qVec, dVec);
+      const overlap = tokenOverlap(qTokens, docTokens[index]);
+      const related = semantic > 0 || overlap > 0;
+      const recency = related
+        ? Math.exp(-(Date.now() - record.ts) / (1000 * 60 * 60 * 24 * 7))
+        : 0;
+      const score = semantic * 0.7 + overlap * 0.2 + recency * 0.1;
+
+      return { r: record, score, _debug: { semantic, overlap, recency } };
+    })
+    .filter((item) => item.score >= threshold);
+
+  scored.sort(sortScored);
+  return scored.slice(0, k).map((item) => ({ ...item.r, _score: item.score }));
 }
 
 module.exports = { recall, recallMany, tokenize };