mumpix 1.0.20 → 1.0.29

package/src/core/recall.js

@@ -1,176 +1 @@
-'use strict';
-
-/**
- * MumpixRecall — hybrid retrieval engine
- *
- * Strategy (in order):
- *   1. Exact substring match (zero-latency)
- *   2. TF-IDF cosine similarity (local semantic approximation, no API needed)
- *   3. Token overlap fallback (always produces a result)
- *
- * Optional: pass embedFn to use your own embeddings (OpenAI, Cohere, etc.)
- */
-
-// ── Stopwords ────────────────────────────────────
-const STOPWORDS = new Set([
-  'a','an','the','is','are','was','were','be','been','being',
-  'have','has','had','do','does','did','will','would','could',
-  'should','may','might','i','you','he','she','it','we','they',
-  'my','your','his','her','its','our','their','what','which',
-  'who','whom','that','this','these','those','and','but','or',
-  'nor','for','so','yet','in','on','at','to','of','up','by',
-  'with','about','into','through','during','before','after',
-  'above','below','from','out','off','over','under','again',
-  'then','once','here','there','when','where','why','how','all',
-  'both','each','few','more','most','other','some','such','no',
-  'not','only','own','same','than','too','very','just','can',
-  'me','him','us','them','am','get','got','put','set','let',
-  'if','as','also','even','still','already','now', 'do', 'did',
-]);
-
-// ── TF-IDF utilities ─────────────────────────────
-
-function tokenize(text) {
-  return text
-    .toLowerCase()
-    .replace(/[^a-z0-9\s'-]/g, ' ')
-    .split(/\s+/)
-    .filter(t => t.length > 1 && !STOPWORDS.has(t));
-}
-
-function tf(tokens) {
-  const freq = {};
-  for (const t of tokens) freq[t] = (freq[t] || 0) + 1;
-  const len = tokens.length || 1;
-  const out = {};
-  for (const [t, c] of Object.entries(freq)) out[t] = c / len;
-  return out;
-}
-
-function buildIDF(corpus) {
-  const df = {};
-  const N  = corpus.length;
-  for (const doc of corpus) {
-    const seen = new Set(doc);
-    for (const t of seen) df[t] = (df[t] || 0) + 1;
-  }
-  const idf = {};
-  for (const [t, c] of Object.entries(df)) {
-    idf[t] = Math.log((N + 1) / (c + 1)) + 1;
-  }
-  return idf;
-}
-
-function tfidfVec(tfMap, idf) {
-  const vec = {};
-  for (const [t, w] of Object.entries(tfMap)) {
-    vec[t] = w * (idf[t] || 1);
-  }
-  return vec;
-}
-
-function cosine(a, b) {
-  let dot = 0, normA = 0, normB = 0;
-  const keys = new Set([...Object.keys(a), ...Object.keys(b)]);
-  for (const k of keys) {
-    const va = a[k] || 0;
-    const vb = b[k] || 0;
-    dot   += va * vb;
-    normA += va * va;
-    normB += vb * vb;
-  }
-  const denom = Math.sqrt(normA) * Math.sqrt(normB);
-  return denom === 0 ? 0 : dot / denom;
-}
-
-// ── Token overlap (tie-breaker / fallback) ───────
-
-function tokenOverlap(queryTokens, docTokens) {
-  if (!queryTokens.length) return 0;
-  const docSet = new Set(docTokens);
-  const hits   = queryTokens.filter(t => docSet.has(t)).length;
-  return hits / queryTokens.length;
-}
-
-// ── Main recall function ──────────────────────────
-
-/**
- * recall(query, records, opts) → Record | null
- *
- * opts.k        — number of results to return (default 1)
- * opts.embedFn  — async fn(texts[]) → number[][] for custom embeddings
- * opts.filter   — fn(record) → bool for pre-filtering
- * opts.since    — timestamp: only consider records newer than this
- * opts.mode     — "exact" | "semantic" | "hybrid" (default "hybrid")
- */
-async function recall(query, records, opts = {}) {
-  const results = await recallMany(query, records, { ...opts, k: opts.k || 1 });
-  return results.length ? results[0] : null;
-}
-
-async function recallMany(query, records, opts = {}) {
-  const k      = opts.k      || 5;
-  const mode   = opts.mode   || 'hybrid';
-  const filter = opts.filter || null;
-  const since  = opts.since  || null;
-
-  let pool = records;
-  if (filter) pool = pool.filter(filter);
-  if (since)  pool = pool.filter(r => r.ts >= since);
-  if (!pool.length) return [];
-
-  // 1. Exact match shortcut
-  const queryLower = query.toLowerCase();
-  if (mode !== 'semantic') {
-    const exact = pool.filter(r => r.content.toLowerCase().includes(queryLower));
-    if (exact.length >= k && mode === 'exact') return exact.slice(0, k);
-    if (exact.length && mode === 'exact') return exact;
-  }
-
-  // 2. Custom embeddings
-  if (opts.embedFn && mode !== 'exact') {
-    try {
-      const texts    = [query, ...pool.map(r => r.content)];
-      const vectors  = await opts.embedFn(texts);
-      const qVec     = vectors[0];
-      const scored   = pool.map((r, i) => ({ r, score: cosineArrays(qVec, vectors[i + 1]) }));
-      scored.sort((a, b) => b.score - a.score);
-      return scored.slice(0, k).map(s => ({ ...s.r, _score: s.score }));
-    } catch (_) { /* fall through to TF-IDF */ }
-  }
-
-  // 3. TF-IDF semantic
-  const qTokens   = tokenize(query);
-  const docTokens = pool.map(r => tokenize(r.content));
-  const corpus    = [qTokens, ...docTokens];
-  const idf       = buildIDF(corpus);
-
-  const qTF  = tf(qTokens);
-  const qVec = tfidfVec(qTF, idf);
-
-  const scored = pool.map((r, i) => {
-    const dVec  = tfidfVec(tf(docTokens[i]), idf);
-    const sem   = cosine(qVec, dVec);
-    const over  = tokenOverlap(qTokens, docTokens[i]);
-    // Blend: 70% semantic + 30% overlap, with recency boost
-    const recency = Math.exp(-(Date.now() - r.ts) / (1000 * 60 * 60 * 24 * 7)); // 7-day half-life
-    const score = (sem * 0.70) + (over * 0.20) + (recency * 0.10);
-    return { r, score, _debug: { sem, over, recency } };
-  });
-
-  scored.sort((a, b) => b.score - a.score);
-  return scored.slice(0, k).map(s => ({ ...s.r, _score: s.score }));
-}
-
-function cosineArrays(a, b) {
-  let dot = 0, normA = 0, normB = 0;
-  for (let i = 0; i < a.length; i++) {
-    dot   += a[i] * b[i];
-    normA += a[i] * a[i];
-    normB += b[i] * b[i];
-  }
-  const denom = Math.sqrt(normA) * Math.sqrt(normB);
-  return denom === 0 ? 0 : dot / denom;
-}
-
-module.exports = { recall, recallMany, tokenize };
+"use strict";const e=new Set(["a","an","the","is","are","was","were","be","been","being","have","has","had","do","does","did","will","would","could","should","may","might","i","you","he","she","it","we","they","my","your","his","her","its","our","their","what","which","who","whom","that","this","these","those","and","but","or","nor","for","so","yet","in","on","at","to","of","up","by","with","about","into","through","during","before","after","above","below","from","out","off","over","under","again","then","once","here","there","when","where","why","how","all","both","each","few","more","most","other","some","such","no","not","only","own","same","than","too","very","just","can","me","him","us","them","am","get","got","put","set","let","if","as","also","even","still","already","now","do","did"]);function t(t){return t.toLowerCase().replace(/[^a-z0-9\s'-]/g," ").split(/\s+/).filter(t=>t.length>1&&!e.has(t))}function o(e){const t={};for(const o of e)t[o]=(t[o]||0)+1;const o=e.length||1,n={};for(const[e,r]of Object.entries(t))n[e]=r/o;return n}function n(e,t){const o={};for(const[n,r]of Object.entries(e))o[n]=r*(t[n]||1);return o}async function r(e,r,c={}){const a=c.k||5,h=c.mode||"hybrid",i=c.filter||null,l=c.since||null;let u=r;if(i&&(u=u.filter(i)),l&&(u=u.filter(e=>e.ts>=l)),!u.length)return[];const f=e.toLowerCase();if("semantic"!==h){const e=u.filter(e=>e.content.toLowerCase().includes(f));if(e.length>=a&&"exact"===h)return e.slice(0,a);if(e.length&&"exact"===h)return e}if(c.embedFn&&"exact"!==h)try{const t=[e,...u.map(e=>e.content)],o=await c.embedFn(t),n=o[0],r=u.map((e,t)=>({r:e,score:s(n,o[t+1])}));return r.sort((e,t)=>t.score-e.score),r.slice(0,a).map(e=>({...e.r,_score:e.score}))}catch(e){}const w=t(e),m=u.map(e=>t(e.content)),d=function(e){const t={},o=e.length;for(const o of e){const e=new Set(o);for(const o of e)t[o]=(t[o]||0)+1}const n={};for(const[e,r]of Object.entries(t))n[e]=Math.log((o+1)/(r+1))+1;return n}([w,...m]),g=n(o(w),d),b=u.map((e,t)=>{const r=n(o(m[t]),d),s=function(e,t){let o=0,n=0,r=0;const s=new Set([...Object.keys(e),...Object.keys(t)]);for(const c of s){const s=e[c]||0,a=t[c]||0;o+=s*a,n+=s*s,r+=a*a}const c=Math.sqrt(n)*Math.sqrt(r);return 0===c?0:o/c}(g,r),c=function(e,t){if(!e.length)return 0;const o=new Set(t);return e.filter(e=>o.has(e)).length/e.length}(w,m[t]),a=Math.exp(-(Date.now()-e.ts)/6048e5);return{r:e,score:.7*s+.2*c+.1*a,_debug:{sem:s,over:c,recency:a}}});return b.sort((e,t)=>t.score-e.score),b.slice(0,a).map(e=>({...e.r,_score:e.score}))}function s(e,t){let o=0,n=0,r=0;for(let s=0;s<e.length;s++)o+=e[s]*t[s],n+=e[s]*e[s],r+=t[s]*t[s];const s=Math.sqrt(n)*Math.sqrt(r);return 0===s?0:o/s}module.exports={recall:async function(e,t,o={}){const n=await r(e,t,{...o,k:o.k||1});return n.length?n[0]:null},recallMany:r,tokenize:t};