@agenticmail/enterprise 0.4.2 → 0.4.3

package/dist/chunk-TYW5XTOW.js

@@ -0,0 +1,395 @@
+// src/lib/text-search.ts
+var BM25_K1 = 1.2;
+var BM25_B = 0.75;
+var FIELD_WEIGHT_TITLE = 3;
+var FIELD_WEIGHT_TAGS = 2;
+var FIELD_WEIGHT_CONTENT = 1;
+var PREFIX_MATCH_PENALTY = 0.7;
+var STOP_WORDS = /* @__PURE__ */ new Set([
+  "a",
+  "about",
+  "above",
+  "after",
+  "again",
+  "against",
+  "all",
+  "am",
+  "an",
+  "and",
+  "any",
+  "are",
+  "as",
+  "at",
+  "be",
+  "because",
+  "been",
+  "before",
+  "being",
+  "below",
+  "between",
+  "both",
+  "but",
+  "by",
+  "can",
+  "could",
+  "did",
+  "do",
+  "does",
+  "doing",
+  "down",
+  "during",
+  "each",
+  "either",
+  "every",
+  "few",
+  "for",
+  "from",
+  "further",
+  "get",
+  "got",
+  "had",
+  "has",
+  "have",
+  "having",
+  "he",
+  "her",
+  "here",
+  "hers",
+  "herself",
+  "him",
+  "himself",
+  "his",
+  "how",
+  "i",
+  "if",
+  "in",
+  "into",
+  "is",
+  "it",
+  "its",
+  "itself",
+  "just",
+  "may",
+  "me",
+  "might",
+  "more",
+  "most",
+  "must",
+  "my",
+  "myself",
+  "neither",
+  "no",
+  "nor",
+  "not",
+  "now",
+  "of",
+  "off",
+  "on",
+  "once",
+  "only",
+  "or",
+  "other",
+  "ought",
+  "our",
+  "ours",
+  "ourselves",
+  "out",
+  "over",
+  "own",
+  "same",
+  "shall",
+  "she",
+  "should",
+  "so",
+  "some",
+  "such",
+  "than",
+  "that",
+  "the",
+  "their",
+  "theirs",
+  "them",
+  "themselves",
+  "then",
+  "there",
+  "these",
+  "they",
+  "this",
+  "those",
+  "through",
+  "to",
+  "too",
+  "under",
+  "until",
+  "up",
+  "us",
+  "very",
+  "was",
+  "we",
+  "were",
+  "what",
+  "when",
+  "where",
+  "which",
+  "while",
+  "who",
+  "whom",
+  "why",
+  "will",
+  "with",
+  "would",
+  "yet",
+  "you",
+  "your",
+  "yours",
+  "yourself",
+  "yourselves"
+]);
+var STEM_RULES = [
+  // Step 1: plurals and past participles
+  [/ies$/, "i", 3],
+  // policies → polici,eries → eri
+  [/sses$/, "ss", 4],
+  // addresses → address
+  [/([^s])s$/, "$1", 3],
+  // items → item, but not "ss"
+  [/eed$/, "ee", 4],
+  // agreed → agree
+  [/ed$/, "", 3],
+  // configured → configur, but min length 3
+  [/ing$/, "", 4],
+  // running → runn → run (handled below)
+  // Step 2: derivational suffixes
+  [/ational$/, "ate", 6],
+  // relational → relate
+  [/tion$/, "t", 5],
+  // adoption → adopt
+  [/ness$/, "", 5],
+  // awareness → aware
+  [/ment$/, "", 5],
+  // deployment → deploy
+  [/able$/, "", 5],
+  // configurable → configur
+  [/ible$/, "", 5],
+  // accessible → access
+  [/ful$/, "", 5],
+  // powerful → power
+  [/ous$/, "", 5],
+  // dangerous → danger
+  [/ive$/, "", 5],
+  // interactive → interact
+  [/ize$/, "", 4],
+  // normalize → normal
+  [/ise$/, "", 4],
+  // organise → organ
+  [/ally$/, "", 5],
+  // automatically → automat
+  [/ly$/, "", 4],
+  // quickly → quick
+  [/er$/, "", 4]
+  // handler → handl
+];
+var DOUBLE_CONSONANT = /([^aeiou])\1$/;
+function stem(word) {
+  if (word.length < 3) return word;
+  let stemmed = word;
+  for (const [pattern, replacement, minLen] of STEM_RULES) {
+    if (stemmed.length >= minLen && pattern.test(stemmed)) {
+      stemmed = stemmed.replace(pattern, replacement);
+      break;
+    }
+  }
+  if (stemmed.length > 2 && DOUBLE_CONSONANT.test(stemmed)) {
+    stemmed = stemmed.slice(0, -1);
+  }
+  return stemmed;
+}
+function tokenize(text) {
+  return text.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 1 && !STOP_WORDS.has(t)).map(stem);
+}
+var MemorySearchIndex = class {
+  /** Posting lists: stemmed term → Set of memory IDs containing it */
+  postings = /* @__PURE__ */ new Map();
+  /** Per-document metadata for BM25 scoring */
+  docs = /* @__PURE__ */ new Map();
+  /** Pre-computed IDF values. Stale flag triggers lazy recomputation. */
+  idf = /* @__PURE__ */ new Map();
+  idfStale = true;
+  /** 3-character prefix map for prefix matching: prefix → Set of full stems */
+  prefixMap = /* @__PURE__ */ new Map();
+  /** Total weighted document length (for computing average) */
+  totalWeightedLen = 0;
+  get docCount() {
+    return this.docs.size;
+  }
+  get avgDocLen() {
+    return this.docs.size > 0 ? this.totalWeightedLen / this.docs.size : 1;
+  }
+  /**
+   * Index a memory entry. Extracts stems from title, content, and tags
+   * with field-specific weighting and builds posting lists.
+   */
+  addDocument(id, entry) {
+    if (this.docs.has(id)) this.removeDocument(id);
+    const titleTokens = tokenize(entry.title);
+    const contentTokens = tokenize(entry.content);
+    const tagTokens = entry.tags.flatMap((t) => tokenize(t));
+    const weightedTf = /* @__PURE__ */ new Map();
+    for (const t of titleTokens) weightedTf.set(t, (weightedTf.get(t) || 0) + FIELD_WEIGHT_TITLE);
+    for (const t of tagTokens) weightedTf.set(t, (weightedTf.get(t) || 0) + FIELD_WEIGHT_TAGS);
+    for (const t of contentTokens) weightedTf.set(t, (weightedTf.get(t) || 0) + FIELD_WEIGHT_CONTENT);
+    const weightedLen = titleTokens.length * FIELD_WEIGHT_TITLE + tagTokens.length * FIELD_WEIGHT_TAGS + contentTokens.length * FIELD_WEIGHT_CONTENT;
+    const allStems = /* @__PURE__ */ new Set();
+    for (const t of weightedTf.keys()) allStems.add(t);
+    const stemSequence = [...titleTokens, ...contentTokens];
+    const docRecord = { weightedTf, weightedLen, allStems, stemSequence };
+    this.docs.set(id, docRecord);
+    this.totalWeightedLen += weightedLen;
+    for (const term of allStems) {
+      let posting = this.postings.get(term);
+      if (!posting) {
+        posting = /* @__PURE__ */ new Set();
+        this.postings.set(term, posting);
+      }
+      posting.add(id);
+      if (term.length >= 3) {
+        const prefix = term.slice(0, 3);
+        let prefixSet = this.prefixMap.get(prefix);
+        if (!prefixSet) {
+          prefixSet = /* @__PURE__ */ new Set();
+          this.prefixMap.set(prefix, prefixSet);
+        }
+        prefixSet.add(term);
+      }
+    }
+    this.idfStale = true;
+  }
+  /** Remove a document from the index. */
+  removeDocument(id) {
+    const doc = this.docs.get(id);
+    if (!doc) return;
+    this.totalWeightedLen -= doc.weightedLen;
+    this.docs.delete(id);
+    for (const term of doc.allStems) {
+      const posting = this.postings.get(term);
+      if (posting) {
+        posting.delete(id);
+        if (posting.size === 0) {
+          this.postings.delete(term);
+          if (term.length >= 3) {
+            const prefixSet = this.prefixMap.get(term.slice(0, 3));
+            if (prefixSet) {
+              prefixSet.delete(term);
+              if (prefixSet.size === 0) this.prefixMap.delete(term.slice(0, 3));
+            }
+          }
+        }
+      }
+    }
+    this.idfStale = true;
+  }
+  /** Recompute IDF values for all terms. Called lazily before search. */
+  refreshIdf() {
+    if (!this.idfStale) return;
+    const N = this.docs.size;
+    this.idf.clear();
+    for (const [term, posting] of this.postings) {
+      const df = posting.size;
+      this.idf.set(term, Math.log((N - df + 0.5) / (df + 0.5) + 1));
+    }
+    this.idfStale = false;
+  }
+  /**
+   * Expand query terms with prefix matches.
+   * "deploy" → ["deploy", "deployment", "deploying", ...] (if they exist in the index)
+   */
+  expandQueryTerms(queryStems) {
+    const expanded = /* @__PURE__ */ new Map();
+    for (const qs of queryStems) {
+      if (this.postings.has(qs)) {
+        expanded.set(qs, Math.max(expanded.get(qs) || 0, 1));
+      }
+      if (qs.length >= 3) {
+        const prefix = qs.slice(0, 3);
+        const candidates = this.prefixMap.get(prefix);
+        if (candidates) {
+          for (const candidate of candidates) {
+            if (candidate !== qs && candidate.startsWith(qs)) {
+              expanded.set(candidate, Math.max(expanded.get(candidate) || 0, PREFIX_MATCH_PENALTY));
+            }
+          }
+        }
+      }
+    }
+    return expanded;
+  }
+  /**
+   * Compute bigram proximity boost: if two query terms appear adjacent
+   * in the document's stem sequence, boost the score.
+   */
+  bigramProximityBoost(docId, queryStems) {
+    if (queryStems.length < 2) return 0;
+    const doc = this.docs.get(docId);
+    if (!doc || doc.stemSequence.length < 2) return 0;
+    let boost = 0;
+    const seq = doc.stemSequence;
+    const querySet = new Set(queryStems);
+    for (let i = 0; i < seq.length - 1; i++) {
+      if (querySet.has(seq[i]) && querySet.has(seq[i + 1]) && seq[i] !== seq[i + 1]) {
+        boost += 0.5;
+      }
+    }
+    return Math.min(boost, 2);
+  }
+  /**
+   * Search the index for documents matching a query.
+   * Returns scored results sorted by BM25F relevance.
+   *
+   * @param query - Raw query string
+   * @param candidateIds - Optional: only score these document IDs (for agent-scoped search)
+   * @returns Array of { id, score } sorted by descending score
+   */
+  search(query, candidateIds) {
+    const queryStems = tokenize(query);
+    if (queryStems.length === 0) return [];
+    this.refreshIdf();
+    const expandedTerms = this.expandQueryTerms(queryStems);
+    if (expandedTerms.size === 0) return [];
+    const avgDl = this.avgDocLen;
+    const candidates = /* @__PURE__ */ new Set();
+    for (const term of expandedTerms.keys()) {
+      const posting = this.postings.get(term);
+      if (posting) {
+        for (const docId of posting) {
+          if (!candidateIds || candidateIds.has(docId)) candidates.add(docId);
+        }
+      }
+    }
+    const results = [];
+    for (const docId of candidates) {
+      const doc = this.docs.get(docId);
+      if (!doc) continue;
+      let score = 0;
+      for (const [term, weight] of expandedTerms) {
+        const tf = doc.weightedTf.get(term) || 0;
+        if (tf === 0) continue;
+        const termIdf = this.idf.get(term) || 0;
+        const numerator = tf * (BM25_K1 + 1);
+        const denominator = tf + BM25_K1 * (1 - BM25_B + BM25_B * (doc.weightedLen / avgDl));
+        score += termIdf * (numerator / denominator) * weight;
+      }
+      score += this.bigramProximityBoost(docId, queryStems);
+      if (score > 0) results.push({ id: docId, score });
+    }
+    results.sort((a, b) => b.score - a.score);
+    return results;
+  }
+  /** Check if a document exists in the index. */
+  has(id) {
+    return this.docs.has(id);
+  }
+};
+
+export {
+  MemorySearchIndex
+};