octalens-mentions 0.1.1 → 0.1.2

package/dist/server/index.js

@@ -85,15 +85,64 @@ async function sanitizeInput(strapi, data, auth) {
   }
 }
 const MENTION_UID$3 = "plugin::octalens-mentions.mention";
+function tokenize(text) {
+  if (!text) return [];
+  return text.toLowerCase().replace(/[^\w\s]/g, " ").split(/\s+/).filter((word) => word.length > 1);
+}
+function calculateIDF(mentions, vocabulary, titleWeight = 2) {
+  const idf = /* @__PURE__ */ new Map();
+  const N = mentions.length;
+  for (const term of vocabulary) {
+    const docsWithTerm = mentions.filter((mention2) => {
+      const titleTokens = tokenize(mention2.title || "");
+      const bodyTokens = tokenize(mention2.body || "");
+      return titleTokens.includes(term) || bodyTokens.includes(term);
+    }).length;
+    idf.set(term, Math.log((N - docsWithTerm + 0.5) / (docsWithTerm + 0.5) + 1));
+  }
+  return idf;
+}
+function bm25Score(mention2, queryTokens, idf, avgDocLength, titleWeight = 2, k1 = 1.5, b = 0.75) {
+  const titleTokens = tokenize(mention2.title || "");
+  const bodyTokens = tokenize(mention2.body || "");
+  const docLength = titleTokens.length * titleWeight + bodyTokens.length;
+  const tf = /* @__PURE__ */ new Map();
+  for (const token of titleTokens) {
+    tf.set(token, (tf.get(token) || 0) + titleWeight);
+  }
+  for (const token of bodyTokens) {
+    tf.set(token, (tf.get(token) || 0) + 1);
+  }
+  let score = 0;
+  for (const term of queryTokens) {
+    const termFreq = tf.get(term) || 0;
+    const termIdf = idf.get(term) || 0;
+    if (termFreq > 0) {
+      const numerator = termFreq * (k1 + 1);
+      const denominator = termFreq + k1 * (1 - b + b * (docLength / avgDocLength));
+      score += termIdf * (numerator / denominator);
+    }
+  }
+  return score;
+}
+function calculateAvgDocLength(mentions, titleWeight = 2) {
+  if (mentions.length === 0) return 1;
+  const totalLength = mentions.reduce((sum, mention2) => {
+    const titleLen = tokenize(mention2.title || "").length;
+    const bodyLen = tokenize(mention2.body || "").length;
+    return sum + titleLen * titleWeight + bodyLen;
+  }, 0);
+  return totalLength / mentions.length;
+}
 const searchMentionsTool = {
   name: "search_mentions",
-  description: "Search through social mentions with various filters. Use this to find mentions by content, source, author, sentiment, and more. Returns paginated results with full mention details.",
+  description: "Search through social mentions using BM25 relevance scoring. Returns results ranked by relevance when a query is provided. Supports filtering by source, author, sentiment, and more. Title matches are weighted higher than body matches for better relevance.",
   inputSchema: {
     type: "object",
     properties: {
       query: {
         type: "string",
-        description: "Search query to filter mentions by title or body content"
+        description: "Search query - uses BM25 algorithm to find and rank relevant mentions by title and body content. More specific queries yield better results."
       },
       source: {
         type: "string",
@@ -142,10 +191,8 @@ const searchMentionsTool = {
 async function handleSearchMentions(strapi, args) {
   const validatedArgs = validateToolInput("search_mentions", args);
   const { query, source, author, keyword, sentimentLabel, bookmarked, viewName, subreddit, page, pageSize, sort } = validatedArgs;
+  const TITLE_WEIGHT = 2;
   const filters = {};
-  if (query) {
-    filters.$or = [{ title: { $containsi: query } }, { body: { $containsi: query } }];
-  }
   if (source) {
     filters.source = { $eqi: source };
   }
@@ -168,6 +215,122 @@ async function handleSearchMentions(strapi, args) {
     filters.subreddit = { $containsi: subreddit };
   }
   try {
+    if (query) {
+      const queryTokens = tokenize(query);
+      if (queryTokens.length === 0) {
+        return {
+          content: [
+            {
+              type: "text",
+              text: JSON.stringify(
+                {
+                  error: true,
+                  message: "Query is empty or contains only single-character words.",
+                  query
+                },
+                null,
+                2
+              )
+            }
+          ]
+        };
+      }
+      const allMentions = await strapi.documents(MENTION_UID$3).findMany({
+        filters,
+        limit: 1e3
+        // Get up to 1000 mentions for BM25 corpus
+      });
+      if (allMentions.length === 0) {
+        return {
+          content: [
+            {
+              type: "text",
+              text: JSON.stringify(
+                {
+                  data: [],
+                  pagination: {
+                    page,
+                    pageSize,
+                    total: 0,
+                    pageCount: 0
+                  },
+                  searchInfo: {
+                    query,
+                    algorithm: "BM25",
+                    matchingResults: 0
+                  },
+                  filters: {
+                    source,
+                    author,
+                    keyword,
+                    sentimentLabel,
+                    bookmarked,
+                    viewName,
+                    subreddit
+                  }
+                },
+                null,
+                2
+              )
+            }
+          ]
+        };
+      }
+      const vocabulary = new Set(queryTokens);
+      const mentionDocs = allMentions;
+      const idf = calculateIDF(mentionDocs, vocabulary, TITLE_WEIGHT);
+      const avgDocLength = calculateAvgDocLength(mentionDocs, TITLE_WEIGHT);
+      const scoredMentions = allMentions.map((mention2) => ({
+        ...mention2,
+        bm25Score: bm25Score(mention2, queryTokens, idf, avgDocLength, TITLE_WEIGHT)
+      }));
+      const rankedMentions = scoredMentions.filter((m) => m.bm25Score > 0).sort((a, b) => b.bm25Score - a.bm25Score);
+      const total2 = rankedMentions.length;
+      const startIndex = (page - 1) * pageSize;
+      const paginatedResults = rankedMentions.slice(startIndex, startIndex + pageSize);
+      const sanitizedResults2 = await sanitizeOutput(strapi, paginatedResults);
+      const resultsWithScores = sanitizedResults2.map((mention2, index2) => ({
+        ...mention2,
+        bm25Score: Math.round(paginatedResults[index2].bm25Score * 100) / 100
+      }));
+      return {
+        content: [
+          {
+            type: "text",
+            text: JSON.stringify(
+              {
+                data: resultsWithScores,
+                pagination: {
+                  page,
+                  pageSize,
+                  total: total2,
+                  pageCount: Math.ceil(total2 / pageSize)
+                },
+                searchInfo: {
+                  query,
+                  algorithm: "BM25",
+                  titleWeight: TITLE_WEIGHT,
+                  matchingResults: total2,
+                  corpusSize: allMentions.length,
+                  hint: total2 > 0 ? "Results are ranked by relevance. Higher bm25Score indicates better match. Title matches are weighted higher than body matches." : "No matches found. Try different or fewer keywords."
+                },
+                filters: {
+                  source,
+                  author,
+                  keyword,
+                  sentimentLabel,
+                  bookmarked,
+                  viewName,
+                  subreddit
+                }
+              },
+              null,
+              2
+            )
+          }
+        ]
+      };
+    }
     const results = await strapi.documents(MENTION_UID$3).findMany({
       filters,
       sort: sort ? [sort] : ["createdAt:desc"],
@@ -190,7 +353,6 @@ async function handleSearchMentions(strapi, args) {
                 pageCount: Math.ceil(total / pageSize)
               },
               filters: {
-                query,
                 source,
                 author,
                 keyword,

package/dist/server/index.mjs

@@ -84,15 +84,64 @@ async function sanitizeInput(strapi, data, auth) {
   }
 }
 const MENTION_UID$3 = "plugin::octalens-mentions.mention";
+function tokenize(text) {
+  if (!text) return [];
+  return text.toLowerCase().replace(/[^\w\s]/g, " ").split(/\s+/).filter((word) => word.length > 1);
+}
+function calculateIDF(mentions, vocabulary, titleWeight = 2) {
+  const idf = /* @__PURE__ */ new Map();
+  const N = mentions.length;
+  for (const term of vocabulary) {
+    const docsWithTerm = mentions.filter((mention2) => {
+      const titleTokens = tokenize(mention2.title || "");
+      const bodyTokens = tokenize(mention2.body || "");
+      return titleTokens.includes(term) || bodyTokens.includes(term);
+    }).length;
+    idf.set(term, Math.log((N - docsWithTerm + 0.5) / (docsWithTerm + 0.5) + 1));
+  }
+  return idf;
+}
+function bm25Score(mention2, queryTokens, idf, avgDocLength, titleWeight = 2, k1 = 1.5, b = 0.75) {
+  const titleTokens = tokenize(mention2.title || "");
+  const bodyTokens = tokenize(mention2.body || "");
+  const docLength = titleTokens.length * titleWeight + bodyTokens.length;
+  const tf = /* @__PURE__ */ new Map();
+  for (const token of titleTokens) {
+    tf.set(token, (tf.get(token) || 0) + titleWeight);
+  }
+  for (const token of bodyTokens) {
+    tf.set(token, (tf.get(token) || 0) + 1);
+  }
+  let score = 0;
+  for (const term of queryTokens) {
+    const termFreq = tf.get(term) || 0;
+    const termIdf = idf.get(term) || 0;
+    if (termFreq > 0) {
+      const numerator = termFreq * (k1 + 1);
+      const denominator = termFreq + k1 * (1 - b + b * (docLength / avgDocLength));
+      score += termIdf * (numerator / denominator);
+    }
+  }
+  return score;
+}
+function calculateAvgDocLength(mentions, titleWeight = 2) {
+  if (mentions.length === 0) return 1;
+  const totalLength = mentions.reduce((sum, mention2) => {
+    const titleLen = tokenize(mention2.title || "").length;
+    const bodyLen = tokenize(mention2.body || "").length;
+    return sum + titleLen * titleWeight + bodyLen;
+  }, 0);
+  return totalLength / mentions.length;
+}
 const searchMentionsTool = {
   name: "search_mentions",
-  description: "Search through social mentions with various filters. Use this to find mentions by content, source, author, sentiment, and more. Returns paginated results with full mention details.",
+  description: "Search through social mentions using BM25 relevance scoring. Returns results ranked by relevance when a query is provided. Supports filtering by source, author, sentiment, and more. Title matches are weighted higher than body matches for better relevance.",
   inputSchema: {
     type: "object",
     properties: {
       query: {
         type: "string",
-        description: "Search query to filter mentions by title or body content"
+        description: "Search query - uses BM25 algorithm to find and rank relevant mentions by title and body content. More specific queries yield better results."
       },
       source: {
         type: "string",
@@ -141,10 +190,8 @@ const searchMentionsTool = {
 async function handleSearchMentions(strapi, args) {
   const validatedArgs = validateToolInput("search_mentions", args);
   const { query, source, author, keyword, sentimentLabel, bookmarked, viewName, subreddit, page, pageSize, sort } = validatedArgs;
+  const TITLE_WEIGHT = 2;
   const filters = {};
-  if (query) {
-    filters.$or = [{ title: { $containsi: query } }, { body: { $containsi: query } }];
-  }
   if (source) {
     filters.source = { $eqi: source };
   }
@@ -167,6 +214,122 @@ async function handleSearchMentions(strapi, args) {
     filters.subreddit = { $containsi: subreddit };
   }
   try {
+    if (query) {
+      const queryTokens = tokenize(query);
+      if (queryTokens.length === 0) {
+        return {
+          content: [
+            {
+              type: "text",
+              text: JSON.stringify(
+                {
+                  error: true,
+                  message: "Query is empty or contains only single-character words.",
+                  query
+                },
+                null,
+                2
+              )
+            }
+          ]
+        };
+      }
+      const allMentions = await strapi.documents(MENTION_UID$3).findMany({
+        filters,
+        limit: 1e3
+        // Get up to 1000 mentions for BM25 corpus
+      });
+      if (allMentions.length === 0) {
+        return {
+          content: [
+            {
+              type: "text",
+              text: JSON.stringify(
+                {
+                  data: [],
+                  pagination: {
+                    page,
+                    pageSize,
+                    total: 0,
+                    pageCount: 0
+                  },
+                  searchInfo: {
+                    query,
+                    algorithm: "BM25",
+                    matchingResults: 0
+                  },
+                  filters: {
+                    source,
+                    author,
+                    keyword,
+                    sentimentLabel,
+                    bookmarked,
+                    viewName,
+                    subreddit
+                  }
+                },
+                null,
+                2
+              )
+            }
+          ]
+        };
+      }
+      const vocabulary = new Set(queryTokens);
+      const mentionDocs = allMentions;
+      const idf = calculateIDF(mentionDocs, vocabulary, TITLE_WEIGHT);
+      const avgDocLength = calculateAvgDocLength(mentionDocs, TITLE_WEIGHT);
+      const scoredMentions = allMentions.map((mention2) => ({
+        ...mention2,
+        bm25Score: bm25Score(mention2, queryTokens, idf, avgDocLength, TITLE_WEIGHT)
+      }));
+      const rankedMentions = scoredMentions.filter((m) => m.bm25Score > 0).sort((a, b) => b.bm25Score - a.bm25Score);
+      const total2 = rankedMentions.length;
+      const startIndex = (page - 1) * pageSize;
+      const paginatedResults = rankedMentions.slice(startIndex, startIndex + pageSize);
+      const sanitizedResults2 = await sanitizeOutput(strapi, paginatedResults);
+      const resultsWithScores = sanitizedResults2.map((mention2, index2) => ({
+        ...mention2,
+        bm25Score: Math.round(paginatedResults[index2].bm25Score * 100) / 100
+      }));
+      return {
+        content: [
+          {
+            type: "text",
+            text: JSON.stringify(
+              {
+                data: resultsWithScores,
+                pagination: {
+                  page,
+                  pageSize,
+                  total: total2,
+                  pageCount: Math.ceil(total2 / pageSize)
+                },
+                searchInfo: {
+                  query,
+                  algorithm: "BM25",
+                  titleWeight: TITLE_WEIGHT,
+                  matchingResults: total2,
+                  corpusSize: allMentions.length,
+                  hint: total2 > 0 ? "Results are ranked by relevance. Higher bm25Score indicates better match. Title matches are weighted higher than body matches." : "No matches found. Try different or fewer keywords."
+                },
+                filters: {
+                  source,
+                  author,
+                  keyword,
+                  sentimentLabel,
+                  bookmarked,
+                  viewName,
+                  subreddit
+                }
+              },
+              null,
+              2
+            )
+          }
+        ]
+      };
+    }
     const results = await strapi.documents(MENTION_UID$3).findMany({
       filters,
       sort: sort ? [sort] : ["createdAt:desc"],
@@ -189,7 +352,6 @@ async function handleSearchMentions(strapi, args) {
                 pageCount: Math.ceil(total / pageSize)
               },
               filters: {
-                query,
                 source,
                 author,
                 keyword,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "octalens-mentions",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "A Strapi v5 plugin that fetches social mentions from Octolens and exposes them via MCP (Model Context Protocol), enabling AI assistants like Claude to search, analyze, and help write responses to social media mentions.",
   "keywords": [
     "strapi",