nodebb-plugin-search-agent 0.0.932 → 0.0.934

package/lib/similarity.js

@@ -0,0 +1,125 @@
+'use strict';
+
+// Common stop-words (English + Hebrew) to exclude from TF-IDF vectors
+const STOP_WORDS = new Set([
+	// English
+	'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
+	'of', 'with', 'by', 'from', 'is', 'it', 'its', 'be', 'as', 'was',
+	'are', 'were', 'been', 'have', 'has', 'had', 'do', 'does', 'did',
+	'will', 'would', 'could', 'should', 'may', 'might', 'can', 'shall',
+	'not', 'no', 'so', 'if', 'this', 'that', 'these', 'those', 'i', 'we',
+	'you', 'he', 'she', 'they', 'my', 'your', 'his', 'her', 'our', 'their',
+	'how', 'what', 'when', 'where', 'why', 'who', 'which', 'am', 'up',
+	'out', 'about', 'into', 'than', 'more', 'also', 'me', 'him', 'us', 'them',
+	// Hebrew
+	'של', 'את', 'אל', 'על', 'עם', 'הם', 'הן', 'זה', 'זו', 'זאת',
+	'כי', 'לא', 'כן', 'יש', 'אם', 'רק', 'גם', 'אבל', 'אנחנו', 'אני',
+	'אתה', 'את', 'הוא', 'היא', 'אנו', 'אתם', 'אתן', 'הם', 'הן',
+	'זה', 'זו', 'אלה', 'אלו', 'כל', 'כך', 'כבר', 'עוד', 'רק', 'כן',
+	'אחד', 'יותר', 'פה', 'שם', 'מה', 'מי', 'איך', 'מתי', 'איפה',
+	'היה', 'הייתה', 'יהיה', 'תהיה', 'הוא', 'היא', 'הם', 'הן',
+	'אסור', 'מותר', 'צריך', 'רוצה', 'יכול', 'יכולה', 'לו', 'לה',
+	'בו', 'בה', 'עליו', 'עליה', 'בין', 'כבר', 'עכשיו', 'היום',
+	'כן', 'לכן', 'כדי', 'כדאי', 'שלי', 'שלך', 'שלו', 'שלה',
+	'שלנו', 'שלכם', 'שלהם', 'שלהן', 'להם', 'להן', 'לנו', 'לכם',
+]);
+
+/**
+ * Strip HTML, lowercase, remove punctuation, drop stop-words and short tokens.
+ * @param {string|null|undefined} text
+ * @returns {string[]}
+ */
+function tokenize(text) {
+	if (!text) return [];
+	return text
+		.replace(/<[^>]*>/g, ' ')           // strip HTML tags
+		.toLowerCase()
+		.replace(/[^\p{L}\p{N}\s]/gu, ' ') // keep all Unicode letters & digits (Hebrew, Latin, etc.)
+		.split(/\s+/)
+		.filter(t => t.length >= 2 && !STOP_WORDS.has(t)); // min 2 chars to keep short Hebrew words
+}
+
+/**
+ * Build a TF-IDF index from an array of topic objects.
+ * Each topic must have: tid, slug, title, mainPostContent (optional).
+ * @param {{ tid: number|string, slug: string, title: string, mainPostContent?: string }[]} topics
+ * @returns {{ tid: number|string, slug: string, vector: Map<string, number> }[]}
+ */
+function buildIndex(topics) {
+	if (!topics || topics.length === 0) return [];
+
+	// Step 1: term frequencies per document
+	const docs = topics.map((t) => {
+		const tokens = tokenize(`${t.title || ''} ${t.mainPostContent || ''}`);
+		const tf = new Map();
+		for (const token of tokens) {
+			tf.set(token, (tf.get(token) || 0) + 1);
+		}
+		return { tid: t.tid, slug: t.slug, tf, len: tokens.length };
+	});
+
+	// Step 2: document frequency (how many docs contain each term)
+	const df = new Map();
+	for (const doc of docs) {
+		for (const term of doc.tf.keys()) {
+			df.set(term, (df.get(term) || 0) + 1);
+		}
+	}
+
+	const N = docs.length;
+
+	// Step 3: compute TF-IDF vector per document
+	return docs.map((doc) => {
+		const vector = new Map();
+		for (const [term, freq] of doc.tf) {
+			const tf = doc.len > 0 ? freq / doc.len : 0;
+			// Smoothed IDF to avoid division by zero
+			const idf = Math.log((N + 1) / (df.get(term) + 1)) + 1;
+			vector.set(term, tf * idf);
+		}
+		return { tid: doc.tid, slug: doc.slug, vector };
+	});
+}
+
+/**
+ * Rank indexed documents against a query using cosine similarity.
+ * @param {string} queryText
+ * @param {{ tid: number|string, slug: string, vector: Map<string, number> }[]} index
+ * @param {number} [topN=10]
+ * @returns {{ tid: number|string, slug: string, score: number }[]}
+ */
+function query(queryText, index, topN = 10) {
+	if (!index || index.length === 0) return [];
+
+	const qTokens = tokenize(queryText);
+	if (qTokens.length === 0) return [];
+
+	// Build raw term-count vector for the query
+	const qVec = new Map();
+	for (const token of qTokens) {
+		qVec.set(token, (qVec.get(token) || 0) + 1);
+	}
+
+	const results = [];
+	for (const doc of index) {
+		let dot = 0;
+		let docNormSq = 0;
+		let qNormSq = 0;
+
+		for (const [term, qVal] of qVec) {
+			dot += qVal * (doc.vector.get(term) || 0);
+		}
+		for (const val of doc.vector.values()) docNormSq += val * val;
+		for (const val of qVec.values()) qNormSq += val * val;
+
+		const norm = Math.sqrt(docNormSq) * Math.sqrt(qNormSq);
+		const score = norm > 0 ? dot / norm : 0;
+		if (score > 0) {
+			results.push({ tid: doc.tid, slug: doc.slug, score });
+		}
+	}
+
+	return results.sort((a, b) => b.score - a.score).slice(0, topN);
+}
+
+module.exports = { tokenize, buildIndex, query };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nodebb-plugin-search-agent",
-  "version": "0.0.932",
+  "version": "0.0.934",
   "description": "NodeBB plugin that adds a floating chat assistant to help users find relevant forum topics using TF-IDF text similarity",
   "main": "library.js",
   "author": "Racheli Bayfus",