nodebb-plugin-search-agent 0.0.4 → 0.0.6

package/lib/controllers.js

@@ -1,6 +1,9 @@
 'use strict';
 
 const { searchTopics, getSettings } = require('./searchHandler');
+const { clearAllEmbeddings } = require('../services/vectorStore');
+const { invalidateIndex } = require('../services/vectorSearchService');
+const { startSync } = require('../services/syncService');
 
 const controllers = {};
 
@@ -66,4 +69,25 @@ controllers.getConfig = async function (req, res, helpers) {
 	}
 };
 
+/**
+ * POST /api/v3/plugins/search-agent/embeddings/resync
+ * Admin-only: wipes all stored embeddings and kicks off a full re-index.
+ */
+controllers.clearAndResync = async function (req, res, helpers) {
+	try {
+		const deleted = await clearAllEmbeddings();
+		invalidateIndex();
+		startSync();
+		require.main.require('winston').info(
+			`[search-agent] clearAndResync: deleted ${deleted} embedding(s); re-sync started by uid ${req.uid}`
+		);
+		helpers.formatApiResponse(200, res, {
+			message: `Cleared ${deleted} embedding(s). Full re-index is running in the background.`,
+		});
+	} catch (err) {
+		require.main.require('winston').error(`[search-agent] clearAndResync error: ${err.message}`);
+		helpers.formatApiResponse(500, res, new Error('Failed to clear or re-sync embeddings.'));
+	}
+};
+
 module.exports = controllers;

package/lib/searchHandler.js

@@ -122,23 +122,56 @@ function callOpenAI(apiKey, model, messages) {
 }
 
 /**
- * Send TF-IDF candidates to OpenAI and ask it to pick the most relevant ones,
- * ordered by relevance.  Falls back to original TF-IDF order on any error.
+ * HyDE: generate a hypothetical forum post that would answer the query.
+ * Embedding this richer text instead of the raw query dramatically improves
+ * cosine similarity against actual post embeddings.
+ * Falls back to the original queryText on any error.
+ *
+ * @param {string} queryText
+ * @param {string} apiKey
+ * @param {string} model
+ * @returns {Promise<string>}
+ */
+async function expandQueryWithHyDE(queryText, apiKey, model) {
+	const response = await callOpenAI(apiKey, model, [
+		{
+			role: 'system',
+			content:
+				'אתה חבר בפורום. בהינתן שאלת חיפוש, כתוב פוסט תגובה קצר וריאליסטי בפורום שעונה ישירות על השאלה. ' +
+				'כתוב רק את תוכן הפוסט — ללא ברכות, הערות מטא, או שורת נושא.',
+		},
+		{ role: 'user', content: queryText },
+	]);
+	return (response.choices[0].message.content || '').trim() || queryText;
+}
+
+/**
+ * Send candidates to OpenAI for independent per-topic relevance scoring.
+ * Each topic is rated 0-10 separately; topics scoring < 7 are excluded.
+ * This is more reliable than asking GPT to rank a list, because each topic
+ * is evaluated on its own merits rather than relative to the others.
+ * @param {object} [snippetByTid] - Map of tid → main post content snippet
  */
-async function reRankWithAI(queryText, candidates, topicMap, apiKey, model, maxResults) {
-	const candidateList = candidates
-		.map((c, i) => `${i + 1}. [tid:${c.tid}] ${(topicMap[String(c.tid)] || {}).title || ''}`)
-		.join('\n');
+async function reRankWithAI(queryText, candidates, topicMap, apiKey, model, maxResults, snippetByTid = {}) {
+	console.log('Re-ranking with AI:', { queryText, candidates: candidates.map(c => ({ tid: c.tid, title: (topicMap[String(c.tid)] || {}).title })) });
+    const candidateList = candidates
+		.map((c) => {
+			const title = (topicMap[String(c.tid)] || {}).title || '';
+			const raw = (snippetByTid[String(c.tid)] || '').replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim();
+			const snippet = raw.length > 0 ? `\n   תוכן: "${raw.slice(0, 500)}"` : '';
+			return `[tid:${c.tid}] ${title}${snippet}`;
+		})
+		.join('\n\n');
 
 	const systemPrompt =
-		'You are a forum search assistant. ' +
-		'Given a user question and a numbered list of forum topic titles, ' +
-		'respond with ONLY a JSON array of the tid values (integers after "tid:") ' +
-		'for the topics that actually answer the question, ordered from most to least relevant. ' +
-		'Include only truly relevant topics. Example response: [12, 5, 33]';
+		'אתה מסנן חיפוש פורום מחמיר. ' +
+		'לכל נושא ברשימה, דרג את הרלוונטיות שלו לשאלת המשתמש בסקלה 0-10: ' +
+		'10 = עונה ישירות ובאופן מלא. 7-9 = עונה על חלק משמעותי. 0-6 = לא רלוונטי. ' +
+		'החזר אך ורק JSON תקני במבנה: {"tid": ציון, ...} — לדוגמה: {"42": 9, "15": 3}. ' +
+		'אין להוסיף הסברים, טקסט נוסף, או עיצוב מחוץ ל-JSON.';
 
 	const userMessage =
-		`User question: "${queryText}"\n\nForum topics:\n${candidateList}`;
+		`שאלת המשתמש: "${queryText}"\n\nנושאים:\n${candidateList}`;
 
 	const response = await callOpenAI(apiKey, model, [
 		{ role: 'system', content: systemPrompt },
@@ -147,19 +180,21 @@ async function reRankWithAI(queryText, candidates, topicMap, apiKey, model, maxR
 
 	const content = (response.choices[0].message.content || '').trim();
 
-	// Robustly extract a JSON integer array from the response
-	const match = content.match(/\[[\d,\s]+\]/);
+	// Extract the JSON object from the response
+	const match = content.match(/\{[^}]*\}/);
 	if (!match) {
-		throw new Error(`Unexpected AI response format: ${content.slice(0, 100)}`);
+		throw new Error(`Unexpected AI scoring response: ${content.slice(0, 100)}`);
 	}
 
-	const rankedTids = JSON.parse(match[0]);
-	const candidateByTid = Object.fromEntries(candidates.map(c => [c.tid, c]));
+	const scores = JSON.parse(match[0]);
+	const candidateByTid = Object.fromEntries(candidates.map(c => [String(c.tid), c]));
 
-	return rankedTids
-		.map(tid => candidateByTid[tid])
-		.filter(Boolean)
-		.slice(0, maxResults);
+	return Object.entries(scores)
+		.filter(([, score]) => Number(score) >= 7)
+		.sort(([, a], [, b]) => Number(b) - Number(a))
+		.slice(0, maxResults)
+		.map(([tid]) => candidateByTid[tid])
+		.filter(Boolean);
 }
 
 // ─── Public API ───────────────────────────────────────────────────────────────
@@ -178,24 +213,80 @@ async function searchTopics(queryText) {
 	// ── Semantic search (primary) ────────────────────────────────────────────
 	try {
 		const { search: vectorSearch } = require('../services/vectorSearchService');
-		const vectorResults = await vectorSearch(queryText);
+		const useAI = settings.aiEnabled && settings.openaiApiKey;
+
+		// HyDE: replace the short raw query with a hypothetical answer so the
+		// embedding matches post content more closely.
+		let embeddingQuery = queryText;
+		if (useAI) {
+			try {
+				embeddingQuery = await expandQueryWithHyDE(
+					queryText, settings.openaiApiKey, settings.openaiModel
+				);
+				winston.verbose(`[search-agent] HyDE expanded query (${embeddingQuery.length} chars)`);
+			} catch (hydeErr) {
+				winston.warn(`[search-agent] HyDE expansion failed, using raw query: ${hydeErr.message}`);
+			}
+		}
+
+		// Request more candidates when AI will re-rank them.
+		const vectorLimit = useAI ? settings.aiCandidates : settings.maxResults;
+		const vectorResults = await vectorSearch(embeddingQuery, vectorLimit);
 
 		if (vectorResults.length > 0) {
 			const Topics = require.main.require('./src/topics');
+			const Posts = require.main.require('./src/posts');
 			const tids = [...new Set(vectorResults.map(r => r.topic_id))];
-			const topics = await Topics.getTopicsFields(tids, ['tid', 'title', 'slug', 'deleted']);
+			const topics = await Topics.getTopicsFields(tids, ['tid', 'title', 'slug', 'deleted', 'mainPid']);
 			const topicByTid = Object.fromEntries(
 				topics.filter(t => t && t.tid && !t.deleted).map(t => [String(t.tid), t])
 			);
 
-			const results = vectorResults
+			// Build snippet map: prefer the main post body (which describes what the topic is about).
+			// Fall back to the best vector-matched post content if no main post is available.
+			const fallbackSnippetByTid = {};
+			for (const r of vectorResults) {
+				const key = String(r.topic_id);
+				if (!fallbackSnippetByTid[key]) fallbackSnippetByTid[key] = r.content;
+			}
+			const snippetByTid = { ...fallbackSnippetByTid };
+			const topicsWithMainPid = topics.filter(t => t && t.tid && !t.deleted && t.mainPid);
+			if (topicsWithMainPid.length > 0) {
+				const mainContents = await Posts.getPostsFields(
+					topicsWithMainPid.map(t => t.mainPid),
+					['pid', 'content']
+				);
+				for (let i = 0; i < topicsWithMainPid.length; i++) {
+					const content = mainContents[i] && mainContents[i].content;
+					if (content) snippetByTid[String(topicsWithMainPid[i].tid)] = content;
+				}
+			}
+
+			let results = vectorResults
 				.filter(r => topicByTid[String(r.topic_id)])
 				.map(r => {
 					const t = topicByTid[String(r.topic_id)];
 					return { tid: t.tid, title: t.title, url: `/topic/${t.slug || t.tid}` };
 				})
-				.filter((r, i, arr) => arr.findIndex(x => x.tid === r.tid) === i) // dedupe by tid
-				.slice(0, settings.maxResults);
+				.filter((r, i, arr) => arr.findIndex(x => x.tid === r.tid) === i); // dedupe by tid
+
+                console.log('Vector search results before AI re-rank:', results);
+			// AI re-ranking: pass both titles AND content snippets so GPT can judge relevance.
+			if (useAI && results.length > 0) {
+				try {
+					results = await reRankWithAI(
+						queryText, results, topicByTid,
+						settings.openaiApiKey, settings.openaiModel, settings.maxResults,
+						snippetByTid
+					);
+					winston.info(`[search-agent] AI re-ranked vector results to ${results.length} result(s).`);
+				} catch (rankErr) {
+					winston.warn(`[search-agent] AI re-rank of vector results failed, using raw order: ${rankErr.message}`);
+					results = results.slice(0, settings.maxResults);
+				}
+			} else {
+				results = results.slice(0, settings.maxResults);
+			}
 
 			if (results.length > 0) {
 				winston.info(`[search-agent] Semantic search returned ${results.length} results for "${queryText}".`);

package/library.js

@@ -91,6 +91,16 @@ plugin.addRoutes = async ({ router, middleware, helpers }) => {
 		}
 	);
 	winston.info('[search-agent] API route registered: POST /api/v3/plugins/search-agent/cache/invalidate');
+
+	// Clear all embeddings and re-index (admin only)
+	routeHelpers.setupApiRoute(
+		router,
+		'post',
+		'/search-agent/embeddings/resync',
+		[middleware.ensureLoggedIn, middleware.admin.checkPrivileges],
+		(req, res) => controllers.clearAndResync(req, res, helpers)
+	);
+	winston.info('[search-agent] API route registered: POST /api/v3/plugins/search-agent/embeddings/resync');
 };
 
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nodebb-plugin-search-agent",
-  "version": "0.0.4",
+  "version": "0.0.6",
   "description": "NodeBB plugin that adds a floating chat assistant to help users find relevant forum topics using TF-IDF text similarity",
   "main": "library.js",
   "author": "Racheli Bayfus",
@@ -37,7 +37,7 @@
   },
   "license": "MIT",
   "bugs": {
-    "url": "https://github.com/nodebb/nodebb-plugin-search-agent/issues"
+    "url": "https://github.com/racheliK9201/nodebb-plugin-search-agent/issues"
   },
   "readmeFilename": "README.md",
   "nbbpm": {

package/plugin.json

@@ -2,7 +2,7 @@
 	"id": "nodebb-plugin-search-agent",
 	"name": "Search Agent",
 	"description": "Floating chat assistant that finds relevant forum topics using TF-IDF similarity",
-	"url": "https://github.com/NodeBB/nodebb-plugin-search-agent",
+	"url": "https://github.com/racheliK9201/nodebb-plugin-search-agent",
 	"library": "./library.js",
 	"hooks": [
 		{ "hook": "static:app.load", "method": "init" },

package/public/lib/acp-main.js

@@ -33,6 +33,39 @@ $(document).ready(function () {
 			console.log('[search-agent] setupAdminPage: loading settings into form');
 			Settings.load('search-agent', $('form.search-agent-settings'));
 		});
+
+		// Re-index button
+		$('#btn-resync-embeddings').on('click', function () {
+			const $btn = $(this);
+			const $status = $('#resync-status');
+
+			if (!confirm('This will delete ALL stored embeddings and regenerate them. Continue?')) {
+				return;
+			}
+
+			$btn.prop('disabled', true).text('Working…');
+			$status.text('');
+
+			$.ajax({
+				url: config.relative_path + '/api/v3/plugins/search-agent/embeddings/resync',
+				type: 'POST',
+				headers: { 'x-csrf-token': config.csrf_token },
+				success: function (data) {
+					const msg = (data && data.response && data.response.message) || 'Re-index started.';
+					$status.html('<span class="text-success">' + msg + '</span>');
+					app.alertSuccess(msg);
+				},
+				error: function (xhr) {
+					const msg = (xhr.responseJSON && xhr.responseJSON.status && xhr.responseJSON.status.message)
+						|| 'Re-index failed.';
+					$status.html('<span class="text-danger">' + msg + '</span>');
+					app.alertError(msg);
+				},
+				complete: function () {
+					$btn.prop('disabled', false).text('Clear & Re-index');
+				},
+			});
+		});
 	}
 });
 

package/public/lib/main.js

@@ -119,63 +119,79 @@ function buildPanelHtml() {
 
 // ─── Mount ────────────────────────────────────────────────────────────────────
 
+let _mounting = false;
+
 function mountSearchAgent({ api, translator }) {
 	console.log('[search-agent] mountSearchAgent: injecting FAB and chat panel into DOM');
-	// Avoid double-mounting if somehow called twice
-	if (document.getElementById('search-agent-fab')) {
-		console.log('[search-agent] mountSearchAgent: widget already mounted, skipping');
+	// Avoid double-mounting if somehow called twice or concurrently
+	if (document.getElementById('search-agent-fab') || _mounting) {
+		console.log('[search-agent] mountSearchAgent: widget already mounted or mount in progress, skipping');
 		return;
 	}
 
+	_mounting = true;
 	const wrapper = document.createElement('div');
 	wrapper.className = 'search-agent-wrapper';
 
-	// Translate [[search-agent:key]] markers before inserting into the DOM
-	translator.translate(buildFabHtml() + buildPanelHtml(), function (translatedHtml) {
-		wrapper.innerHTML = translatedHtml;
-		document.body.appendChild(wrapper);
-
-		const fab = document.getElementById('search-agent-fab');
-		const panel = document.getElementById('search-agent-panel');
-		const closeBtn = document.getElementById('search-agent-close');
-		const input = document.getElementById('search-agent-input');
-		const sendBtn = document.getElementById('search-agent-send');
-		const messages = document.getElementById('search-agent-messages');
-
-		// ── Toggle panel ──────────────────────────────────────────────────────────
-		fab.addEventListener('click', () => {
-			const isOpen = !panel.hidden;
-			panel.hidden = isOpen;
-			fab.classList.toggle('search-agent-fab--active', !isOpen);
-			console.log(`[search-agent] FAB clicked — panel is now ${isOpen ? 'closed' : 'open'}`);
-			if (!isOpen) {
-				input.focus();
-			}
-		});
-
-		closeBtn.addEventListener('click', () => {
-			panel.hidden = true;
-			fab.classList.remove('search-agent-fab--active');
-		});
+	// Translate [[search-agent:key]] markers before inserting into the DOM.
+	// Use Promise style so a translation failure (e.g. language-file 404)
+	// still falls back to raw HTML rather than silently never mounting.
+	const rawHtml = buildFabHtml() + buildPanelHtml();
+	const translatePromise = translator.translate(rawHtml);
+	(translatePromise && typeof translatePromise.then === 'function'
+		? translatePromise
+		: Promise.resolve(rawHtml)
+	)
+		.catch(function (err) {
+			console.warn('[search-agent] mountSearchAgent: translation failed, using raw HTML', err);
+			return rawHtml;
+		})
+		.then(function (translatedHtml) {
+			wrapper.innerHTML = translatedHtml;
+			_mounting = false;
+			document.body.appendChild(wrapper);
+
+			const fab = document.getElementById('search-agent-fab');
+			const panel = document.getElementById('search-agent-panel');
+			const closeBtn = document.getElementById('search-agent-close');
+			const input = document.getElementById('search-agent-input');
+			const sendBtn = document.getElementById('search-agent-send');
+			const messages = document.getElementById('search-agent-messages');
+
+			// ── Toggle panel ──────────────────────────────────────────────────────────
+			fab.addEventListener('click', () => {
+				const isOpen = !panel.hidden;
+				panel.hidden = isOpen;
+				fab.classList.toggle('search-agent-fab--active', !isOpen);
+				console.log(`[search-agent] FAB clicked — panel is now ${isOpen ? 'closed' : 'open'}`);
+				if (!isOpen) {
+					input.focus();
+				}
+			});
 
-		// Close on Escape
-		document.addEventListener('keydown', (e) => {
-			if (e.key === 'Escape' && !panel.hidden) {
+			closeBtn.addEventListener('click', () => {
 				panel.hidden = true;
 				fab.classList.remove('search-agent-fab--active');
-			}
-		});
+			});
+
+			// Close on Escape
+			document.addEventListener('keydown', (e) => {
+				if (e.key === 'Escape' && !panel.hidden) {
+					panel.hidden = true;
+					fab.classList.remove('search-agent-fab--active');
+				}
+			});
 
-		// ── Submit on Enter or button click ───────────────────────────────────────
-		sendBtn.addEventListener('click', () => submitQuery({ api, translator, input, messages, sendBtn }));
+			// ── Submit on Enter or button click ───────────────────────────────────────
+			sendBtn.addEventListener('click', () => submitQuery({ api, translator, input, messages, sendBtn }));
 
-		input.addEventListener('keydown', (e) => {
-			if (e.key === 'Enter' && !e.shiftKey) {
-				e.preventDefault();
-				submitQuery({ api, translator, input, messages, sendBtn });
-			}
+			input.addEventListener('keydown', (e) => {
+				if (e.key === 'Enter' && !e.shiftKey) {
+					e.preventDefault();
+					submitQuery({ api, translator, input, messages, sendBtn });
+				}
+			});
 		});
-	});
 }
 
 // ─── Chat logic ───────────────────────────────────────────────────────────────

package/services/syncService.js

@@ -108,12 +108,23 @@ async function runSync() {
 				continue;
 			}
 
+			// Fetch topic titles so embeddings carry the topic context
+			const Topics = require.main.require('./src/topics');
+			const uniqueTids = [...new Set(posts.map(p => parseInt(p.tid, 10)))];
+			const topicFields = await Topics.getTopicsFields(uniqueTids, ['tid', 'title']);
+			const titleByTid = Object.fromEntries(
+				topicFields.filter(t => t && t.tid).map(t => [String(t.tid), t.title || ''])
+			);
+
 			// ------------------------------------------------------------------
-			// 3. Generate embeddings for this sub-batch
+			// 3. Generate embeddings for this sub-batch (title + content for context)
 			// ------------------------------------------------------------------
 			let vectors;
 			try {
-				vectors = await embedBatch(posts.map(p => p.content));
+				vectors = await embedBatch(posts.map((p) => {
+					const title = titleByTid[String(p.tid)] || '';
+					return title ? `${title}\n\n${p.content}` : p.content;
+				}));
 			} catch (err) {
 				winston().error(`[search-agent] syncService: failed to generate embeddings (offset ${offset}): ${err.message}`);
 				totalErrors++;

package/services/vectorSearchService.js

@@ -8,7 +8,12 @@ function winston() {
 	return require.main.require('winston');
 }
 
-const TOP_K = 10;
+// Fetch this many candidates from Orama — cast a wide net so the AI has enough to choose from
+const TOP_K = 20;
+// Absolute minimum cosine similarity — only filters pure noise (near-zero similarity).
+// Do NOT raise this: the relevant result often scores lower than irrelevant ones.
+// The AI re-ranker (which reads content) is the precision gate, not this floor.
+const MIN_SCORE = 0.10;
 // Rebuild the Orama index after this interval (mirrors TF-IDF cache TTL)
 const INDEX_TTL_MS = 5 * 60 * 1000;
 
@@ -84,7 +89,7 @@ function invalidateIndex() {
  * @returns {Promise<Array<{ topic_id: number, post_id: number, content: string, score: number }>>}
  *   Top results sorted by cosine similarity descending.
  */
-async function search(query) {
+async function search(query, limit = TOP_K) {
 	if (typeof query !== 'string' || query.trim() === '') {
 		throw new Error('search() requires a non-empty query string');
 	}
@@ -99,13 +104,19 @@ async function search(query) {
 	const results = await oramaSearch(db, {
 		mode: 'vector',
 		vector: { value: queryEmbedding, property: 'embedding' },
-		limit: TOP_K,
+		limit,
+		similarity: 0.1, 
 		includeVectors: false,
 	});
 
 	winston().verbose(`[search-agent] vectorSearchService: Orama returned ${results.hits.length} hit(s)`);
 
-	return results.hits.map(hit => ({
+	const filtered = results.hits.filter(hit => hit.score >= MIN_SCORE);
+	winston().verbose(
+		`[search-agent] vectorSearchService: ${filtered.length}/${results.hits.length} hit(s) passed noise floor (MIN_SCORE=${MIN_SCORE})`
+	);
+
+	return filtered.map(hit => ({
 		topic_id: hit.document.topic_id,
 		post_id: hit.document.post_id,
 		content: hit.document.content,

package/services/vectorStore.js

@@ -134,4 +134,21 @@ async function getMissingEmbeddings(postIds) {
 	return missing;
 }
 
-module.exports = { saveEmbedding, getAllEmbeddings, findByPostId, getMissingEmbeddings };
+/**
+ * Delete every stored embedding and reset the in-memory cache.
+ * Call this before a full re-index.
+ *
+ * @returns {Promise<number>} Number of documents deleted
+ */
+async function clearAllEmbeddings() {
+	winston().info('[search-agent] vectorStore: clearing ALL embeddings from database…');
+	await ensureIndexes();
+	const col = getCollection();
+	const result = await col.deleteMany({});
+	_cache = [];
+	_cachePromise = null;
+	winston().info(`[search-agent] vectorStore: deleted ${result.deletedCount} embedding(s)`);
+	return result.deletedCount;
+}
+
+module.exports = { saveEmbedding, getAllEmbeddings, findByPostId, getMissingEmbeddings, clearAllEmbeddings };

package/templates/admin/plugins/search-agent.tpl

@@ -181,6 +181,23 @@
 					</button>
 				</div>
 			</div>
+
+			<div class="card mt-3">
+				<div class="card-header">Re-index Embeddings</div>
+				<div class="card-body">
+					<p class="card-text small">
+						Delete all stored embeddings and re-generate them with the
+						current strategy (title&nbsp;+ post content).
+						The re-index runs in the background — search continues working
+						via TF-IDF until it finishes.
+					</p>
+					<button id="btn-resync-embeddings" class="btn btn-danger btn-sm fw-semibold">
+						Clear &amp; Re-index
+					</button>
+					<div id="resync-status" class="mt-2 small"></div>
+				</div>
+			</div>
+		</div>
 		</div>
 	</div>
 </div>