nodebb-plugin-dbsearch 6.3.2 → 6.3.4

package/lib/dbsearch.js

@@ -247,9 +247,9 @@ search.filterMessagingSearchMessages = async function (data) {
 
 search.reindex = async function () {
 	await db.setObject('nodebb-plugin-dbsearch', {
-		topicsIndexed: 0,
-		postsIndexed: 0,
-		messagesIndexed: 0,
+		topicsProgress: 0,
+		postsProgress: 0,
+		messagesProgress: 0,
 		working: 1,
 	});
 	await Promise.all([
@@ -264,8 +264,11 @@ search.reindex = async function () {
 
 async function reIndexTopics() {
 	await batch.processSortedSet('topics:tid', async (tids) => {
-		const topicData = await topics.getTopicsFields(tids, ['tid', 'title', 'uid', 'cid', 'deleted', 'timestamp']);
+		const topicData = await topics.getTopicsFields(tids, [
+			'tid', 'title', 'uid', 'cid', 'deleted', 'timestamp',
+		]);
 		await topicsSave(topicData);
+		await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'topicsProgress', tids.length);
 	}, {
 		batch: batchSize,
 	});
@@ -274,6 +277,7 @@ async function reIndexTopics() {
 async function topicsSave(topics) {
 	topics = topics.filter(
 		t => t && utils.isNumber(t.tid) && parseInt(t.deleted, 10) !== 1 &&
+		utils.isNumber(t.cid) &&
 		!pluginConfig.excludeCategories.includes(String(t.cid))
 	);
 
@@ -305,7 +309,6 @@ async function topicsSave(topics) {
 
 	const result = await plugins.hooks.fire('filter:search.indexTopics', { data: data, tids: tids, topics: topics });
 	await db.searchIndex('topic', result.data, result.tids);
-	await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'topicsIndexed', result.tids.length);
 }
 
 async function reIndexPosts() {
@@ -322,6 +325,7 @@ async function reIndexPosts() {
 		});
 		postData = postData.filter(post => tidToTopic[post.tid].deleted !== 1);
 		await postsSave(postData);
+		await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'postsProgress', pids.length);
 	}, {
 		batch: batchSize,
 	});
@@ -330,6 +334,7 @@ async function reIndexPosts() {
 async function postsSave(posts) {
 	posts = posts.filter(
 		p => p && utils.isNumber(p.pid) && parseInt(p.deleted, 10) !== 1 &&
+		utils.isNumber(p.cid) &&
 		!pluginConfig.excludeCategories.includes(String(p.cid))
 	);
 
@@ -361,7 +366,6 @@ async function postsSave(posts) {
 
 	const result = await plugins.hooks.fire('filter:search.indexPosts', { data: data, pids: pids, posts: posts });
 	await db.searchIndex('post', result.data, result.pids);
-	await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'postsIndexed', result.pids.length);
 }
 
 async function reIndexMessages() {
@@ -369,13 +373,16 @@ async function reIndexMessages() {
 		let messageData = await messaging.getMessagesFields(mids, ['mid', 'content', 'roomId', 'fromuid', 'deleted', 'system']);
 		messageData = messageData.filter(p => p && p.deleted !== 1 && p.system !== 1);
 		await messagesSave(messageData);
+		await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'messagesProgress', mids.length);
 	}, {
 		batch: batchSize,
 	});
 }
 
 async function messagesSave(msgs) {
-	msgs = msgs.filter(m => m && utils.isNumber(m.mid) && parseInt(m.deleted, 10) !== 1 && parseInt(m.system, 10) !== 1);
+	msgs = msgs.filter(
+		m => m && utils.isNumber(m.mid) && parseInt(m.deleted, 10) !== 1 && parseInt(m.system, 10) !== 1
+	);
 
 	let data = msgs.map((msgData) => {
 		const indexData = {};
@@ -402,19 +409,11 @@ async function messagesSave(msgs) {
 
 	const result = await plugins.hooks.fire('filter:search.indexMessages', { data: data, mids: mids, messages: msgs });
 	await searchModule.chat.index(result.data, result.mids);
-	await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'messagesIndexed', result.mids.length);
 }
 
 async function searchRemove(key, ids) {
 	ids = ids.filter(id => id && utils.isNumber(id));
 	await db.searchRemove(key, ids);
-	if (key === 'topic') {
-		await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'topicsIndexed', -ids.length);
-	} else if (key === 'post') {
-		await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'postsIndexed', -ids.length);
-	} else if (key === 'chat') {
-		await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'messagesIndexed', -ids.length);
-	}
 }
 
 async function reIndexTids(tids) {
@@ -466,6 +465,14 @@ async function reIndexPids(pids, topic) {
 
 async function renderAdmin(req, res) {
 	const results = await getGlobalAndPluginData();
+	const [topicsIndexed, postsIndexed, messagesIndexed] = await Promise.all([
+		searchModule.getIndexedTopicCount(),
+		searchModule.getIndexedPostCount(),
+		searchModule.getIndexedChatMessageCount(),
+	]);
+	results.plugin.topicsIndexed = parseInt(topicsIndexed, 10) || 0;
+	results.plugin.postsIndexed = parseInt(postsIndexed, 10) || 0;
+	results.plugin.messagesIndexed = parseInt(messagesIndexed, 10) || 0;
 	results.plugin.progressData = await getProgress();
 	results.plugin.title = 'DB Search';
 	res.render('admin/plugins/dbsearch', results.plugin);
@@ -496,14 +503,12 @@ socketAdmin.plugins.dbsearch.checkProgress = async function () {
 
 async function getPluginData() {
 	const data = await db.getObject('nodebb-plugin-dbsearch') || {};
-	data.topicsIndexed = parseInt(data.topicsIndexed, 10) || 0;
-	data.postsIndexed = parseInt(data.postsIndexed, 10) || 0;
-	data.messagesIndexed = parseInt(data.messagesIndexed, 10) || 0;
+
 	data.excludeCategories = data.excludeCategories || '[]';
 	data.postLimit = data.postLimit || defaultPostLimit;
 	data.topicLimit = data.topicLimit || defaultTopicLimit;
 	data.indexLanguage = data.indexLanguage || 'en';
-	data.working = data.working || 0;
+	data.working = parseInt(data.working, 10) || 0;
 
 	try {
 		data.excludeCategories = JSON.parse(data.excludeCategories);
@@ -535,9 +540,6 @@ async function getGlobalAndPluginData() {
 	plugin.messageCount = parseInt(global.messageCount, 10);
 	plugin.topicLimit = plugin.topicLimit || defaultTopicLimit;
 	plugin.postLimit = plugin.postLimit || defaultPostLimit;
-	plugin.topicsIndexed = plugin.topicsIndexed > plugin.topicCount ? plugin.topicCount : plugin.topicsIndexed;
-	plugin.postsIndexed = plugin.postsIndexed > plugin.postCount ? plugin.postCount : plugin.postsIndexed;
-	plugin.messagesIndexed = plugin.messagesIndexed > plugin.messageCount ? plugin.messageCount : plugin.messagesIndexed;
 	plugin.languageSupported = languageSupported;
 	plugin.languages = languages;
 	plugin.indexLanguage = plugin.indexLanguage || 'en';
@@ -555,21 +557,18 @@ async function getGlobalAndPluginData() {
 async function getProgress() {
 	const [global, pluginData] = await Promise.all([
 		db.getObjectFields('global', ['topicCount', 'postCount', 'messageCount']),
-		getPluginData(),
+		db.getObjectFields('nodebb-plugin-dbsearch', ['topicsProgress', 'postsProgress', 'messagesProgress', 'working']),
 	]);
 	const topicCount = parseInt(global.topicCount, 10);
 	const postCount = parseInt(global.postCount, 10);
 	const messageCount = parseInt(global.messageCount, 10);
-	const topicsPercent = topicCount ? (pluginData.topicsIndexed / topicCount) * 100 : 0;
-	const postsPercent = postCount ? (pluginData.postsIndexed / postCount) * 100 : 0;
-	const messagesPercent = messageCount ? (pluginData.messagesIndexed / messageCount) * 100 : 0;
+	const topicsPercent = topicCount ? (pluginData.topicsProgress / topicCount) * 100 : 0;
+	const postsPercent = postCount ? (pluginData.postsProgress / postCount) * 100 : 0;
+	const messagesPercent = messageCount ? (pluginData.messagesProgress / messageCount) * 100 : 0;
 	return {
 		topicsPercent: Math.max(0, Math.min(100, topicsPercent.toFixed(2))),
 		postsPercent: Math.max(0, Math.min(100, postsPercent.toFixed(2))),
 		messagesPercent: Math.max(0, Math.min(100, messagesPercent.toFixed(2))),
-		topicsIndexed: topicsPercent >= 100 ? topicCount : Math.max(0, pluginData.topicsIndexed),
-		postsIndexed: postsPercent >= 100 ? postCount : Math.max(0, pluginData.postsIndexed),
-		messagesIndexed: messagesPercent >= 100 ? messageCount : Math.max(0, pluginData.messagesIndexed),
 		working: pluginData.working,
 	};
 }
@@ -597,26 +596,27 @@ socketAdmin.plugins.dbsearch.clearIndex = async function () {
 
 async function clearIndex() {
 	await db.setObject('nodebb-plugin-dbsearch', {
+		postsProgress: 0,
+		topicsProgress: 0,
+		messagesProgress: 0,
 		working: 1,
 	});
 
 	await Promise.all([
-		clearSet('topics:tid', 'topic'),
-		clearSet('posts:pid', 'post'),
-		clearSet('messages:mid', 'chat'),
+		clearSet('topics:tid', 'topic', 'topicsProgress'),
+		clearSet('posts:pid', 'post', 'postsProgress'),
+		clearSet('messages:mid', 'chat', 'messagesProgress'),
 	]);
 
 	await db.setObject('nodebb-plugin-dbsearch', {
-		postsIndexed: 0,
-		topicsIndexed: 0,
-		messagesIndexed: 0,
 		working: 0,
 	});
 }
 
-async function clearSet(set, key) {
+async function clearSet(set, key, progressKey) {
 	await batch.processSortedSet(set, async (ids) => {
 		await searchRemove(key, ids);
+		await db.incrObjectFieldBy('nodebb-plugin-dbsearch', progressKey, ids.length);
 	}, {
 		batch: batchSize,
 	});

package/lib/mongo.js

@@ -226,3 +226,14 @@ function buildTextQuery(content, matchWords) {
 	return { $search: words.join(' ') };
 }
 
+exports.getIndexedTopicCount = async () => {
+	return await db.client.collection('searchtopic').estimatedDocumentCount();
+};
+
+exports.getIndexedPostCount = async () => {
+	return await db.client.collection('searchpost').estimatedDocumentCount();
+};
+
+exports.getIndexedChatMessageCount = async () => {
+	return await db.client.collection('searchchat').estimatedDocumentCount();
+};

package/lib/postgres.js

@@ -164,3 +164,19 @@ exports.chat.search = async (data, limit) => {
 		return [];
 	}
 };
+
+
+exports.getIndexedTopicCount = async () => {
+	const res = await db.pool.query(`SELECT reltuples::bigint AS estimate FROM pg_class WHERE relname = 'searchtopic'`);
+	return res.rows && res.rows[0] ? parseInt(res.rows[0].estimate, 10) : 0;
+};
+
+exports.getIndexedPostCount = async () => {
+	const res = await db.pool.query(`SELECT reltuples::bigint AS estimate FROM pg_class WHERE relname = 'searchpost'`);
+	return res.rows && res.rows[0] ? parseInt(res.rows[0].estimate, 10) : 0;
+};
+
+exports.getIndexedChatMessageCount = async () => {
+	const res = await db.pool.query(`SELECT reltuples::bigint AS estimate FROM pg_class WHERE relname = 'searchchat'`);
+	return res.rows && res.rows[0] ? parseInt(res.rows[0].estimate, 10) : 0;
+};

package/lib/redis.js

@@ -86,3 +86,15 @@ exports.chat.search = async (data, limit) => {
 
 	return await db.chatSearch.query(query, 0, limit - 1);
 };
+
+exports.getIndexedTopicCount = async () => {
+	return await db.topicSearch.count('nodebbtopicsearch');
+};
+
+exports.getIndexedPostCount = async () => {
+	return await db.postSearch.count('nodebbpostsearch');
+};
+
+exports.getIndexedChatMessageCount = async () => {
+	return await db.chatSearch.count('nodebbchatsearch');
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nodebb-plugin-dbsearch",
-  "version": "6.3.2",
+  "version": "6.3.4",
   "description": "A Plugin that lets users search posts and topics",
   "main": "index.js",
   "repository": {
@@ -20,7 +20,7 @@
   "license": "BSD-2-Clause",
   "dependencies": {
     "lodash": "4.17.21",
-    "redisearch": "^2.0.0"
+    "redisearch": "^2.0.1"
   },
   "devDependencies": {
     "eslint": "^9.25.1",

package/public/admin.js

@@ -78,6 +78,11 @@ define('admin/plugins/dbsearch', [
 	};
 
 	function startProgress() {
+		$('#topics-indexed').text(0);
+		$('#posts-indexed').text(0);
+		$('#messages-indexed').text(0);
+		$('.progress-bar').css('width', '0%').text('0%');
+		$('.progress').removeClass('invisible');
 		clearProgress();
 		intervalId = setInterval(checkProgress, 1000);
 	}
@@ -85,6 +90,7 @@ define('admin/plugins/dbsearch', [
 	function clearProgress() {
 		if (intervalId) {
 			clearInterval(intervalId);
+			$('.progress').addClass('invisible');
 			intervalId = 0;
 		}
 	}
@@ -96,27 +102,31 @@ define('admin/plugins/dbsearch', [
 				return alerts.error(err);
 			}
 
+			$('.topic-progress').css('width', progress.topicsPercent + '%').text(progress.topicsPercent + '%');
+			$('.post-progress').css('width', progress.postsPercent + '%').text(progress.postsPercent + '%');
+			$('.message-progress').css('width', progress.messagesPercent + '%').text(progress.messagesPercent + '%');
+
 			var working = parseInt(progress.working, 10);
 			if (!working) {
 				clearInterval(intervalId);
 				$('#reindex').removeAttr('disabled');
+				$('#clear-index').removeAttr('disabled');
+				setTimeout(() => {
+					$('.progress').addClass('invisible');
+				}, 2000);
+
+				$.getJSON(config.relative_path + '/api/admin/plugins/dbsearch', function (data) {
+					$('#topics-indexed').text(data.topicsIndexed);
+					$('#posts-indexed').text(data.postsIndexed);
+					$('#messages-indexed').text(data.messagesIndexed);
+				});
 			} else {
 				$('#reindex').attr('disabled', true);
+				$('#clear-index').attr('disabled', true);
+				$('.progress').removeClass('invisible');
 			}
 
 			$('#work-in-progress').toggleClass('hidden', !working);
-
-			if (progress.topicsPercent >= 100 && progress.postsPercent >= 100) {
-				progress.topicsPercent = 100;
-				progress.postsPercent = 100;
-			}
-
-			$('#topics-indexed').text(progress.topicsIndexed);
-			$('#posts-indexed').text(progress.postsIndexed);
-			$('#messages-indexed').text(progress.messagesIndexed);
-			$('.topic-progress').css('width', progress.topicsPercent + '%').text(progress.topicsPercent + '%');
-			$('.post-progress').css('width', progress.postsPercent + '%').text(progress.postsPercent + '%');
-			$('.message-progress').css('width', progress.messagesPercent + '%').text(progress.messagesPercent + '%');
 		});
 	}
 

package/templates/admin/plugins/dbsearch.tpl

@@ -9,27 +9,27 @@
 					<div class="col-6">
 						<div class="mb-3">
 							<div class="alert alert-info">
-								Topics Indexed: <strong id="topics-indexed">{topicsIndexed}</strong> / <strong>{topicCount}</strong>
+								Topics Indexed: <strong id="topics-indexed">{topicsIndexed}</strong> <i class="fa fa-circle-question" title="Deleted topics are not indexed" data-bs-toggle="tooltip"></i>
 							</div>
 
-							<div class="progress" style="height:24px;">
+							<div class="progress {{{ if !working }}}invisible{{{ end }}}" style="height:24px;">
 								<div class="topic-progress progress-bar" role="progressbar" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100" style="width:{progressData.topicsPercent}%;min-width: 2em;">{progressData.topicsPercent}%</div>
 							</div>
 						</div>
 						<div class="mb-3">
 							<div class="alert alert-info">
-								Posts Indexed: <strong id="posts-indexed">{postsIndexed}</strong> / <strong>{postCount}</strong>
+								Posts Indexed: <strong id="posts-indexed">{postsIndexed}</strong> <i class="fa fa-circle-question" title="Deleted posts are not indexed" data-bs-toggle="tooltip"></i>
 							</div>
-							<div class="progress" style="height:24px;">
+							<div class="progress {{{ if !working }}}invisible{{{ end }}}" style="height:24px;">
 								<div class="post-progress progress-bar" role="progressbar" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100" style="width:{progressData.postsPercent}%;min-width: 2em;">{progressData.postsPercent}%</div>
 							</div>
 						</div>
 
 						<div class="mb-3">
 							<div class="alert alert-info">
-								Messages Indexed: <strong id="messages-indexed">{messagesIndexed}</strong> / <strong>{messageCount}</strong>
+								Messages Indexed: <strong id="messages-indexed">{messagesIndexed}</strong> <i class="fa fa-circle-question" title="Deleted messages are not indexed" data-bs-toggle="tooltip"></i>
 							</div>
-							<div class="progress" style="height:24px;">
+							<div class="progress {{{ if !working }}}invisible{{{ end }}}" style="height:24px;">
 								<div class="message-progress progress-bar" role="progressbar" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100" style="width:{progressData.messagesPercent}%;min-width: 2em;">{progressData.messagesPercent}%</div>
 							</div>
 						</div>