nodebb-plugin-dbsearch 6.1.0 → 6.2.0

package/lib/dbsearch.js

@@ -8,6 +8,7 @@ const nconf = require.main.require('nconf');
 const db = require.main.require('./src/database');
 const topics = require.main.require('./src/topics');
 const posts = require.main.require('./src/posts');
+const messaging = require.main.require('./src/messaging');
 const utils = require.main.require('./src/utils');
 const socketAdmin = require.main.require('./src/socket.io/admin');
 const batch = require.main.require('./src/batch');
@@ -156,6 +157,22 @@ search.actionTopicChangeOwner = function (hookData) {
 	topicsSave(hookData.topics);
 };
 
+search.actionMessagingSave = async function (hookData) {
+	await messagesSave([hookData.message]);
+};
+
+search.actionMessagingDelete = async function (hookData) {
+	await searchRemove('chat', [hookData.message.mid]);
+};
+
+search.actionMessagingRestore = async function (hookData) {
+	await messagesSave([hookData.message]);
+};
+
+search.actionMessagingEdit = async function (hookData) {
+	await messagesSave([hookData.message]);
+};
+
 search.filterSearchQuery = async function (data) {
 	if (!data || !data.index) {
 		return data;
@@ -199,15 +216,43 @@ search.filterSearchTopic = async function (hookData) {
 	return hookData;
 };
 
+search.filterMessagingSearchMessages = async function (data) {
+	if (!data || !data.content) {
+		return data;
+	}
+	const limit = 100;
+	const query = {};
+	if (data.hasOwnProperty('roomId') && data.roomId) {
+		query.roomId = data.roomId;
+	}
+	if (data.hasOwnProperty('uid') && data.uid) {
+		query.uid = data.uid;
+	}
+	if (data.hasOwnProperty('content') && data.content) {
+		query.content = data.content;
+	}
+	if (!Object.keys(query).length) {
+		return [];
+	}
+	if (data.hasOwnProperty('matchWords')) {
+		query.matchWords = data.matchWords;
+	}
+
+	data.ids = data.ids.concat(await searchModule.chat.search(query, limit));
+	return data;
+};
+
 search.reindex = async function () {
 	await db.setObject('nodebb-plugin-dbsearch', {
 		topicsIndexed: 0,
 		postsIndexed: 0,
+		messagesIndexed: 0,
 		working: 1,
 	});
 	await Promise.all([
 		reIndexTopics(),
 		reIndexPosts(),
+		reIndexMessages(),
 	]);
 	await db.setObject('nodebb-plugin-dbsearch', {
 		working: 0,
@@ -308,12 +353,55 @@ async function postsSave(posts) {
 	await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'postsIndexed', result.pids.length);
 }
 
+async function reIndexMessages() {
+	await batch.processSortedSet(`messages:mid`, async (mids) => {
+		let messageData = await messaging.getMessagesFields(mids, ['mid', 'content', 'roomId', 'fromuid', 'deleted', 'system']);
+		messageData = messageData.filter(p => p && p.deleted !== 1 && p.system !== 1);
+		await messagesSave(messageData);
+	}, {
+		batch: batchSize,
+	});
+}
+
+async function messagesSave(msgs) {
+	msgs = msgs.filter(m => m && m.mid && parseInt(m.deleted, 10) !== 1 && parseInt(m.system, 10) !== 1);
+
+	let data = msgs.map((msgData) => {
+		const indexData = {};
+		if (msgData.content) {
+			indexData.content = msgData.content;
+		}
+		if (msgData.roomId) {
+			indexData.roomId = msgData.roomId;
+		}
+		if (msgData.fromuid) {
+			indexData.uid = msgData.fromuid;
+		}
+		if (!Object.keys(indexData).length) {
+			return null;
+		}
+		return indexData;
+	});
+
+	const mids = msgs.filter((msg, index) => !!data[index]).map(msg => msg.mid);
+	data = data.filter(Boolean);
+	if (!data.length) {
+		return;
+	}
+
+	const result = await plugins.hooks.fire('filter:search.indexMessages', { data: data, mids: mids, messages: msgs });
+	await searchModule.chat.index(result.data, result.mids);
+	await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'messagesIndexed', result.mids.length);
+}
+
 async function searchRemove(key, ids) {
 	await db.searchRemove(key, ids);
 	if (key === 'topic') {
 		await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'topicsIndexed', -ids.length);
 	} else if (key === 'post') {
 		await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'postsIndexed', -ids.length);
+	} else if (key === 'chat') {
+		await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'messagesIndexed', -ids.length);
 	}
 }
 
@@ -398,6 +486,7 @@ async function getPluginData() {
 	const data = await db.getObject('nodebb-plugin-dbsearch') || {};
 	data.topicsIndexed = parseInt(data.topicsIndexed, 10) || 0;
 	data.postsIndexed = parseInt(data.postsIndexed, 10) || 0;
+	data.messagesIndexed = parseInt(data.messagesIndexed, 10) || 0;
 	data.excludeCategories = data.excludeCategories || '[]';
 	data.postLimit = data.postLimit || defaultPostLimit;
 	data.topicLimit = data.topicLimit || defaultTopicLimit;
@@ -415,7 +504,7 @@ async function getPluginData() {
 
 async function getGlobalAndPluginData() {
 	const [global, plugin, allCategories] = await Promise.all([
-		db.getObjectFields('global', ['topicCount', 'postCount']),
+		db.getObjectFields('global', ['topicCount', 'postCount', 'messageCount']),
 		getPluginData(),
 		categories.buildForSelectAll(['value', 'text']),
 	]);
@@ -431,10 +520,12 @@ async function getGlobalAndPluginData() {
 	plugin.allCategories = allCategories;
 	plugin.topicCount = parseInt(global.topicCount, 10);
 	plugin.postCount = parseInt(global.postCount, 10);
+	plugin.messageCount = parseInt(global.messageCount, 10);
 	plugin.topicLimit = plugin.topicLimit || defaultTopicLimit;
 	plugin.postLimit = plugin.postLimit || defaultPostLimit;
 	plugin.topicsIndexed = plugin.topicsIndexed > plugin.topicCount ? plugin.topicCount : plugin.topicsIndexed;
 	plugin.postsIndexed = plugin.postsIndexed > plugin.postCount ? plugin.postCount : plugin.postsIndexed;
+	plugin.messagesIndexed = plugin.messagesIndexed > plugin.messageCount ? plugin.messageCount : plugin.messagesIndexed;
 	plugin.languageSupported = languageSupported;
 	plugin.languages = languages;
 	plugin.indexLanguage = plugin.indexLanguage || 'en';
@@ -451,35 +542,40 @@ async function getGlobalAndPluginData() {
 
 async function getProgress() {
 	const [global, pluginData] = await Promise.all([
-		db.getObjectFields('global', ['topicCount', 'postCount']),
+		db.getObjectFields('global', ['topicCount', 'postCount', 'messageCount']),
 		getPluginData(),
 	]);
 	const topicCount = parseInt(global.topicCount, 10);
 	const postCount = parseInt(global.postCount, 10);
+	const messageCount = parseInt(global.messageCount, 10);
 	const topicsPercent = topicCount ? (pluginData.topicsIndexed / topicCount) * 100 : 0;
 	const postsPercent = postCount ? (pluginData.postsIndexed / postCount) * 100 : 0;
+	const messagesPercent = messageCount ? (pluginData.messagesIndexed / messageCount) * 100 : 0;
 	return {
 		topicsPercent: Math.max(0, Math.min(100, topicsPercent.toFixed(2))),
 		postsPercent: Math.max(0, Math.min(100, postsPercent.toFixed(2))),
+		messagesPercent: Math.max(0, Math.min(100, messagesPercent.toFixed(2))),
 		topicsIndexed: topicsPercent >= 100 ? topicCount : Math.max(0, pluginData.topicsIndexed),
 		postsIndexed: postsPercent >= 100 ? postCount : Math.max(0, pluginData.postsIndexed),
+		messagesIndexed: messagesPercent >= 100 ? messageCount : Math.max(0, pluginData.messagesIndexed),
 		working: pluginData.working,
 	};
 }
 
 socketAdmin.plugins.dbsearch.reindex = function (socket, data, callback) {
-	try {
-		search.reindex();
-	} catch (err) {
-		winston.error(err);
-	}
+	setTimeout(async () => {
+		try {
+			await search.reindex();
+		} catch (err) {
+			winston.error(err);
+		}
+	}, 0);
 	callback();
 };
 
 socketAdmin.plugins.dbsearch.clearIndex = async function () {
 	setTimeout(async () => {
 		try {
-			console.log('clear called');
 			await clearIndex();
 		} catch (err) {
 			winston.error(err.stack);
@@ -495,11 +591,13 @@ async function clearIndex() {
 	await Promise.all([
 		clearSet('topics:tid', 'topic'),
 		clearSet('posts:pid', 'post'),
+		clearSet('messages:mid', 'chat'),
 	]);
 
 	await db.setObject('nodebb-plugin-dbsearch', {
 		postsIndexed: 0,
 		topicsIndexed: 0,
+		messagesIndexed: 0,
 		working: 0,
 	});
 }

package/lib/mongo.js

@@ -13,6 +13,7 @@ exports.createIndices = async function (language) {
 	if (nconf.get('isPrimary') && !nconf.get('jobsDisabled')) {
 		await db.client.collection('searchtopic').createIndex({ content: 'text', uid: 1, cid: 1 }, options);
 		await db.client.collection('searchpost').createIndex({ content: 'text', uid: 1, cid: 1 }, options);
+		await db.client.collection('searchchat').createIndex({ content: 'text', roomId: 1, uid: 1 }, options);
 	}
 };
 
@@ -26,6 +27,8 @@ exports.changeIndexLanguage = async function (language) {
 	await db.client.collection('searchtopic').createIndex(indexSpec, options);
 	await db.client.collection('searchpost').dropIndex('content_text_uid_1_cid_1');
 	await db.client.collection('searchpost').createIndex(indexSpec, options);
+	await db.client.collection('searchchat').dropIndex('content_text_roomId_1_uid_1');
+	await db.client.collection('searchchat').createIndex({ content: 'text', roomId: 1, uid: 1 }, options);
 };
 
 exports.searchIndex = async function (key, data, ids) {
@@ -53,18 +56,7 @@ exports.searchIndex = async function (key, data, ids) {
 exports.search = async function (key, data, limit) {
 	const searchQuery = {};
 	if (data.content) {
-		let words = data.content.split(' ');
-		const allQuoted = data.content.startsWith('"') && data.content.endsWith('"');
-		if (data.matchWords === 'all' && !allQuoted) {
-			words = words.map((word) => {
-				if (!word.startsWith('"') && !word.endsWith('"')) {
-					return `"${word}"`;
-				}
-				return word;
-			});
-		}
-
-		searchQuery.$text = { $search: words.join(' ') };
+		searchQuery.$text = buildTextQuery(data.content, data.matchWords);
 	}
 
 	if (Array.isArray(data.cid) && data.cid.length) {
@@ -112,3 +104,80 @@ exports.searchRemove = async function (key, ids) {
 
 	await db.client.collection(`search${key}`).deleteMany({ _id: { $in: ids } });
 };
+
+exports.chat = {};
+exports.chat.index = async (data, ids) => {
+	if (!ids.length) {
+		return;
+	}
+
+	ids = ids.map(id => parseInt(id, 10));
+
+	const bulk = db.client.collection(`searchchat`).initializeUnorderedBulkOp();
+	ids.forEach((id, index) => {
+		const d = data[index];
+		if (d && d.content && d.uid && d.roomId) {
+			bulk.find({ _id: id }).upsert().updateOne({
+				$set: {
+					content: String(d.content),
+					roomId: String(d.roomId),
+					uid: String(d.uid),
+				},
+			});
+		}
+	});
+
+	await bulk.execute();
+};
+
+exports.chat.search = async (data, limit) => {
+	const searchQuery = {};
+	if (!data.content) {
+		return [];
+	}
+	searchQuery.$text = buildTextQuery(data.content, data.matchWords);
+
+	if (Array.isArray(data.roomId) && data.roomId.filter(Boolean).length) {
+		if (data.roomId.length > 1) {
+			searchQuery.roomId = { $in: data.roomId.filter(Boolean).map(String) };
+		} else {
+			searchQuery.roomId = String(data.roomId[0]);
+		}
+	}
+
+	if (Array.isArray(data.uid) && data.uid.filter(Boolean).length) {
+		if (data.uid.length > 1) {
+			searchQuery.uid = { $in: data.uid.filter(Boolean).map(String) };
+		} else {
+			searchQuery.uid = String(data.uid[0]);
+		}
+	}
+
+	const collection = db.client.collection(`searchchat`);
+	const results = await collection.aggregate([
+		{ $match: searchQuery },
+		{ $sort: { score: { $meta: 'textScore' } } },
+		{ $limit: parseInt(limit, 10) },
+		{ $project: { _id: 1 } },
+	]).toArray();
+	if (!results || !results.length) {
+		return [];
+	}
+	return results.map(item => item._id);
+};
+
+function buildTextQuery(content, matchWords) {
+	let words = content.split(' ');
+	const allQuoted = content.startsWith('"') && content.endsWith('"');
+	if (matchWords === 'all' && !allQuoted) {
+		words = words.map((word) => {
+			if (!word.startsWith('"') && !word.endsWith('"')) {
+				return `"${word}"`;
+			}
+			return word;
+		});
+	}
+
+	return { $search: words.join(' ') };
+}
+

package/lib/postgres.js

@@ -17,10 +17,16 @@ async function initDB() {
 	await db.pool.query(`CREATE INDEX IF NOT EXISTS "idx__searchtopic__content" ON "searchtopic" USING GIN (to_tsvector('${searchLanguage}', "content"))`);
 	await db.pool.query('CREATE INDEX IF NOT EXISTS "idx__searchtopic__uid" ON "searchtopic"("uid")');
 	await db.pool.query('CREATE INDEX IF NOT EXISTS "idx__searchtopic__cid" ON "searchtopic"("cid")');
+
 	await db.pool.query('CREATE TABLE IF NOT EXISTS "searchpost" ( "id" BIGINT NOT NULL PRIMARY KEY, "content" TEXT, "uid" BIGINT, "cid" BIGINT )');
 	await db.pool.query(`CREATE INDEX IF NOT EXISTS "idx__searchpost__content" ON "searchpost" USING GIN (to_tsvector('${searchLanguage}', "content"))`);
 	await db.pool.query('CREATE INDEX IF NOT EXISTS "idx__searchpost__uid" ON "searchpost"("uid")');
 	await db.pool.query('CREATE INDEX IF NOT EXISTS "idx__searchpost__cid" ON "searchpost"("cid")');
+
+	await db.pool.query('CREATE TABLE IF NOT EXISTS "searchchat" ( "id" BIGINT NOT NULL PRIMARY KEY, "content" TEXT, "rid" BIGINT, "uid" BIGINT, )');
+	await db.pool.query(`CREATE INDEX IF NOT EXISTS "idx__searchchat__content" ON "searchchat" USING GIN (to_tsvector('${searchLanguage}', "content"))`);
+	await db.pool.query('CREATE INDEX IF NOT EXISTS "idx__searchchat__rid" ON "searchchat"("rid")');
+	await db.pool.query('CREATE INDEX IF NOT EXISTS "idx__searchchat__uid" ON "searchchat"("uid")');
 }
 
 async function handleError(err) {
@@ -45,8 +51,12 @@ exports.changeIndexLanguage = async function (language) {
 	pubsub.publish('dbsearch-language-changed', language);
 	await db.pool.query('DROP INDEX "idx__searchtopic__content"');
 	await db.pool.query(`CREATE INDEX "idx__searchtopic__content" ON "searchtopic" USING GIN (to_tsvector('${language}', "content"))`);
+
 	await db.pool.query('DROP INDEX "idx__searchpost__content"');
 	await db.pool.query(`CREATE INDEX "idx__searchpost__content" ON "searchpost" USING GIN (to_tsvector('${language}', "content"))`);
+
+	await db.pool.query('DROP INDEX "idx__searchchat__content"');
+	await db.pool.query(`CREATE INDEX "idx__searchchat__content" ON "searchchat" USING GIN (to_tsvector('${language}', "content"))`);
 };
 
 exports.searchIndex = async function (key, data, ids) {
@@ -109,3 +119,48 @@ exports.searchRemove = async function (key, ids) {
 		await handleError(err);
 	}
 };
+
+exports.chat = {};
+exports.chat.index = async (data, ids) => {
+	if (!ids.length) {
+		return;
+	}
+
+	ids = ids.map(id => parseInt(id, 10));
+	try {
+		await db.pool.query({
+			name: `dbsearch-searchIndex-chat`,
+			text: `INSERT INTO "searchchat" SELECT d."id", d."data"->>'content' "content", (d."data"->>'uid')::bigint "uid", (d."data"->>'roomId')::bigint "roomId" FROM UNNEST($1::bigint[], $2::jsonb[]) d("id", "data") ON CONFLICT ("id") DO UPDATE SET "content" = COALESCE(EXCLUDED."content", "searchchat"."content"), "uid" = COALESCE(EXCLUDED."uid", "searchchat"."uid"), "roomId" = COALESCE(EXCLUDED."roomId", "searchchat"."roomId")`,
+			values: [ids, data],
+		});
+	} catch (err) {
+		winston.error(`Error indexing ${err.stack}`);
+		await handleError(err);
+		await exports.chat.index(data, ids);
+	}
+};
+
+exports.chat.search = async (data, limit) => {
+	if (Array.isArray(data.uid) && data.uid.filter(Boolean).length) {
+		data.uid = data.uid.filter(Boolean);
+	} else {
+		data.uid = null;
+	}
+
+	if (Array.isArray(data.roomId) && data.roomId.filter(Boolean).length) {
+		data.roomId = data.roomId.filter(Boolean);
+	} else {
+		data.roomId = null;
+	}
+	try {
+		const res = await db.pool.query({
+			name: `dbsearch-search-chat`,
+			text: `SELECT ARRAY(SELECT s."id" FROM "searchchat" s WHERE ($1::text IS NULL OR to_tsvector($5::regconfig, "content") @@ plainto_tsquery($5::regconfig, $1::text)) AND ($2::bigint[] IS NULL OR "uid" = ANY($2::bigint[])) AND ($3::bigint[] IS NULL OR "roomId" = ANY($3::bigint[])) ORDER BY ts_rank_cd(to_tsvector($5::regconfig, "content"), plainto_tsquery($5::regconfig, $1::text)) DESC, s."id" ASC LIMIT $4::integer) r`,
+			values: [data.content, data.uid, data.roomId, parseInt(limit, 10), searchLanguage],
+		});
+		return res.rows[0].r;
+	} catch (err) {
+		await handleError(err);
+		return [];
+	}
+};

package/lib/redis.js

@@ -9,6 +9,7 @@ const db = require.main.require('./src/database');
 exports.createIndices = async function () {
 	db.postSearch = redisSearch.createSearch('nodebbpostsearch', db.client);
 	db.topicSearch = redisSearch.createSearch('nodebbtopicsearch', db.client);
+	db.chatSearch = redisSearch.createSearch('nodebbchatsearch', db.client);
 };
 
 exports.changeIndexLanguage = async function () {
@@ -44,9 +45,39 @@ exports.searchRemove = async function (key, ids) {
 	if (!key || !ids.length) {
 		return;
 	}
-	const method = key === 'post' ? db.postSearch : db.topicSearch;
-
 	await async.eachLimit(ids, 500, async (ids) => {
-		await method.remove(ids);
+		await db[`${key}Search`].remove(ids);
+	});
+};
+
+exports.chat = {};
+exports.chat.index = async (data, ids) => {
+	if (!ids.length) {
+		return;
+	}
+	const indexData = ids.map((id, index) => ({
+		id: id,
+		data: {
+			content: String(data[index].content),
+			roomId: String(data[index].roomId),
+			uid: String(data[index].uid),
+		},
+	}));
+
+	await async.eachLimit(indexData, 500, async (indexData) => {
+		await db.chatSearch.index(indexData.data, indexData.id);
 	});
 };
+
+exports.chat.search = async (data, limit) => {
+	const query = {
+		matchWords: data.matchWords,
+		query: {
+			roomId: data.roomId,
+			uid: data.uid,
+			content: data.content,
+		},
+	};
+
+	return await db.chatSearch.query(query, 0, limit - 1);
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nodebb-plugin-dbsearch",
-  "version": "6.1.0",
+  "version": "6.2.0",
   "description": "A Plugin that lets users search posts and topics",
   "main": "index.js",
   "repository": {
@@ -19,7 +19,7 @@
   "license": "BSD-2-Clause",
   "dependencies": {
     "lodash": "4.17.21",
-    "redisearch": "^1.0.1"
+    "redisearch": "^1.0.3"
   },
   "devDependencies": {
     "eslint": "7.32.0",
@@ -27,6 +27,6 @@
     "eslint-plugin-import": "2.25.2"
   },
   "nbbpm": {
-    "compatibility": "^3.0.0"
+    "compatibility": "^3.3.0"
   }
 }

package/plugin.json

@@ -27,8 +27,14 @@
         { "hook": "action:topic.move", "method": "actionTopicMove" },
         { "hook": "action:topic.changeOwner", "method": "actionTopicChangeOwner" },
 
+        { "hook": "action:messaging.save", "method": "actionMessagingSave" },
+        { "hook": "action:messaging.delete", "method": "actionMessagingDelete" },
+        { "hook": "action:messaging.restore", "method": "actionMessagingRestore" },
+        { "hook": "action:messaging.edit", "method": "actionMessagingEdit" },
+
         { "hook": "filter:search.query", "method": "filterSearchQuery" },
-        { "hook": "filter:topic.search", "method": "filterSearchTopic" }
+        { "hook": "filter:topic.search", "method": "filterSearchTopic" },
+        { "hook": "filter:messaging.searchMessages", "method": "filterMessagingSearchMessages" }
     ],
     "modules": {
         "../admin/plugins/dbsearch.js": "public/admin.js"

package/public/admin.js

@@ -113,8 +113,10 @@ define('admin/plugins/dbsearch', [
 
 			$('#topics-indexed').text(progress.topicsIndexed);
 			$('#posts-indexed').text(progress.postsIndexed);
+			$('#messages-indexed').text(progress.messagesIndexed);
 			$('.topic-progress').css('width', progress.topicsPercent + '%').text(progress.topicsPercent + '%');
 			$('.post-progress').css('width', progress.postsPercent + '%').text(progress.postsPercent + '%');
+			$('.message-progress').css('width', progress.messagesPercent + '%').text(progress.messagesPercent + '%');
 		});
 	}
 

package/templates/admin/plugins/dbsearch.tpl

@@ -25,6 +25,15 @@
 							</div>
 						</div>
 
+						<div class="mb-3">
+							<div class="alert alert-info">
+								Messages Indexed: <strong id="messages-indexed">{messagesIndexed}</strong> / <strong>{messageCount}</strong>
+							</div>
+							<div class="progress" style="height:24px;">
+								<div class="message-progress progress-bar" role="progressbar" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100" style="width:{progressData.messagesPercent}%;min-width: 2em;">{progressData.messagesPercent}%</div>
+							</div>
+						</div>
+
 						<button class="btn btn-warning" id="reindex" <!-- IF working -->disabled<!-- ENDIF working -->>Re Index</button>
 						<button class="btn btn-danger" id="clear-index">Clear Index</button>
 						<span id="work-in-progress" class="<!-- IF !working -->hidden<!-- ENDIF !working -->">