@jasonwch/nodebb-plugin-meilisearch-r 1.0.2 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -2
- package/library.js +437 -18
- package/package.json +5 -4
- package/plugin.json +27 -2
- package/static/languages/en-GB/meilisearch.json +22 -9
- package/static/languages/pl/meilisearch.json +21 -8
- package/static/languages/zh-CN/meilisearch.json +21 -8
- package/static/less/meilisearch.less +28 -5
- package/static/lib/admin.js +26 -3
- package/static/lib/chat-search-global.js +237 -0
- package/static/scss/meilisearch.scss +28 -5
- package/static/templates/admin/plugins/meilisearch/partials/rankingRules/form.tpl +1 -1
- package/static/templates/admin/plugins/meilisearch/partials/stopWords/form.tpl +1 -1
- package/static/templates/admin/plugins/meilisearch.tpl +175 -137
- package/static/templates/client/partials/chats/search-bar.tpl +9 -0
- package/static/templates/client/partials/chats/search-result.tpl +38 -0
package/README.md
CHANGED
|
@@ -7,11 +7,22 @@ A plugin for integrating MeiliSearch with NodeBB (This plugin is inspired and re
|
|
|
7
7
|
|
|
8
8
|
- Refactored for NodeBB 4.14 up.
|
|
9
9
|
- Updated to Meilisearch JS API 0.59.
|
|
10
|
-
-
|
|
10
|
+
- ACP panel comply to NodeBB standard.
|
|
11
11
|
- Reindex / Force reindex all working as expected
|
|
12
12
|
- Composer suggestion optimization.
|
|
13
13
|
- Bug fixes from existing plugin.
|
|
14
|
+
- Chat search support
|
|
15
|
+
- Global chat search support (will auto detect and override ```nodebb-plugin-chat-search```)
|
|
14
16
|
|
|
15
17
|
\
|
|
16
|
-
|
|
18
|
+
<ins>**Composer suggestions**</ins>
|
|
19
|
+
\
|
|
17
20
|

|
|
21
|
+
|
|
22
|
+
<ins>**Global search**</ins>
|
|
23
|
+
\
|
|
24
|
+

|
|
25
|
+
|
|
26
|
+
<ins>**ACP**</ins>
|
|
27
|
+
\
|
|
28
|
+

|
package/library.js
CHANGED
|
@@ -7,6 +7,9 @@ const routeHelpers = nodebb.require('./src/routes/helpers');
|
|
|
7
7
|
const settings = nodebb.require('./src/meta/settings');
|
|
8
8
|
const Posts = nodebb.require('./src/posts');
|
|
9
9
|
const Topics = nodebb.require('./src/topics');
|
|
10
|
+
const Messaging = nodebb.require('./src/messaging');
|
|
11
|
+
const user = nodebb.require('./src/user');
|
|
12
|
+
const translator = nodebb.require('./src/translator');
|
|
10
13
|
const batch = nodebb.require('./src/batch');
|
|
11
14
|
const Sockets = nodebb.require('./src/socket.io');
|
|
12
15
|
const pubsub = nodebb.require('./src/pubsub');
|
|
@@ -26,7 +29,9 @@ const DRAIN_MAX = 500;
|
|
|
26
29
|
const REINDEX_LOCK_KEY = 'plugin:meilisearch:reindex:lock';
|
|
27
30
|
const REINDEX_LOCK_TTL = 3600;
|
|
28
31
|
const LOCK_REFRESH_INTERVAL = 5 * 60 * 1000;
|
|
29
|
-
const REPLAY_OPS = ['indexPost', 'deindexPost', 'indexTopic', 'deindexTopic', 'deindexPostsPurge', 'deindexTopicsPurge', 'reindexTopicPosts', 'deindexTopicPosts', 'restoreTopic', 'changePostOwner', 'changeTopicOwner', 'onTopicMerge', 'onScheduledPublish'];
|
|
32
|
+
const REPLAY_OPS = ['indexPost', 'deindexPost', 'indexTopic', 'deindexTopic', 'deindexPostsPurge', 'deindexTopicsPurge', 'reindexTopicPosts', 'deindexTopicPosts', 'restoreTopic', 'changePostOwner', 'changeTopicOwner', 'onTopicMerge', 'onScheduledPublish', 'indexMessage', 'deindexMessage'];
|
|
33
|
+
const GLOBAL_CHAT_SEARCH_LIMIT = 200;
|
|
34
|
+
const GLOBAL_CHAT_SEARCH_FETCH = 300;
|
|
30
35
|
|
|
31
36
|
const emitProgress = _.throttle(() => {
|
|
32
37
|
pubsub.publish('meilisearch:reindex', plugin.indexing);
|
|
@@ -42,6 +47,10 @@ plugin.indexing = {
|
|
|
42
47
|
total: null,
|
|
43
48
|
current: null,
|
|
44
49
|
},
|
|
50
|
+
message_progress: {
|
|
51
|
+
total: null,
|
|
52
|
+
current: null,
|
|
53
|
+
},
|
|
45
54
|
};
|
|
46
55
|
|
|
47
56
|
plugin.reindexingForced = false;
|
|
@@ -71,13 +80,18 @@ plugin.defaults = {
|
|
|
71
80
|
synonyms: [],
|
|
72
81
|
healthCheckInterval: 60,
|
|
73
82
|
searchMinTermLength: 2,
|
|
83
|
+
globalChatSearchEnabled: 'on',
|
|
74
84
|
lastReindexResult: {
|
|
75
85
|
success: false,
|
|
76
86
|
finishedAt: null,
|
|
77
87
|
topic_progress: { current: null, total: null },
|
|
78
88
|
post_progress: { current: null, total: null },
|
|
89
|
+
message_progress: { current: null, total: null },
|
|
79
90
|
skippedDeletedTopics: 0,
|
|
80
91
|
skippedDeletedPosts: 0,
|
|
92
|
+
skippedDeletedMessages: 0,
|
|
93
|
+
skippedSystemMessages: 0,
|
|
94
|
+
skippedOrphanMessages: 0,
|
|
81
95
|
error: null,
|
|
82
96
|
},
|
|
83
97
|
};
|
|
@@ -115,23 +129,241 @@ plugin.init = async function (params) {
|
|
|
115
129
|
const topicCurrent = Number(plugin.indexing.topic_progress.current) || 0;
|
|
116
130
|
const postTotal = Number(plugin.indexing.post_progress.total) || 0;
|
|
117
131
|
const postCurrent = Number(plugin.indexing.post_progress.current) || 0;
|
|
132
|
+
const messageTotal = Number(plugin.indexing.message_progress.total) || 0;
|
|
133
|
+
const messageCurrent = Number(plugin.indexing.message_progress.current) || 0;
|
|
118
134
|
const topicPercent = topicTotal > 0 ? Math.min(100, Math.max(0, Math.round(100 * topicCurrent / topicTotal))) : 0;
|
|
119
135
|
const postPercent = postTotal > 0 ? Math.min(100, Math.max(0, Math.round(100 * postCurrent / postTotal))) : 0;
|
|
136
|
+
const messagePercent = messageTotal > 0 ? Math.min(100, Math.max(0, Math.round(100 * messageCurrent / messageTotal))) : 0;
|
|
120
137
|
const lastReindexResult = await settings.getOne(plugin.id, 'lastReindexResult') || {};
|
|
121
138
|
res.render('admin/plugins/meilisearch', {
|
|
139
|
+
title: '[[meilisearch:admin.settings]]',
|
|
122
140
|
indexing: plugin.indexing,
|
|
123
141
|
topicPercent,
|
|
124
142
|
postPercent,
|
|
143
|
+
messagePercent,
|
|
125
144
|
lastReindexResult,
|
|
126
145
|
});
|
|
127
146
|
});
|
|
128
147
|
await settings.setOnEmpty(plugin.id, plugin.defaults);
|
|
129
148
|
await plugin.prepareSearch();
|
|
149
|
+
await plugin.ensureChatMessageIndex();
|
|
130
150
|
plugin.patchTopicsSearch();
|
|
131
151
|
plugin.initialized = true;
|
|
132
152
|
};
|
|
133
153
|
|
|
134
|
-
//
|
|
154
|
+
// Ensure the chat_message index + filterable attributes exist, independent of reindex state.
|
|
155
|
+
// prepareSearch() returns early when `indexed=true`, so updateIndexSettings (which configures
|
|
156
|
+
// chat_message) is skipped on normal restarts — but indexMessage auto-creates the index via
|
|
157
|
+
// document add, leaving filterableAttributes empty. Without roomId/uid filterable, searchMessages
|
|
158
|
+
// would error on its filter clause. Idempotent + cheap (2 calls, settings deduped by Meilisearch).
|
|
159
|
+
plugin.ensureChatMessageIndex = async function () {
|
|
160
|
+
if (!plugin.client) return;
|
|
161
|
+
try {
|
|
162
|
+
await ensureIndex('chat_message', 'mid');
|
|
163
|
+
await plugin.client.index('chat_message').updateFilterableAttributes(['roomId', 'uid', 'timestamp']);
|
|
164
|
+
} catch (err) {
|
|
165
|
+
winston.error(`[plugin/meilisearch] ensureChatMessageIndex failed: ${err.message}`);
|
|
166
|
+
plugin.healthy = false;
|
|
167
|
+
}
|
|
168
|
+
};
|
|
169
|
+
|
|
170
|
+
// Global chat search: queries the chat_message Meilisearch index across ALL rooms
|
|
171
|
+
// the user belongs to (unlike the per-room searchMessages which is scoped to one roomId).
|
|
172
|
+
// Uses 1.5× over-fetch (300) to compensate for join-timestamp/deleted-message filtering,
|
|
173
|
+
// then renders only surviving hits through NodeBB's getMessagesData pipeline.
|
|
174
|
+
// Results are sorted by message timestamp (newest first).
|
|
175
|
+
plugin.chatSearchGlobal = async function (socket, data) {
|
|
176
|
+
data = data || {};
|
|
177
|
+
if (!socket.uid) throw new Error('Not logged in');
|
|
178
|
+
const chatSearchEnabled = await settings.getOne(plugin.id, 'globalChatSearchEnabled');
|
|
179
|
+
if (chatSearchEnabled === false || chatSearchEnabled === 'off' || chatSearchEnabled === 'false') {
|
|
180
|
+
return [];
|
|
181
|
+
}
|
|
182
|
+
if (!plugin.healthy && !(await plugin.checkHealth())) {
|
|
183
|
+
throw new Error('MeiliSearch is not healthy');
|
|
184
|
+
}
|
|
185
|
+
const targetUid = socket.uid;
|
|
186
|
+
const userSettings = await user.getSettings(socket.uid);
|
|
187
|
+
const userLang = userSettings.userLang || 'en-GB';
|
|
188
|
+
|
|
189
|
+
const query = String(data.query || '').trim();
|
|
190
|
+
if (!query) {
|
|
191
|
+
return [];
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
let roomIds = await db.getSortedSetRevRange('uid:' + targetUid + ':chat:rooms', 0, -1);
|
|
195
|
+
// Chunked membership check with event loop yields to prevent blocking on large room lists.
|
|
196
|
+
// Each chunk fires hooks synchronously inside Promise.all (~0.6ms per 100 rooms);
|
|
197
|
+
// setImmediate between chunks lets other users' I/O callbacks run.
|
|
198
|
+
const MEMBERSHIP_CHUNK_SIZE = 100;
|
|
199
|
+
let inRoom = [];
|
|
200
|
+
for (let i = 0; i < roomIds.length; i += MEMBERSHIP_CHUNK_SIZE) {
|
|
201
|
+
const chunk = roomIds.slice(i, i + MEMBERSHIP_CHUNK_SIZE);
|
|
202
|
+
inRoom = inRoom.concat(await Messaging.isUserInRoom(targetUid, chunk));
|
|
203
|
+
await new Promise(resolve => setImmediate(resolve));
|
|
204
|
+
}
|
|
205
|
+
roomIds = roomIds.filter((roomId, idx) => inRoom[idx]);
|
|
206
|
+
if (!roomIds.length) {
|
|
207
|
+
return [];
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
const minTermLength = await resolveMinTermLength();
|
|
211
|
+
if (!query.split(' ').some(word => word.length >= minTermLength)) {
|
|
212
|
+
return [];
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
const filter = [roomIds.map(rid => `roomId = ${parseInt(rid, 10)}`)];
|
|
216
|
+
winston.debug(`[plugin/meilisearch] Global chat search for "${query}" in ${roomIds.length} rooms`);
|
|
217
|
+
const result = await plugin.client.index('chat_message').search(query, {
|
|
218
|
+
attributesToRetrieve: ['mid', 'roomId', 'uid', 'timestamp'],
|
|
219
|
+
limit: GLOBAL_CHAT_SEARCH_FETCH,
|
|
220
|
+
filter: filter,
|
|
221
|
+
matchingStrategy: 'last',
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
if (!result.hits || !result.hits.length) {
|
|
225
|
+
return [];
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// Batch-fetch room public flags (1 query) + join timestamps for private rooms (N queries).
|
|
229
|
+
const hitRoomIds = [...new Set(result.hits.map(h => parseInt(h.roomId, 10)))];
|
|
230
|
+
const roomData = await db.getObjectsFields(
|
|
231
|
+
hitRoomIds.map(rid => `chat:room:${rid}`),
|
|
232
|
+
['public'],
|
|
233
|
+
);
|
|
234
|
+
const roomIsPublic = {};
|
|
235
|
+
hitRoomIds.forEach((rid, i) => {
|
|
236
|
+
roomIsPublic[rid] = parseInt(roomData[i] && roomData[i].public, 10) === 1;
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
const privateRoomIds = hitRoomIds.filter(rid => !roomIsPublic[rid]);
|
|
240
|
+
const joinTimestamps = {};
|
|
241
|
+
if (privateRoomIds.length) {
|
|
242
|
+
const scores = await Promise.all(
|
|
243
|
+
privateRoomIds.map(rid => db.sortedSetScore(`chat:room:${rid}:uids`, targetUid)),
|
|
244
|
+
);
|
|
245
|
+
privateRoomIds.forEach((rid, i) => {
|
|
246
|
+
joinTimestamps[rid] = scores[i] || 0;
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// Filter + sort by timestamp (newest first) + slice to limit.
|
|
251
|
+
const visibleHits = result.hits
|
|
252
|
+
.filter((hit) => {
|
|
253
|
+
const rid = parseInt(hit.roomId, 10);
|
|
254
|
+
if (roomIsPublic[rid]) return true;
|
|
255
|
+
const ts = parseInt(hit.timestamp, 10);
|
|
256
|
+
return ts >= (joinTimestamps[rid] || 0);
|
|
257
|
+
})
|
|
258
|
+
.sort((a, b) => parseInt(b.timestamp, 10) - parseInt(a.timestamp, 10))
|
|
259
|
+
.slice(0, GLOBAL_CHAT_SEARCH_LIMIT);
|
|
260
|
+
|
|
261
|
+
if (!visibleHits.length) {
|
|
262
|
+
return [];
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// Group by room for decoration, then reassemble in timestamp order via mid→message map.
|
|
266
|
+
const hitsByRoom = {};
|
|
267
|
+
visibleHits.forEach((hit) => {
|
|
268
|
+
const rid = parseInt(hit.roomId, 10);
|
|
269
|
+
if (!hitsByRoom[rid]) hitsByRoom[rid] = [];
|
|
270
|
+
hitsByRoom[rid].push(hit.mid);
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
// Process rooms in bounded-concurrency batches (10 at a time) to avoid
|
|
274
|
+
// saturating the MongoDB connection pool (maxPoolSize: 20).
|
|
275
|
+
const DECORATION_BATCH_SIZE = 10;
|
|
276
|
+
const roomIdsWithHits = Object.keys(hitsByRoom);
|
|
277
|
+
const decoratedMap = {};
|
|
278
|
+
for (let i = 0; i < roomIdsWithHits.length; i += DECORATION_BATCH_SIZE) {
|
|
279
|
+
const batch = roomIdsWithHits.slice(i, i + DECORATION_BATCH_SIZE);
|
|
280
|
+
const batchResults = await Promise.all(batch.map(async (ridStr) => {
|
|
281
|
+
const rid = parseInt(ridStr, 10);
|
|
282
|
+
const mids = hitsByRoom[rid];
|
|
283
|
+
const messages = await Messaging.getMessagesData(mids, targetUid, rid, false);
|
|
284
|
+
if (!messages || !messages.length) return [];
|
|
285
|
+
return decorateRoomMatches({ matches: messages, roomId: rid, targetUid, userLang });
|
|
286
|
+
}));
|
|
287
|
+
batchResults.forEach((decorated) => {
|
|
288
|
+
decorated.forEach((msg) => {
|
|
289
|
+
const mid = msg.mid || msg.messageId;
|
|
290
|
+
if (mid) decoratedMap[mid] = msg;
|
|
291
|
+
});
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
// Reassemble in pre-sorted timestamp order.
|
|
296
|
+
const allResults = [];
|
|
297
|
+
for (const hit of visibleHits) {
|
|
298
|
+
const msg = decoratedMap[hit.mid];
|
|
299
|
+
if (msg) allResults.push(msg);
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
return allResults;
|
|
303
|
+
};
|
|
304
|
+
|
|
305
|
+
// Register socket handler at module level per NodeBB 4.14.x convention
|
|
306
|
+
// (src/socket.io/plugins.js docs: require and add listeners at load time).
|
|
307
|
+
// The health check inside chatSearchGlobal guards against calls before init completes.
|
|
308
|
+
const socketPlugins = nodebb.require('./src/socket.io/plugins');
|
|
309
|
+
socketPlugins.meilisearch = socketPlugins.meilisearch || {};
|
|
310
|
+
socketPlugins.meilisearch.chatSearchGlobal = plugin.chatSearchGlobal;
|
|
311
|
+
|
|
312
|
+
// Expose globalChatSearchEnabled to client config (standard filter:config.get pattern,
|
|
313
|
+
// same as nodebb-plugin-emoji's emojiCustomFirst).
|
|
314
|
+
plugin.addConfig = async function (config) {
|
|
315
|
+
const enabled = await settings.getOne(plugin.id, 'globalChatSearchEnabled');
|
|
316
|
+
config.globalChatSearchEnabled = enabled !== false && enabled !== 'off' && enabled !== 'false';
|
|
317
|
+
return config;
|
|
318
|
+
};
|
|
319
|
+
|
|
320
|
+
// Attaches room name, participants, and sender metadata to each message for the client UI.
|
|
321
|
+
async function decorateRoomMatches({ matches, roomId, targetUid, userLang }) {
|
|
322
|
+
const uids = await Messaging.getUidsInRoom(roomId, 0, -1);
|
|
323
|
+
const usersData = await user.getUsersFields(uids, ['uid', 'username', 'picture', 'icon:text', 'icon:bgColor']);
|
|
324
|
+
const otherUsers = usersData.filter(u => parseInt(u.uid, 10) !== parseInt(targetUid, 10));
|
|
325
|
+
|
|
326
|
+
let displayName = '';
|
|
327
|
+
if (otherUsers.length === 0) {
|
|
328
|
+
displayName = await translateMeili(userLang, 'chatSearch.room.self-chat');
|
|
329
|
+
} else if (otherUsers.length <= 2) {
|
|
330
|
+
displayName = otherUsers.map(u => u.username).join(', ');
|
|
331
|
+
} else {
|
|
332
|
+
const firstTwo = otherUsers.slice(0, 2).map(u => u.username).join(', ');
|
|
333
|
+
const remaining = otherUsers.length - 2;
|
|
334
|
+
displayName = await translateMeili(userLang, 'chatSearch.room.and-more-users', firstTwo, remaining);
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
const roomData = await Messaging.getRoomData(roomId);
|
|
338
|
+
const roomName = (roomData && roomData.roomName) || displayName;
|
|
339
|
+
|
|
340
|
+
// Batch-fetch sorted-set ranks for all matched mids.
|
|
341
|
+
// Core's /message/:mid redirect drops the index segment when rank is 0 (falsy),
|
|
342
|
+
// so we attach the rank here and construct the chat URL client-side with rank+1
|
|
343
|
+
// to always include a truthy 1-based index segment.
|
|
344
|
+
const midsForRank = matches.map(m => m.mid || m.messageId);
|
|
345
|
+
const ranks = await db.sortedSetRanks(`chat:room:${roomId}:mids`, midsForRank);
|
|
346
|
+
|
|
347
|
+
matches.forEach((m, i) => {
|
|
348
|
+
if (!m.mid && m.messageId) m.mid = m.messageId;
|
|
349
|
+
if (!m.roomId) m.roomId = roomId;
|
|
350
|
+
if (!m.user || !m.user.username) {
|
|
351
|
+
m.user = m.fromUser || { username: 'Unknown', 'icon:text': '?', 'icon:bgColor': 'var(--bs-tertiary-bg)' };
|
|
352
|
+
}
|
|
353
|
+
m.roomName = roomName;
|
|
354
|
+
m.participants = otherUsers.length ? [otherUsers[0]] : [];
|
|
355
|
+
m.rank = ranks[i];
|
|
356
|
+
});
|
|
357
|
+
|
|
358
|
+
return matches;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
async function translateMeili(language, key, ...args) {
|
|
362
|
+
return await translator.translate(
|
|
363
|
+
translator.compile(`meilisearch:${key}`, ...args),
|
|
364
|
+
language,
|
|
365
|
+
);
|
|
366
|
+
}
|
|
135
367
|
plugin.patchTopicsSearch = function () {
|
|
136
368
|
if (Topics.__meiliOriginalSearch) { return; }
|
|
137
369
|
Topics.__meiliOriginalSearch = Topics.search;
|
|
@@ -154,11 +386,14 @@ plugin.prepareSearch = async function (data, connectionChanged = false) {
|
|
|
154
386
|
? Math.min(Math.max(intervalRaw, 10), 3600)
|
|
155
387
|
: 60;
|
|
156
388
|
plugin.healthCheckTask = setInterval(plugin.checkHealth, intervalSeconds * 1000);
|
|
389
|
+
// #15: Always sync plugin.healthy on (re)connect — otherwise it stays false from
|
|
390
|
+
// initialization until the first setInterval tick, causing stale "MeiliSearch
|
|
391
|
+
// Unreachable" warnings on save during the startup window (MS online, flag stale).
|
|
392
|
+
await plugin.checkHealth();
|
|
157
393
|
// #14: Don't auto-reindex after a failed reindex unless connection settings changed.
|
|
158
394
|
const indexed = await settings.getOne(plugin.id, 'indexed');
|
|
159
395
|
if (indexed) return;
|
|
160
|
-
|
|
161
|
-
if (!healthy || plugin.initializingOnAnotherInstance) return;
|
|
396
|
+
if (!plugin.healthy || plugin.initializingOnAnotherInstance) return;
|
|
162
397
|
const lastResult = await settings.getOne(plugin.id, 'lastReindexResult') || {};
|
|
163
398
|
const allowAutoReindex = connectionChanged || lastResult.success !== false;
|
|
164
399
|
if (!allowAutoReindex) {
|
|
@@ -290,6 +525,7 @@ plugin.getNotices = async function (notices) {
|
|
|
290
525
|
plugin.updateIndexSettings = async (data) => {
|
|
291
526
|
await ensureIndex('post', 'pid');
|
|
292
527
|
await ensureIndex('topic', 'tid');
|
|
528
|
+
await ensureIndex('chat_message', 'mid');
|
|
293
529
|
data = {
|
|
294
530
|
maxDocuments: parseInt(data?.maxDocuments || await settings.getOne(plugin.id, 'maxDocuments') || 500, 10),
|
|
295
531
|
rankingRules: (data?.rankingRules || await settings.getOne(plugin.id, 'rankingRules'))?.map(value => value.rule),
|
|
@@ -354,7 +590,26 @@ plugin.updateIndexSettings = async (data) => {
|
|
|
354
590
|
},
|
|
355
591
|
synonyms: data.synonyms,
|
|
356
592
|
});
|
|
357
|
-
|
|
593
|
+
const messageTask = await plugin.client.index('chat_message').updateSettings({
|
|
594
|
+
filterableAttributes: ['roomId', 'uid', 'timestamp'],
|
|
595
|
+
sortableAttributes: ['timestamp'],
|
|
596
|
+
searchableAttributes: ['content'],
|
|
597
|
+
pagination: {
|
|
598
|
+
maxTotalHits: data.maxDocuments,
|
|
599
|
+
},
|
|
600
|
+
rankingRules: data.rankingRules,
|
|
601
|
+
stopWords: data.stopWords,
|
|
602
|
+
typoTolerance: {
|
|
603
|
+
enabled: data.typoTolerance,
|
|
604
|
+
minWordSizeForTypos: {
|
|
605
|
+
oneTypo: data.typoToleranceMinWordSizeOneTypo,
|
|
606
|
+
twoTypos: data.typoToleranceMinWordSizeTwoTypos,
|
|
607
|
+
},
|
|
608
|
+
disableOnWords: data.typoToleranceDisableOnWords,
|
|
609
|
+
},
|
|
610
|
+
synonyms: data.synonyms,
|
|
611
|
+
});
|
|
612
|
+
return [postTask, topicTask, messageTask];
|
|
358
613
|
};
|
|
359
614
|
|
|
360
615
|
plugin.reindex = async function (force = false) {
|
|
@@ -372,9 +627,13 @@ plugin.reindex = async function (force = false) {
|
|
|
372
627
|
current: 0,
|
|
373
628
|
total: 0,
|
|
374
629
|
},
|
|
630
|
+
message_progress: {
|
|
631
|
+
current: 0,
|
|
632
|
+
total: 0,
|
|
633
|
+
},
|
|
375
634
|
};
|
|
376
635
|
emitProgress();
|
|
377
|
-
winston.info(`[plugin/meilisearch] Indexing posts and
|
|
636
|
+
winston.info(`[plugin/meilisearch] Indexing posts, topics and chat messages${force ? ' (forced)' : ''}`);
|
|
378
637
|
plugin._reindexPromise = runReindex(force);
|
|
379
638
|
return plugin._reindexPromise;
|
|
380
639
|
};
|
|
@@ -411,6 +670,9 @@ async function runReindex(force) {
|
|
|
411
670
|
let errorMsg = null;
|
|
412
671
|
let skippedDeletedTopics = 0;
|
|
413
672
|
let skippedDeletedPosts = 0;
|
|
673
|
+
let skippedDeletedMessages = 0;
|
|
674
|
+
let skippedSystemMessages = 0;
|
|
675
|
+
let skippedOrphanMessages = 0;
|
|
414
676
|
const locked = await acquireReindexLock();
|
|
415
677
|
if (!locked) {
|
|
416
678
|
plugin.indexing.running = false;
|
|
@@ -423,14 +685,18 @@ async function runReindex(force) {
|
|
|
423
685
|
plugin.reindexingForced = true;
|
|
424
686
|
const postTask = await plugin.client.index('post').deleteAllDocuments();
|
|
425
687
|
const topicTask = await plugin.client.index('topic').deleteAllDocuments();
|
|
426
|
-
await plugin.client.
|
|
427
|
-
await plugin.client.tasks.waitForTask(
|
|
688
|
+
const messageTask = await plugin.client.index('chat_message').deleteAllDocuments();
|
|
689
|
+
await plugin.client.tasks.waitForTask(postTask.taskUid, { timeout: 600000, interval: 200 });
|
|
690
|
+
await plugin.client.tasks.waitForTask(topicTask.taskUid, { timeout: 600000, interval: 200 });
|
|
691
|
+
await plugin.client.tasks.waitForTask(messageTask.taskUid, { timeout: 600000, interval: 200 });
|
|
428
692
|
}
|
|
429
|
-
const [postSettingsTask, topicSettingsTask] = await plugin.updateIndexSettings();
|
|
430
|
-
await plugin.client.tasks.waitForTask(postSettingsTask.taskUid);
|
|
431
|
-
await plugin.client.tasks.waitForTask(topicSettingsTask.taskUid);
|
|
693
|
+
const [postSettingsTask, topicSettingsTask, messageSettingsTask] = await plugin.updateIndexSettings();
|
|
694
|
+
await plugin.client.tasks.waitForTask(postSettingsTask.taskUid, { timeout: 600000, interval: 200 });
|
|
695
|
+
await plugin.client.tasks.waitForTask(topicSettingsTask.taskUid, { timeout: 600000, interval: 200 });
|
|
696
|
+
await plugin.client.tasks.waitForTask(messageSettingsTask.taskUid, { timeout: 600000, interval: 200 });
|
|
432
697
|
plugin.indexing.topic_progress.total = await db.sortedSetCard('topics:tid') || 0;
|
|
433
698
|
plugin.indexing.post_progress.total = await db.sortedSetCard('posts:pid') || 0;
|
|
699
|
+
plugin.indexing.message_progress.total = await db.sortedSetCard('messages:mid') || 0;
|
|
434
700
|
emitProgress();
|
|
435
701
|
await Promise.all([
|
|
436
702
|
batch.processSortedSet(
|
|
@@ -513,10 +779,64 @@ async function runReindex(force) {
|
|
|
513
779
|
batch: REINDEX_BATCH_SIZE,
|
|
514
780
|
progress: plugin.indexing.post_progress,
|
|
515
781
|
},
|
|
516
|
-
|
|
517
|
-
|
|
782
|
+
),
|
|
783
|
+
]);
|
|
784
|
+
// Reindex chat messages SEQUENTIALLY after topics+posts (keeps Meilisearch enqueue rate bounded).
|
|
785
|
+
// Option B: cross-check each batch's roomIds against existing rooms and deindex orphan messages
|
|
786
|
+
// whose room was deleted via admin (messaging.deleteRooms fires no message hook and leaves
|
|
787
|
+
// message:${mid} objects + messages:mid entries, so they would otherwise be re-indexed as orphans).
|
|
788
|
+
await batch.processSortedSet(
|
|
789
|
+
'messages:mid',
|
|
790
|
+
async (mids) => {
|
|
791
|
+
plugin.indexing.message_progress.current += mids.length;
|
|
792
|
+
emitProgress();
|
|
793
|
+
const messages = await Messaging.getMessagesFields(mids, ['mid', 'content', 'roomId', 'fromuid', 'timestamp', 'deleted', 'system']);
|
|
794
|
+
// Resolve deleted/system messages and orphan rooms in one pass.
|
|
795
|
+
const seenRoomIds = [...new Set(messages.map(m => m && m.roomId).filter(rid => rid != null && rid !== ''))];
|
|
796
|
+
const roomsData = seenRoomIds.length ? await Messaging.getRoomsData(seenRoomIds) : [];
|
|
797
|
+
const roomIdToExists = {};
|
|
798
|
+
roomsData.forEach((room) => {
|
|
799
|
+
if (room && room.roomId != null) roomIdToExists[room.roomId] = true;
|
|
800
|
+
});
|
|
801
|
+
const activeMessages = [];
|
|
802
|
+
const deletedMids = [];
|
|
803
|
+
const systemMids = [];
|
|
804
|
+
const orphanMids = [];
|
|
805
|
+
messages.forEach((msg) => {
|
|
806
|
+
if (!msg || !msg.mid) return;
|
|
807
|
+
if (parseInt(msg.deleted, 10) === 1) {
|
|
808
|
+
deletedMids.push(msg.mid);
|
|
809
|
+
} else if (parseInt(msg.system, 10) === 1) {
|
|
810
|
+
systemMids.push(msg.mid);
|
|
811
|
+
} else if (!roomIdToExists[msg.roomId]) {
|
|
812
|
+
orphanMids.push(msg.mid);
|
|
813
|
+
} else {
|
|
814
|
+
activeMessages.push(msg);
|
|
815
|
+
}
|
|
816
|
+
});
|
|
817
|
+
skippedDeletedMessages += deletedMids.length;
|
|
818
|
+
skippedSystemMessages += systemMids.length;
|
|
819
|
+
skippedOrphanMessages += orphanMids.length;
|
|
820
|
+
const allDeindexMids = [...deletedMids, ...systemMids, ...orphanMids];
|
|
821
|
+
if (allDeindexMids.length) {
|
|
822
|
+
await plugin.client.index('chat_message').deleteDocuments(allDeindexMids);
|
|
823
|
+
}
|
|
824
|
+
if (!activeMessages.length) return;
|
|
825
|
+
await plugin.client.index('chat_message').updateDocuments(
|
|
826
|
+
activeMessages.map(msg => ({
|
|
827
|
+
mid: msg.mid,
|
|
828
|
+
roomId: msg.roomId,
|
|
829
|
+
uid: msg.fromuid,
|
|
830
|
+
content: msg.content,
|
|
831
|
+
timestamp: msg.timestamp,
|
|
832
|
+
})),
|
|
833
|
+
{ primaryKey: 'mid' },
|
|
834
|
+
);
|
|
835
|
+
},
|
|
836
|
+
{ batch: REINDEX_BATCH_SIZE, progress: plugin.indexing.message_progress },
|
|
837
|
+
);
|
|
518
838
|
succeeded = true;
|
|
519
|
-
winston.info('[plugin/meilisearch] Indexing complete');
|
|
839
|
+
winston.info('[plugin/meilisearch] Indexing of posts, topics and chat messages complete');
|
|
520
840
|
} catch (err) {
|
|
521
841
|
winston.error(`[plugin/meilisearch] Indexing failed: ${err.message}`);
|
|
522
842
|
errorMsg = err.message;
|
|
@@ -547,8 +867,15 @@ async function runReindex(force) {
|
|
|
547
867
|
current: plugin.indexing.post_progress.current,
|
|
548
868
|
total: plugin.indexing.post_progress.total,
|
|
549
869
|
},
|
|
870
|
+
message_progress: {
|
|
871
|
+
current: plugin.indexing.message_progress.current,
|
|
872
|
+
total: plugin.indexing.message_progress.total,
|
|
873
|
+
},
|
|
550
874
|
skippedDeletedTopics,
|
|
551
875
|
skippedDeletedPosts,
|
|
876
|
+
skippedDeletedMessages,
|
|
877
|
+
skippedSystemMessages,
|
|
878
|
+
skippedOrphanMessages,
|
|
552
879
|
error: errorMsg,
|
|
553
880
|
};
|
|
554
881
|
try {
|
|
@@ -933,6 +1260,7 @@ plugin.checkConflict = function () {
|
|
|
933
1260
|
const hooksToCheck = [
|
|
934
1261
|
'filter:search.query',
|
|
935
1262
|
'filter:topic.search',
|
|
1263
|
+
'filter:messaging.searchMessages',
|
|
936
1264
|
];
|
|
937
1265
|
// blacklist, in case someone makes a plugin using these hooks that doesn't conflict.
|
|
938
1266
|
// also, outside of dbsearch the user is expected to realize they installed two search plugins.
|
|
@@ -950,6 +1278,97 @@ plugin.checkConflict = function () {
|
|
|
950
1278
|
return false;
|
|
951
1279
|
};
|
|
952
1280
|
|
|
1281
|
+
plugin.indexMessage = async function ({ message }) {
|
|
1282
|
+
const deferPayload = { message };
|
|
1283
|
+
if (await maybeDefer('indexMessage', deferPayload)) return;
|
|
1284
|
+
if (!message || !message.mid) return;
|
|
1285
|
+
// Re-fetch from DB if content/roomId/fromuid/timestamp missing or if this is an edit (payload may be partial).
|
|
1286
|
+
// Restore payload (delete.js) omits timestamp; save/edit payloads carry it. Re-fetch fills the gap.
|
|
1287
|
+
if (message.content === undefined || message.roomId === undefined || message.fromuid === undefined || message.timestamp === undefined) {
|
|
1288
|
+
const fresh = await Messaging.getMessageFields(message.mid, ['mid', 'content', 'roomId', 'fromuid', 'timestamp', 'deleted', 'system']);
|
|
1289
|
+
// Bail if message was hard-purged mid-flight (fresh=null or stub with mid=0) — avoids indexing a phantom doc.
|
|
1290
|
+
if (!fresh || !fresh.mid) return;
|
|
1291
|
+
message = { ...fresh, ...message };
|
|
1292
|
+
}
|
|
1293
|
+
// Skip deleted messages (admin editing a deleted message should not re-index it).
|
|
1294
|
+
if (parseInt(message.deleted, 10) === 1) {
|
|
1295
|
+
winston.debug(`[plugin/meilisearch] Skipping deleted message ${message.mid}`);
|
|
1296
|
+
return;
|
|
1297
|
+
}
|
|
1298
|
+
// Skip system messages (join/leave/rename etc. — no searchable content, mirrors dbsearch).
|
|
1299
|
+
if (parseInt(message.system, 10) === 1) {
|
|
1300
|
+
winston.debug(`[plugin/meilisearch] Skipping system message ${message.mid}`);
|
|
1301
|
+
return;
|
|
1302
|
+
}
|
|
1303
|
+
const doc = {
|
|
1304
|
+
mid: message.mid,
|
|
1305
|
+
roomId: message.roomId,
|
|
1306
|
+
uid: message.fromuid,
|
|
1307
|
+
content: message.content,
|
|
1308
|
+
timestamp: message.timestamp,
|
|
1309
|
+
};
|
|
1310
|
+
await meiliWrite('indexMessage', { message: { mid: message.mid } }, () =>
|
|
1311
|
+
plugin.client.index('chat_message').updateDocuments([doc], { primaryKey: 'mid' }));
|
|
1312
|
+
};
|
|
1313
|
+
|
|
1314
|
+
plugin.deindexMessage = async function ({ message }) {
|
|
1315
|
+
const deferPayload = { message };
|
|
1316
|
+
if (await maybeDefer('deindexMessage', deferPayload)) return;
|
|
1317
|
+
if (!message || !message.mid) return;
|
|
1318
|
+
await meiliWrite('deindexMessage', { message }, () =>
|
|
1319
|
+
plugin.client.index('chat_message').deleteDocument(message.mid));
|
|
1320
|
+
};
|
|
1321
|
+
|
|
1322
|
+
plugin.searchMessages = async function (data) {
|
|
1323
|
+
if (!data || !data.content) {
|
|
1324
|
+
return data;
|
|
1325
|
+
}
|
|
1326
|
+
if (plugin.checkConflict()) {
|
|
1327
|
+
winston.warn('[plugin/meilisearch] Another search plugin (most likely dbsearch) is enabled, so chat search via Meilisearch was aborted.');
|
|
1328
|
+
return data;
|
|
1329
|
+
}
|
|
1330
|
+
if (!plugin.healthy && !(await plugin.checkHealth())) {
|
|
1331
|
+
winston.warn('[plugin/meilisearch] Meilisearch instance did not return a healthy response, so chat search via Meilisearch was aborted.');
|
|
1332
|
+
return data;
|
|
1333
|
+
}
|
|
1334
|
+
const rawQuery = String(data.content).trim();
|
|
1335
|
+
const minTermLength = await resolveMinTermLength();
|
|
1336
|
+
if (!rawQuery || !rawQuery.split(' ').some(word => word.length >= minTermLength)) {
|
|
1337
|
+
winston.debug(`[plugin/meilisearch] Skipping chat search: no query word >= ${minTermLength} char(s)`);
|
|
1338
|
+
return data;
|
|
1339
|
+
}
|
|
1340
|
+
const num = (v) => {
|
|
1341
|
+
const n = parseInt(v, 10);
|
|
1342
|
+
return Number.isFinite(n) ? n : null;
|
|
1343
|
+
};
|
|
1344
|
+
const filter = [];
|
|
1345
|
+
// Core passes roomId/uid as arrays (filter:messaging.searchMessages payload in src/api/search.js).
|
|
1346
|
+
const roomIds = Array.isArray(data.roomId) ? data.roomId : [data.roomId];
|
|
1347
|
+
const roomNums = roomIds.map(num).filter(r => r !== null);
|
|
1348
|
+
// Fail closed: if no valid roomId, return no results rather than searching globally across all rooms.
|
|
1349
|
+
if (!roomNums.length) {
|
|
1350
|
+
winston.warn('[plugin/meilisearch] searchMessages called with no valid roomId; returning no results');
|
|
1351
|
+
return data;
|
|
1352
|
+
}
|
|
1353
|
+
filter.push(roomNums.map(rid => `roomId = ${rid}`));
|
|
1354
|
+
// uid is an optional sender filter (message author, not the searcher); never populated by core callers.
|
|
1355
|
+
const uids = Array.isArray(data.uid) ? data.uid : [data.uid];
|
|
1356
|
+
const uidNums = uids.map(num).filter(u => u !== null);
|
|
1357
|
+
if (uidNums.length) {
|
|
1358
|
+
filter.push(uidNums.map(u => `uid = ${u}`));
|
|
1359
|
+
}
|
|
1360
|
+
const limit = parseInt(await settings.getOne(plugin.id, 'maxDocuments') || 100, 10);
|
|
1361
|
+
winston.debug(`[plugin/meilisearch] Searching chat messages for "${rawQuery}" in room ${roomNums.join(',')}`);
|
|
1362
|
+
const result = await plugin.client.index('chat_message').search(rawQuery, {
|
|
1363
|
+
attributesToRetrieve: ['mid'],
|
|
1364
|
+
limit: Math.min(limit, 100),
|
|
1365
|
+
filter: filter.length ? filter : undefined,
|
|
1366
|
+
matchingStrategy: data.matchWords === 'all' ? 'all' : 'last',
|
|
1367
|
+
});
|
|
1368
|
+
data.ids = data.ids.concat(result.hits.map(hit => hit.mid));
|
|
1369
|
+
return data;
|
|
1370
|
+
};
|
|
1371
|
+
|
|
953
1372
|
plugin.search = async function (data) {
|
|
954
1373
|
if (plugin.checkConflict()) {
|
|
955
1374
|
// The dbsearch plugin was detected, abort search!
|
|
@@ -1043,10 +1462,10 @@ plugin.buildSort = function (sortBy, sortDirection) {
|
|
|
1043
1462
|
plugin.saveSettings = async (data) => {
|
|
1044
1463
|
if (data.plugin === plugin.id && !data.quiet && plugin.initialized) {
|
|
1045
1464
|
try {
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1465
|
+
if (data.settings && Object.prototype.hasOwnProperty.call(data.settings, 'searchMinTermLength')) {
|
|
1466
|
+
data.settings.searchMinTermLength = clampMinTermLength(data.settings.searchMinTermLength);
|
|
1467
|
+
}
|
|
1468
|
+
// #8: Only re-connect when connection settings changed.
|
|
1050
1469
|
const connChanged = await connectionSettingsChanged(data.settings);
|
|
1051
1470
|
if (connChanged) {
|
|
1052
1471
|
// Fix 10: If host changed, clear indexed so prepareSearch auto-reindexes the new (empty) host.
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jasonwch/nodebb-plugin-meilisearch-r",
|
|
3
|
-
"version": "1.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "1.1.1",
|
|
4
|
+
"description": "Enhanced plugin for integrating MeiliSearch with NodeBB",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "JasonCH",
|
|
7
7
|
"email": "jasonwch@msn.com"
|
|
@@ -26,8 +26,9 @@
|
|
|
26
26
|
"keywords": [
|
|
27
27
|
"nodebb",
|
|
28
28
|
"plugin",
|
|
29
|
-
"
|
|
30
|
-
"
|
|
29
|
+
"chat",
|
|
30
|
+
"search",
|
|
31
|
+
"meilisearch"
|
|
31
32
|
],
|
|
32
33
|
"nbbpm": {
|
|
33
34
|
"compatibility": "^4.14.0"
|