@jasonwch/nodebb-plugin-meilisearch-r 1.1.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,210 @@
1
+ 'use strict';
2
+
3
+ const winston = nodebb.require('winston');
4
+ const Topics = nodebb.require('./src/topics');
5
+ const batch = nodebb.require('./src/batch');
6
+ const { REINDEX_BATCH_SIZE } = require('../constants');
7
+ const { maybeDefer, meiliWrite, enqueuePending } = require('../pending-queue');
8
+
9
+ // action:topic.* hooks (post/save/edit/move/restore/delete/merge/changeOwner/purge)
10
+ // plus scheduled-topic publish handling.
11
+ module.exports = function attachTopicHooks(plugin) {
12
+ plugin.indexTopic = async function ({ topic, tid, fromCid, post }) {
13
+ const isMove = fromCid !== undefined;
14
+ if (await maybeDefer(plugin, 'indexTopic', { topic, tid, fromCid, post })) return;
15
+ // #2: On topic.move, no 'topic' key is provided — re-fetch by tid.
16
+ if (!topic && tid) {
17
+ topic = await Topics.getTopicData(tid);
18
+ }
19
+ if (!topic) {
20
+ winston.warn('[plugin/meilisearch] indexTopic: no topic data');
21
+ return;
22
+ }
23
+ // Issue 3: Skip deleted topics (admin editing/moving a deleted topic should not re-index it).
24
+ if (parseInt(topic.deleted, 10) === 1) {
25
+ winston.debug(`[plugin/meilisearch] Skipping deleted topic ${topic.tid}`);
26
+ return;
27
+ }
28
+ // Issue 5: Skip scheduled topics (timestamp in future) — indexed on publish via onScheduledPublish.
29
+ // action:topic.post fires AFTER action:post.save, so the main post may already be in Meili.
30
+ // Deindex it here to prevent full-search from finding scheduled content prematurely.
31
+ if (topic.scheduled || (topic.timestamp && topic.timestamp > Date.now())) {
32
+ const mainPid = post ? post.pid : (parseInt(topic.mainPid, 10) > 0 ? topic.mainPid : null);
33
+ if (mainPid) {
34
+ await plugin.client.index('post').deleteDocument(mainPid).catch(() => {});
35
+ }
36
+ winston.debug(`[plugin/meilisearch] Skipping scheduled topic ${topic.tid}, deindexed main post ${mainPid || 'N/A'}`);
37
+ return;
38
+ }
39
+ await meiliWrite(plugin, 'indexTopic', { topic, tid, fromCid, post }, () =>
40
+ plugin.client.index('topic').updateDocuments([{
41
+ tid: topic.tid,
42
+ cid: topic.cid,
43
+ uid: topic.uid,
44
+ mainPid: topic.mainPid,
45
+ title: topic.title,
46
+ timestamp: topic.timestamp,
47
+ }], { primaryKey: 'tid' }));
48
+ // #4: On topic move, re-index child posts' cids (fire-and-forget with fallback).
49
+ if (isMove && topic.tid) {
50
+ if (plugin.healthy) {
51
+ plugin.reindexTopicPosts({ tid: topic.tid }).catch(err =>
52
+ winston.error(`[plugin/meilisearch] reindexTopicPosts failed: ${err.message}`));
53
+ } else {
54
+ await enqueuePending('reindexTopicPosts', { tid: topic.tid });
55
+ }
56
+ }
57
+ };
58
+
59
+ plugin.deindexTopic = async function ({ topic, tid }) {
60
+ if (await maybeDefer(plugin, 'deindexTopic', { topic, tid })) return;
61
+ // Tolerate topic.move-style payloads (no 'topic' key) — extract tid.
62
+ const topicTid = topic ? topic.tid : tid;
63
+ if (!topicTid) {
64
+ winston.warn('[plugin/meilisearch] deindexTopic: no tid');
65
+ return;
66
+ }
67
+ await meiliWrite(plugin, 'deindexTopic', { topic, tid }, () =>
68
+ plugin.client.index('topic').deleteDocument(topicTid));
69
+ // Also deindex the topic's posts from the post index so admin /search
70
+ // (which queries both indexes and bypasses NodeBB's relevance-sort
71
+ // deleted-content filter for admins) can't surface deleted-topic content.
72
+ if (plugin.healthy) {
73
+ plugin.deindexTopicPosts({ tid: topicTid }).catch(err =>
74
+ winston.error(`[plugin/meilisearch] deindexTopicPosts failed: ${err.message}`));
75
+ } else {
76
+ await enqueuePending('deindexTopicPosts', { tid: topicTid });
77
+ }
78
+ };
79
+
80
+ plugin.restoreTopic = async function ({ topic, tid }) {
81
+ const topicTid = topic ? topic.tid : tid;
82
+ if (await maybeDefer(plugin, 'restoreTopic', { topic, tid })) return;
83
+ // Re-fetch topic if missing (defensive — same pattern as indexTopic).
84
+ if (!topic && topicTid) {
85
+ topic = await Topics.getTopicData(topicTid);
86
+ }
87
+ if (!topic) {
88
+ winston.warn('[plugin/meilisearch] restoreTopic: no topic data');
89
+ return;
90
+ }
91
+ // Re-index the topic document.
92
+ await meiliWrite(plugin, 'restoreTopic', { topic, tid }, () =>
93
+ plugin.client.index('topic').updateDocuments([{
94
+ tid: topic.tid,
95
+ cid: topic.cid,
96
+ uid: topic.uid,
97
+ mainPid: topic.mainPid,
98
+ title: topic.title,
99
+ timestamp: topic.timestamp,
100
+ }], { primaryKey: 'tid' }));
101
+ // Re-index the topic's posts (they were deindexed on delete).
102
+ if (plugin.healthy) {
103
+ plugin.reindexTopicPosts({ tid: topic.tid }).catch(err =>
104
+ winston.error(`[plugin/meilisearch] restoreTopic reindexTopicPosts failed: ${err.message}`));
105
+ } else {
106
+ await enqueuePending('reindexTopicPosts', { tid: topic.tid });
107
+ }
108
+ };
109
+
110
+ plugin.changeTopicOwner = async function ({ topics, toUid }) {
111
+ if (await maybeDefer(plugin, 'changeTopicOwner', { topics, toUid })) return;
112
+ // topics have OLD uid — re-fetch from DB to get fresh uid.
113
+ // Skip deleted or scheduled topics.
114
+ const validTids = (Array.isArray(topics) ? topics : [])
115
+ .filter(t => t && t.tid && parseInt(t.deleted, 10) !== 1)
116
+ .map(t => t.tid);
117
+ if (!validTids.length) return;
118
+ const freshTopics = await Topics.getTopicsFields(validTids, ['tid', 'cid', 'uid', 'mainPid', 'title', 'timestamp']);
119
+ // Fix 7: Skip scheduled topics (timestamp in future).
120
+ const activeTopics = freshTopics.filter(t => t && !(t.timestamp && t.timestamp > Date.now()));
121
+ if (!activeTopics.length) return;
122
+ const docs = activeTopics.map(topic => ({
123
+ tid: topic.tid,
124
+ cid: topic.cid,
125
+ uid: topic.uid,
126
+ mainPid: topic.mainPid,
127
+ title: topic.title,
128
+ timestamp: topic.timestamp,
129
+ }));
130
+ await meiliWrite(plugin, 'changeTopicOwner', { topics: activeTopics.map(t => ({ tid: t.tid })), toUid }, () =>
131
+ plugin.client.index('topic').updateDocuments(docs, { primaryKey: 'tid' }));
132
+ };
133
+
134
+ plugin.onTopicMerge = async function ({ otherTids, mergeIntoTid }) {
135
+ if (await maybeDefer(plugin, 'onTopicMerge', { otherTids, mergeIntoTid })) return;
136
+ const postIndex = plugin.client.index('post');
137
+ // Deindex merged-out topics + their posts (Topics.delete in merge fires no action:topic.delete).
138
+ for (const tid of otherTids) {
139
+ await plugin.deindexTopic({ tid });
140
+ // Merge zeros mainPid before Topics.delete, so deindexTopicPosts can't find the main post.
141
+ // Fallback: search Meili for any orphaned posts still carrying this tid.
142
+ try {
143
+ const result = await postIndex.search('', { filter: `tid = ${parseInt(tid, 10)}`, limit: 1000 });
144
+ if (result.hits && result.hits.length) {
145
+ const orphanPids = result.hits.map(h => h.pid);
146
+ await meiliWrite(plugin, 'deindexPostsPurge', { posts: orphanPids.map(pid => ({ pid })) }, () =>
147
+ postIndex.deleteDocuments(orphanPids));
148
+ }
149
+ } catch (err) {
150
+ winston.error(`[plugin/meilisearch] onTopicMerge orphan cleanup for tid ${tid} failed: ${err.message}`);
151
+ plugin.healthy = false;
152
+ // Don't enqueue — orphans are cleaned by next reindex. Continue to next tid.
153
+ }
154
+ }
155
+ // Re-index the merge target (postcount/lastpost changed; posts already moved via action:post.move).
156
+ await plugin.indexTopic({ tid: mergeIntoTid });
157
+ if (plugin.healthy) {
158
+ plugin.reindexTopicPosts({ tid: mergeIntoTid }).catch(err =>
159
+ winston.error(`[plugin/meilisearch] onTopicMerge reindexTopicPosts failed: ${err.message}`));
160
+ }
161
+ };
162
+
163
+ plugin.onScheduledPublish = async function ({ topics }) {
164
+ const tids = (Array.isArray(topics) ? topics : []).filter(t => t && t.tid).map(t => t.tid);
165
+ if (!tids.length) return;
166
+ if (await maybeDefer(plugin, 'onScheduledPublish', { topics: tids.map(tid => ({ tid })) })) return;
167
+ // Re-index each published topic + its posts (DB deleted=0, timestamp in past now).
168
+ for (const tid of tids) {
169
+ await plugin.indexTopic({ tid });
170
+ if (plugin.healthy) {
171
+ plugin.reindexTopicPosts({ tid }).catch(err =>
172
+ winston.error(`[plugin/meilisearch] onScheduledPublish reindexTopicPosts failed: ${err.message}`));
173
+ }
174
+ }
175
+ };
176
+
177
+ plugin.deindexTopicsPurge = async function ({ topics }) {
178
+ if (await maybeDefer(plugin, 'deindexTopicsPurge', { topics })) return;
179
+ const topicsArr = Array.isArray(topics) ? topics.filter(t => t && t.tid) : [];
180
+ if (!topicsArr.length) {
181
+ return;
182
+ }
183
+ const tids = topicsArr.map(t => t.tid);
184
+ const mainPids = topicsArr.map(t => t.mainPid).filter(Boolean);
185
+ const postIndex = plugin.client.index('post');
186
+ const reducedPayload = { topics: topicsArr.map(t => ({ tid: t.tid, mainPid: t.mainPid })) };
187
+ try {
188
+ await Promise.all([
189
+ plugin.client.index('topic').deleteDocuments(tids),
190
+ mainPids.length ? postIndex.deleteDocuments(mainPids) : Promise.resolve(),
191
+ ]);
192
+ await Promise.all(topicsArr.map(async (t) => {
193
+ await batch.processSortedSet(
194
+ `tid:${t.tid}:posts`,
195
+ async (pids) => {
196
+ if (!pids.length) {
197
+ return;
198
+ }
199
+ await postIndex.deleteDocuments(pids);
200
+ },
201
+ { batch: REINDEX_BATCH_SIZE },
202
+ );
203
+ }));
204
+ } catch (err) {
205
+ winston.error(`[plugin/meilisearch] deindexTopicsPurge failed: ${err.message}`);
206
+ plugin.healthy = false;
207
+ await enqueuePending('deindexTopicsPurge', reducedPayload);
208
+ }
209
+ };
210
+ };
@@ -0,0 +1,69 @@
1
+ 'use strict';
2
+
3
+ const winston = nodebb.require('winston');
4
+ const db = nodebb.require('./src/database');
5
+ const { PENDING_KEY, PENDING_MAX, DRAIN_MAX, REPLAY_OPS } = require('./constants');
6
+
7
+ // Persists a write that couldn't be applied to MeiliSearch yet, so it can be replayed later.
8
+ async function enqueuePending(op, payload) {
9
+ const count = await db.sortedSetCard(PENDING_KEY);
10
+ if (count >= PENDING_MAX) {
11
+ const oldest = await db.getSortedSetRange(PENDING_KEY, 0, 0);
12
+ if (oldest.length) await db.sortedSetRemove(PENDING_KEY, ...oldest);
13
+ }
14
+ await db.sortedSetAdd(PENDING_KEY, Date.now(), JSON.stringify({ op, payload }));
15
+ }
16
+
17
+ // Decides whether a write hook should be deferred instead of applied immediately:
18
+ // buffered in memory during a forced reindex, or queued to the DB while MeiliSearch is unhealthy.
19
+ async function maybeDefer(plugin, op, payload) {
20
+ if (plugin.reindexingForced) {
21
+ if (plugin.pendingDuringReindex.length >= PENDING_MAX) {
22
+ plugin.pendingDuringReindex.shift();
23
+ }
24
+ plugin.pendingDuringReindex.push({ op, payload });
25
+ return true;
26
+ }
27
+ if (!plugin.healthy && !(await plugin.checkHealth())) {
28
+ await enqueuePending(op, payload);
29
+ return true;
30
+ }
31
+ return false;
32
+ }
33
+
34
+ // Runs a MeiliSearch write, falling back to the pending queue on failure.
35
+ async function meiliWrite(plugin, op, payload, fn) {
36
+ try {
37
+ await fn();
38
+ } catch (err) {
39
+ winston.error(`[plugin/meilisearch] ${op} write failed: ${err.message}`);
40
+ plugin.healthy = false;
41
+ await enqueuePending(op, payload);
42
+ }
43
+ }
44
+
45
+ // Replays queued writes once MeiliSearch is healthy again.
46
+ async function drainPending(plugin) {
47
+ if (!plugin.healthy || plugin.reindexingForced) return;
48
+ let drained = 0;
49
+ for (;;) {
50
+ if (!plugin.healthy) break;
51
+ const members = await db.getSortedSetRange(PENDING_KEY, 0, 49);
52
+ if (!members.length) break;
53
+ await db.sortedSetRemove(PENDING_KEY, ...members);
54
+ for (const member of members) {
55
+ try {
56
+ const { op, payload } = JSON.parse(member);
57
+ if (op && REPLAY_OPS.includes(op)) {
58
+ await plugin[op](payload);
59
+ }
60
+ } catch (err) {
61
+ winston.error(`[plugin/meilisearch] replay failed: ${err.message}`);
62
+ }
63
+ drained += 1;
64
+ }
65
+ if (drained >= DRAIN_MAX) break;
66
+ }
67
+ }
68
+
69
+ module.exports = { enqueuePending, maybeDefer, meiliWrite, drainPending };
package/lib/reindex.js ADDED
@@ -0,0 +1,311 @@
1
+ 'use strict';
2
+
3
+ const _ = nodebb.require('lodash');
4
+ const winston = nodebb.require('winston');
5
+ const db = nodebb.require('./src/database');
6
+ const batch = nodebb.require('./src/batch');
7
+ const Posts = nodebb.require('./src/posts');
8
+ const Topics = nodebb.require('./src/topics');
9
+ const Messaging = nodebb.require('./src/messaging');
10
+ const pubsub = nodebb.require('./src/pubsub');
11
+ const settings = nodebb.require('./src/meta/settings');
12
+ const {
13
+ REINDEX_BATCH_SIZE, REINDEX_LOCK_KEY, REINDEX_LOCK_TTL, LOCK_REFRESH_INTERVAL, REPLAY_OPS,
14
+ } = require('./constants');
15
+ const { isSemanticSearchEnabled } = require('./embedder');
16
+ const { waitForSucceededTask } = require('./client');
17
+
18
+ // Full reindex of posts, topics, and chat messages, guarded by a cross-instance lock.
19
+ module.exports = function attachReindex(plugin) {
20
+ const emitProgress = _.throttle(() => {
21
+ pubsub.publish('meilisearch:reindex', plugin.indexing);
22
+ }, 500, { leading: true, trailing: true });
23
+
24
+ async function acquireReindexLock() {
25
+ const ttl = await db.ttl(REINDEX_LOCK_KEY);
26
+ if (ttl > 0) {
27
+ winston.warn('[plugin/meilisearch] Reindex lock held by another instance');
28
+ return false;
29
+ }
30
+ await db.set(REINDEX_LOCK_KEY, String(Date.now()));
31
+ await db.expire(REINDEX_LOCK_KEY, REINDEX_LOCK_TTL);
32
+ plugin.reindexLockRefresh = setInterval(() => {
33
+ db.expire(REINDEX_LOCK_KEY, REINDEX_LOCK_TTL).catch(err =>
34
+ winston.error(`[plugin/meilisearch] lock refresh failed: ${err.message}`));
35
+ }, LOCK_REFRESH_INTERVAL);
36
+ return true;
37
+ }
38
+
39
+ async function releaseReindexLock() {
40
+ if (plugin.reindexLockRefresh) {
41
+ clearInterval(plugin.reindexLockRefresh);
42
+ plugin.reindexLockRefresh = null;
43
+ }
44
+ try {
45
+ await db.delete(REINDEX_LOCK_KEY);
46
+ } catch (err) {
47
+ winston.warn(`[plugin/meilisearch] Could not release reindex lock: ${err.message}`);
48
+ }
49
+ }
50
+
51
+ async function runReindex(force) {
52
+ let succeeded = false;
53
+ let errorMsg = null;
54
+ let skippedDeletedTopics = 0;
55
+ let skippedDeletedPosts = 0;
56
+ let skippedDeletedMessages = 0;
57
+ let skippedSystemMessages = 0;
58
+ let skippedOrphanMessages = 0;
59
+ const locked = await acquireReindexLock();
60
+ if (!locked) {
61
+ plugin.indexing.running = false;
62
+ return;
63
+ }
64
+ pubsub.publish('meilisearch:init', true);
65
+ plugin.initializingOnAnotherInstance = true;
66
+ try {
67
+ if (force) {
68
+ plugin.reindexingForced = true;
69
+ if (await isSemanticSearchEnabled(plugin)) {
70
+ // Every document added back after this will need a fresh embedding call
71
+ // against the configured (often paid) embedding API - there is no way
72
+ // around that once the documents themselves are gone, so surface it here.
73
+ winston.warn('[plugin/meilisearch] Force reindex with semantic search enabled: all vector embeddings will be regenerated from scratch via the embedding API.');
74
+ }
75
+ const postTask = await plugin.client.index('post').deleteAllDocuments();
76
+ const topicTask = await plugin.client.index('topic').deleteAllDocuments();
77
+ const messageTask = await plugin.client.index('chat_message').deleteAllDocuments();
78
+ await waitForSucceededTask(plugin, postTask.taskUid, { timeout: 600000, interval: 200 });
79
+ await waitForSucceededTask(plugin, topicTask.taskUid, { timeout: 600000, interval: 200 });
80
+ await waitForSucceededTask(plugin, messageTask.taskUid, { timeout: 600000, interval: 200 });
81
+ }
82
+ const [postSettingsTask, topicSettingsTask, messageSettingsTask] = await plugin.updateIndexSettings();
83
+ await waitForSucceededTask(plugin, postSettingsTask.taskUid, { timeout: 600000, interval: 200 });
84
+ await waitForSucceededTask(plugin, topicSettingsTask.taskUid, { timeout: 600000, interval: 200 });
85
+ await waitForSucceededTask(plugin, messageSettingsTask.taskUid, { timeout: 600000, interval: 200 });
86
+ // Explicit, deliberate part of a reindex run - ensures the embedder config
87
+ // matches current ACP settings before documents start flowing back in. Gated
88
+ // internally (plugin.updateEmbedders) so it's a no-op unless something about
89
+ // the semantic search config actually changed since it was last applied.
90
+ await plugin.updateEmbedders();
91
+ plugin.indexing.topic_progress.total = await db.sortedSetCard('topics:tid') || 0;
92
+ plugin.indexing.post_progress.total = await db.sortedSetCard('posts:pid') || 0;
93
+ plugin.indexing.message_progress.total = await db.sortedSetCard('messages:mid') || 0;
94
+ emitProgress();
95
+ await Promise.all([
96
+ batch.processSortedSet(
97
+ 'topics:tid',
98
+ async (tids) => {
99
+ plugin.indexing.topic_progress.current += tids.length;
100
+ emitProgress();
101
+ const topics = await Topics.getTopicsFields(tids, ['tid', 'cid', 'uid', 'mainPid', 'title', 'timestamp', 'deleted']);
102
+ const activeTopics = [];
103
+ const deletedTids = [];
104
+ topics.forEach((topic) => {
105
+ if (parseInt(topic.deleted, 10) === 1) {
106
+ deletedTids.push(topic.tid);
107
+ } else {
108
+ activeTopics.push(topic);
109
+ }
110
+ });
111
+ skippedDeletedTopics += deletedTids.length;
112
+ if (deletedTids.length) {
113
+ await plugin.client.index('topic').deleteDocuments(deletedTids);
114
+ }
115
+ if (!activeTopics.length) return;
116
+ await plugin.client.index('topic').updateDocuments(
117
+ activeTopics.map(topic => ({
118
+ tid: topic.tid,
119
+ cid: topic.cid,
120
+ uid: topic.uid,
121
+ mainPid: topic.mainPid,
122
+ title: topic.title,
123
+ timestamp: topic.timestamp,
124
+ })),
125
+ { primaryKey: 'tid' },
126
+ );
127
+ },
128
+ {
129
+ batch: REINDEX_BATCH_SIZE,
130
+ progress: plugin.indexing.topic_progress,
131
+ },
132
+ ),
133
+ batch.processSortedSet(
134
+ 'posts:pid',
135
+ async (pids) => {
136
+ plugin.indexing.post_progress.current += pids.length;
137
+ emitProgress();
138
+ const posts = await Posts.getPostsFields(pids, ['pid', 'tid', 'uid', 'content', 'timestamp', 'deleted']);
139
+ // Fetch topic deleted status for posts in this batch
140
+ const tids = [...new Set(posts.map(p => p && p.tid).filter(Boolean))];
141
+ const topicsData = tids.length ? await Topics.getTopicsFields(tids, ['tid', 'deleted']) : [];
142
+ const tidToDeleted = {};
143
+ topicsData.forEach((t) => { tidToDeleted[t.tid] = parseInt(t.deleted, 10) === 1; });
144
+
145
+ const activePosts = [];
146
+ const deletedPids = [];
147
+ posts.forEach((post) => {
148
+ if (parseInt(post.deleted, 10) === 1 || tidToDeleted[post.tid]) {
149
+ deletedPids.push(post.pid);
150
+ } else {
151
+ activePosts.push(post);
152
+ }
153
+ });
154
+ skippedDeletedPosts += deletedPids.length;
155
+ if (deletedPids.length) {
156
+ await plugin.client.index('post').deleteDocuments(deletedPids);
157
+ }
158
+ const activePids = activePosts.map(p => p.pid);
159
+ const cids = activePids.length ? await Posts.getCidsByPids(activePids) : [];
160
+ await plugin.client.index('post').updateDocuments(
161
+ activePosts.map((post, index) => ({
162
+ pid: post.pid,
163
+ tid: post.tid,
164
+ cid: cids[index],
165
+ uid: post.uid,
166
+ content: post.content,
167
+ timestamp: post.timestamp,
168
+ })),
169
+ { primaryKey: 'pid' },
170
+ );
171
+ },
172
+ {
173
+ batch: REINDEX_BATCH_SIZE,
174
+ progress: plugin.indexing.post_progress,
175
+ },
176
+ ),
177
+ ]);
178
+ // Reindex chat messages SEQUENTIALLY after topics+posts (keeps Meilisearch enqueue rate bounded).
179
+ // Option B: cross-check each batch's roomIds against existing rooms and deindex orphan messages
180
+ // whose room was deleted via admin (messaging.deleteRooms fires no message hook and leaves
181
+ // message:${mid} objects + messages:mid entries, so they would otherwise be re-indexed as orphans).
182
+ await batch.processSortedSet(
183
+ 'messages:mid',
184
+ async (mids) => {
185
+ plugin.indexing.message_progress.current += mids.length;
186
+ emitProgress();
187
+ const messages = await Messaging.getMessagesFields(mids, ['mid', 'content', 'roomId', 'fromuid', 'timestamp', 'deleted', 'system']);
188
+ // Resolve deleted/system messages and orphan rooms in one pass.
189
+ const seenRoomIds = [...new Set(messages.map(m => m && m.roomId).filter(rid => rid != null && rid !== ''))];
190
+ const roomsData = seenRoomIds.length ? await Messaging.getRoomsData(seenRoomIds) : [];
191
+ const roomIdToExists = {};
192
+ roomsData.forEach((room) => {
193
+ if (room && room.roomId != null) roomIdToExists[room.roomId] = true;
194
+ });
195
+ const activeMessages = [];
196
+ const deletedMids = [];
197
+ const systemMids = [];
198
+ const orphanMids = [];
199
+ messages.forEach((msg) => {
200
+ if (!msg || !msg.mid) return;
201
+ if (parseInt(msg.deleted, 10) === 1) {
202
+ deletedMids.push(msg.mid);
203
+ } else if (parseInt(msg.system, 10) === 1) {
204
+ systemMids.push(msg.mid);
205
+ } else if (!roomIdToExists[msg.roomId]) {
206
+ orphanMids.push(msg.mid);
207
+ } else {
208
+ activeMessages.push(msg);
209
+ }
210
+ });
211
+ skippedDeletedMessages += deletedMids.length;
212
+ skippedSystemMessages += systemMids.length;
213
+ skippedOrphanMessages += orphanMids.length;
214
+ const allDeindexMids = [...deletedMids, ...systemMids, ...orphanMids];
215
+ if (allDeindexMids.length) {
216
+ await plugin.client.index('chat_message').deleteDocuments(allDeindexMids);
217
+ }
218
+ if (!activeMessages.length) return;
219
+ await plugin.client.index('chat_message').updateDocuments(
220
+ activeMessages.map(msg => ({
221
+ mid: msg.mid,
222
+ roomId: msg.roomId,
223
+ uid: msg.fromuid,
224
+ content: msg.content,
225
+ timestamp: msg.timestamp,
226
+ })),
227
+ { primaryKey: 'mid' },
228
+ );
229
+ },
230
+ { batch: REINDEX_BATCH_SIZE, progress: plugin.indexing.message_progress },
231
+ );
232
+ succeeded = true;
233
+ winston.info('[plugin/meilisearch] Indexing of posts, topics and chat messages complete');
234
+ } catch (err) {
235
+ winston.error(`[plugin/meilisearch] Indexing failed: ${err.message}`);
236
+ errorMsg = err.message;
237
+ } finally {
238
+ plugin.indexing.running = false;
239
+ plugin.reindexingForced = false;
240
+ plugin.initializingOnAnotherInstance = false;
241
+ pubsub.publish('meilisearch:init', false);
242
+ await releaseReindexLock();
243
+ emitProgress.flush();
244
+ const pending = plugin.pendingDuringReindex.splice(0);
245
+ for (const { op, payload } of pending) {
246
+ if (!REPLAY_OPS.includes(op)) continue;
247
+ try {
248
+ await plugin[op](payload);
249
+ } catch (err) {
250
+ winston.error(`[plugin/meilisearch] replay ${op} failed: ${err.message}`);
251
+ }
252
+ }
253
+ const lastIndexResult = {
254
+ success: succeeded,
255
+ finishedAt: Date.now(),
256
+ topic_progress: {
257
+ current: plugin.indexing.topic_progress.current,
258
+ total: plugin.indexing.topic_progress.total,
259
+ },
260
+ post_progress: {
261
+ current: plugin.indexing.post_progress.current,
262
+ total: plugin.indexing.post_progress.total,
263
+ },
264
+ message_progress: {
265
+ current: plugin.indexing.message_progress.current,
266
+ total: plugin.indexing.message_progress.total,
267
+ },
268
+ skippedDeletedTopics,
269
+ skippedDeletedPosts,
270
+ skippedDeletedMessages,
271
+ skippedSystemMessages,
272
+ skippedOrphanMessages,
273
+ error: errorMsg,
274
+ };
275
+ try {
276
+ await settings.set(plugin.id, { lastReindexResult: lastIndexResult }, true);
277
+ if (succeeded) {
278
+ await settings.set(plugin.id, { indexed: true }, true);
279
+ }
280
+ } catch (err) {
281
+ winston.error(`[plugin/meilisearch] Failed to persist reindex result: ${err.message}`);
282
+ }
283
+ }
284
+ }
285
+
286
+ plugin.reindex = async function (force = false) {
287
+ if (plugin.indexing.running) {
288
+ winston.warn('[plugin/meilisearch] Already indexing');
289
+ return;
290
+ }
291
+ plugin.indexing = {
292
+ running: true,
293
+ topic_progress: {
294
+ current: 0,
295
+ total: 0,
296
+ },
297
+ post_progress: {
298
+ current: 0,
299
+ total: 0,
300
+ },
301
+ message_progress: {
302
+ current: 0,
303
+ total: 0,
304
+ },
305
+ };
306
+ emitProgress();
307
+ winston.info(`[plugin/meilisearch] Indexing posts, topics and chat messages${force ? ' (forced)' : ''}`);
308
+ plugin._reindexPromise = runReindex(force);
309
+ return plugin._reindexPromise;
310
+ };
311
+ };