@jasonwch/nodebb-plugin-meilisearch-r 1.1.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/client.js ADDED
@@ -0,0 +1,445 @@
1
+ 'use strict';
2
+
3
+ const winston = nodebb.require('winston');
4
+ const settings = nodebb.require('./src/meta/settings');
5
+ const Topics = nodebb.require('./src/topics');
6
+ const { drainPending } = require('./pending-queue');
7
+ const {
8
+ EMBEDDER_NAME, buildEmbedderConfig, validateEmbedderConfig, deepEqual,
9
+ getAppliedConfig, getAppliedTaskUid, setAppliedConfigs, clearAppliedConfig,
10
+ } = require('./embedder');
11
+ const {
12
+ EMBEDDER_TASK_TIMEOUT_MS, EMBEDDER_TASK_BG_TIMEOUT_MS, EMBEDDER_TASK_BG_INTERVAL_MS,
13
+ MEILI_HTTP_TIMEOUT_MS,
14
+ } = require('./constants');
15
+
16
+ async function ensureIndex(plugin, uid, primaryKey) {
17
+ try {
18
+ await plugin.client.getIndex(uid);
19
+ } catch (e) {
20
+ await plugin.client.createIndex(uid, { primaryKey });
21
+ }
22
+ }
23
+
24
+ // Meilisearch settings/document mutations are enqueued tasks - the initial call resolving
25
+ // just means Meilisearch accepted the request, not that it succeeded. waitForTask() itself
26
+ // only ever throws on ITS OWN polling timeout (confirmed against the installed
27
+ // meilisearch-js's TaskClient#waitForTask, node_modules/meilisearch/dist/index.js): a task
28
+ // that fails on the server resolves normally with status "failed" and an `error` object, so
29
+ // callers that don't check `.status` (as every waitForTask call in this plugin used to)
30
+ // silently treat a failed settings/document change as if it had gone through.
31
+ async function waitForSucceededTask(plugin, taskUidOrTask, options) {
32
+ const task = await plugin.client.tasks.waitForTask(taskUidOrTask, options);
33
+ if (task.status !== 'succeeded') {
34
+ throw new Error(task.error?.message || `Meilisearch task ${task.status}`);
35
+ }
36
+ return task;
37
+ }
38
+
39
+ // Module-level tracker for in-flight embedder background pollers. Keyed by taskUid so the
40
+ // same task can't be polled twice (e.g. if updateEmbedders is somehow re-entered during
41
+ // the 60s sync wait). Cleared on terminal status (success or failure) by the poller itself.
42
+ const embedderPollers = new Set();
43
+
44
+ // Background-verify an embedder task that exceeded the sync wait window. Per the Meilisearch
45
+ // SDK (node_modules/meilisearch/dist/index.js:152), a poll timeout throws
46
+ // MeilisearchTaskTimeOutError — task is still running server-side. Re-embedding every
47
+ // document legitimately exceeds 60s on a forum of any size; the task may yet succeed.
48
+ // Strategy: poll at a longer interval with a longer timeout; alert only on delayed failure.
49
+ // On delayed success: update the persisted taskUid to null (sync-verified) so reattachPollers
50
+ // skips it on future restarts instead of hitting a 404 when Meilisearch purges task history.
51
+ // On delayed failure: revert the optimistic fingerprint via clearAppliedConfig so the next
52
+ // save pushes fresh config, alert admins, flip plugin.healthy=false to defer subsequent
53
+ // writes to the pending queue.
54
+ // On poll timeout (MeilisearchTaskTimeOutError): task is STILL RUNNING — do NOT clear the
55
+ // fingerprint (would re-open the double-paid-API-cost loop). Log and let the task continue
56
+ // server-side; if it ultimately fails, the next restart's reattachPollers will catch it.
57
+ // On network error: task status is genuinely unknown — log warning but do NOT clear, since
58
+ // clearing on a transient blip would cause an unnecessary re-push. reattachPollers catches
59
+ // actual failures on the next restart.
60
+ function startEmbedderPoller(plugin, plan, taskUid) {
61
+ if (embedderPollers.has(taskUid)) return;
62
+ embedderPollers.add(taskUid);
63
+ const startTime = Date.now();
64
+ const poll = async () => {
65
+ try {
66
+ const task = await plugin.client.tasks.waitForTask(taskUid, {
67
+ timeout: EMBEDDER_TASK_BG_TIMEOUT_MS,
68
+ interval: EMBEDDER_TASK_BG_INTERVAL_MS,
69
+ });
70
+ embedderPollers.delete(taskUid);
71
+ if (task.status !== 'succeeded') {
72
+ // Delayed failure: fingerprint was optimistically persisted at enqueue time
73
+ // (see plugin.updateEmbedders catch-block). Clear it so the next save pushes
74
+ // fresh config rather than silently skipping as "already applied".
75
+ await clearAppliedConfig(plugin, plan.indexName);
76
+ plugin.healthy = false;
77
+ plugin.notifyAdmins(`embedder:${plan.indexName}`, {
78
+ type: 'danger',
79
+ titleKey: '[[meilisearch:admin.semanticEmbedderDelayedFailed]]',
80
+ message: `${plan.indexName}: ${task.error?.message || `task ${task.status}`}`,
81
+ });
82
+ } else {
83
+ // Delayed success: mark taskUid as null (sync-verified) so reattachPollers
84
+ // skips this index on future restarts instead of hitting a 404 when Meilisearch
85
+ // purges task history (~24h). Fingerprint config is already correct — no change.
86
+ winston.info(`[plugin/meilisearch] Embedder task for "${plan.indexName}" succeeded after ${Date.now() - startTime}ms (background-poll)`);
87
+ await setAppliedConfigs(plugin, [{ indexName: plan.indexName, config: plan.config, taskUid: null }]);
88
+ }
89
+ } catch (err) {
90
+ embedderPollers.delete(taskUid);
91
+ if (err.name === 'MeilisearchTaskTimeOutError') {
92
+ // Poller's own 30-min timeout fired — task is STILL RUNNING server-side.
93
+ // Do NOT clear the fingerprint: that would make the next save re-push identical
94
+ // config, re-opening the double-paid-API-cost loop this whole mechanism prevents.
95
+ // The task will eventually succeed or fail; if NodeBB restarts before that,
96
+ // reattachPollers will pick it up via the persisted taskUid.
97
+ winston.warn(`[plugin/meilisearch] Embedder task ${taskUid} for "${plan.indexName}" still running after ${EMBEDDER_TASK_BG_TIMEOUT_MS}ms; giving up poll. Task continues server-side — reattachPollers will reconcile on next restart.`);
98
+ return;
99
+ }
100
+ // Network error or other non-timeout failure — task status is genuinely unknown.
101
+ // Don't clear the fingerprint on a transient blip (would cause unnecessary re-push).
102
+ // If the task actually failed, reattachPollers will catch it on the next restart
103
+ // via the persisted taskUid → getTask(uid) → status: "failed" → clearAppliedConfig.
104
+ winston.warn(`[plugin/meilisearch] Embedder poller for "${plan.indexName}" (task ${taskUid}) hit an error (non-fatal, fingerprint preserved): ${err.message}`);
105
+ }
106
+ };
107
+ poll().catch(err => winston.error(`[plugin/meilisearch] embedder poller crashed: ${err.message}`));
108
+ }
109
+
110
+ // Recover embedder pollers after process restart. embedderPollers (line 42) is module-level
111
+ // in-memory; a NodeBB restart/crash/container-redeploy during the 30-min background-poller
112
+ // window (EMBEDDER_TASK_BG_TIMEOUT_MS) wipes it. This sweep reads the persisted taskUid
113
+ // (stored alongside each fingerprint in appliedEmbedders via setAppliedConfigs) and
114
+ // reconciles directly via plugin.client.tasks.getTask(uid):
115
+ // succeeded → no-op (fingerprint correct, taskUid was null from sync-success path)
116
+ // enqueued/processing → reattach startEmbedderPoller (task still running server-side)
117
+ // failed → clearAppliedConfig + alert (Scenario A: task failed during downtime)
118
+ // 404/network error → assume success, log warning (self-heals on next config change)
119
+ // For old-format fingerprints (no taskUid), falls through to query-based sweep as fallback.
120
+ async function reattachPollers(plugin) {
121
+ if (!plugin.client) return;
122
+ const ourIndexes = ['post', 'topic', 'chat_message'];
123
+ // Phase 1: Direct reconciliation via persisted taskUid (new-format fingerprints)
124
+ for (const indexName of ourIndexes) {
125
+ try {
126
+ const taskUid = await getAppliedTaskUid(plugin, indexName);
127
+ if (taskUid === undefined || taskUid === null) continue;
128
+ const task = await plugin.client.tasks.getTask(taskUid);
129
+ if (task.status === 'succeeded') continue;
130
+ if (task.status === 'enqueued' || task.status === 'processing') {
131
+ if (embedderPollers.has(taskUid)) continue;
132
+ const config = await buildEmbedderConfig(plugin, null, indexName);
133
+ winston.info(`[plugin/meilisearch] Reattaching embedder poller for "${indexName}" (task ${taskUid} still ${task.status})`);
134
+ startEmbedderPoller(plugin, { indexName, config }, taskUid);
135
+ continue;
136
+ }
137
+ // Task failed during downtime — clear stale fingerprint + alert
138
+ winston.warn(`[plugin/meilisearch] Embedder task ${taskUid} for "${indexName}" failed during downtime: ${task.error?.message || task.status}`);
139
+ await clearAppliedConfig(plugin, indexName);
140
+ plugin.healthy = false;
141
+ plugin.notifyAdmins(`embedder:${indexName}`, {
142
+ type: 'danger',
143
+ titleKey: '[[meilisearch:admin.semanticEmbedderDelayedFailed]]',
144
+ message: `${indexName}: ${task.error?.message || `task ${task.status}`}`,
145
+ });
146
+ } catch (err) {
147
+ // Non-fatal: taskUid may be 404 (purged from Meilisearch's ~24h history), or
148
+ // Meilisearch may be momentarily unreachable. Assume success — self-heals on next
149
+ // legitimate semantic-key change.
150
+ winston.warn(`[plugin/meilisearch] reattachPollers: could not reconcile taskUid for "${indexName}" (non-fatal): ${err.message}`);
151
+ }
152
+ }
153
+ // Phase 2: Query-based fallback for old-format fingerprints (no taskUid persisted).
154
+ // Catches in-flight embedder tasks that have a persisted fingerprint but no taskUid
155
+ // (pre-Option-1 format). Will be a no-op once all fingerprints are migrated to new format.
156
+ try {
157
+ const tasksResp = await plugin.client.tasks.getTasks({
158
+ statuses: ['enqueued', 'processing'],
159
+ types: ['settingsUpdate'],
160
+ limit: 50,
161
+ });
162
+ const candidates = [];
163
+ for (const t of (tasksResp.results || [])) {
164
+ if (!ourIndexes.includes(t.indexUid)) continue;
165
+ if (embedderPollers.has(t.uid)) continue;
166
+ if (!(t.details && t.details.embedders)) continue;
167
+ const tu = await getAppliedTaskUid(plugin, t.indexUid);
168
+ if (tu !== undefined) continue; // new-format fingerprint — already reconciled in Phase 1
169
+ candidates.push(t);
170
+ }
171
+ if (!candidates.length) return;
172
+ winston.info(`[plugin/meilisearch] Query fallback: reattaching ${candidates.length} embedder poller(s) for old-format fingerprints`);
173
+ for (const task of candidates) {
174
+ const applied = await getAppliedConfig(plugin, task.indexUid);
175
+ if (applied === undefined) continue;
176
+ const config = await buildEmbedderConfig(plugin, null, task.indexUid);
177
+ startEmbedderPoller(plugin, { indexName: task.indexUid, config }, task.uid);
178
+ }
179
+ } catch (err) {
180
+ winston.warn(`[plugin/meilisearch] reattachPollers query fallback failed (non-fatal): ${err.message}`);
181
+ }
182
+ }
183
+
184
+ // MeiliSearch connection lifecycle: connecting, health checks, and index/settings sync.
185
+ module.exports = function attachClient(plugin) {
186
+ plugin.patchTopicsSearch = function () {
187
+ if (Topics.__meiliOriginalSearch) { return; }
188
+ Topics.__meiliOriginalSearch = Topics.search;
189
+ Topics.search = async function (tid, term) {
190
+ if (!tid || !term || !String(term).trim()) { return []; }
191
+ return Topics.__meiliOriginalSearch.call(Topics, tid, term);
192
+ };
193
+ };
194
+
195
+ plugin.checkHealth = async function () {
196
+ try {
197
+ plugin.healthy = await plugin.client.isHealthy();
198
+ if (!plugin.healthy) {
199
+ winston.warn('[plugin/meilisearch] MeiliSearch host is unhealthy');
200
+ } else {
201
+ drainPending(plugin).catch(err => winston.error(`[plugin/meilisearch] drain failed: ${err.message}`));
202
+ }
203
+ return plugin.healthy;
204
+ } catch (err) {
205
+ plugin.healthy = false;
206
+ winston.warn(`[plugin/meilisearch] ${err.message}`);
207
+ return false;
208
+ }
209
+ };
210
+
211
+ // Ensure the chat_message index + filterable attributes exist, independent of reindex state.
212
+ // prepareSearch() returns early when `indexed=true`, so updateIndexSettings (which configures
213
+ // chat_message) is skipped on normal restarts — but indexMessage auto-creates the index via
214
+ // document add, leaving filterableAttributes empty. Without roomId/uid filterable, searchMessages
215
+ // would error on its filter clause. Idempotent + cheap (2 calls, settings deduped by Meilisearch).
216
+ plugin.ensureChatMessageIndex = async function () {
217
+ if (!plugin.client) return;
218
+ try {
219
+ await ensureIndex(plugin, 'chat_message', 'mid');
220
+ await plugin.client.index('chat_message').updateFilterableAttributes(['roomId', 'uid', 'timestamp']);
221
+ } catch (err) {
222
+ winston.error(`[plugin/meilisearch] ensureChatMessageIndex failed: ${err.message}`);
223
+ plugin.healthy = false;
224
+ }
225
+ };
226
+
227
+ plugin.prepareSearch = async function (data, connectionChanged = false) {
228
+ winston.debug(`[plugin/meilisearch] Connecting to MeiliSearch host: ${data?.host || await settings.getOne(plugin.id, 'host')}`);
229
+ const { Meilisearch } = await import('meilisearch');
230
+ plugin.client = new Meilisearch({
231
+ host: data?.host || await settings.getOne(plugin.id, 'host'),
232
+ apiKey: data?.apiKey || await settings.getOne(plugin.id, 'apiKey') || undefined,
233
+ timeout: MEILI_HTTP_TIMEOUT_MS,
234
+ });
235
+ if (plugin.healthCheckTask) clearInterval(plugin.healthCheckTask);
236
+ const intervalRaw = parseInt(data?.healthCheckInterval || await settings.getOne(plugin.id, 'healthCheckInterval'), 10);
237
+ const intervalSeconds = Number.isFinite(intervalRaw) && intervalRaw > 0
238
+ ? Math.min(Math.max(intervalRaw, 10), 3600)
239
+ : 60;
240
+ plugin.healthCheckTask = setInterval(plugin.checkHealth, intervalSeconds * 1000);
241
+ // #15: Always sync plugin.healthy on (re)connect — otherwise it stays false from
242
+ // initialization until the first setInterval tick, causing stale "MeiliSearch
243
+ // Unreachable" warnings on save during the startup window (MS online, flag stale).
244
+ await plugin.checkHealth();
245
+ // G1: Recover in-flight embedder pollers after process restart (runs once on startup,
246
+ // not on every health-check tick — the sweep is unnecessary after the first run since
247
+ // embedderPollers Set is populated and deduplicates subsequent calls).
248
+ reattachPollers(plugin).catch(err => winston.warn(`[plugin/meilisearch] reattachPollers failed: ${err.message}`));
249
+ // #14: Don't auto-reindex after a failed reindex unless connection settings changed.
250
+ const indexed = await settings.getOne(plugin.id, 'indexed');
251
+ if (indexed) return;
252
+ if (!plugin.healthy || plugin.initializingOnAnotherInstance) return;
253
+ const lastResult = await settings.getOne(plugin.id, 'lastReindexResult') || {};
254
+ const allowAutoReindex = connectionChanged || lastResult.success !== false;
255
+ if (!allowAutoReindex) {
256
+ winston.warn('[plugin/meilisearch] Skipping auto-reindex: last reindex failed. Trigger manually from ACP.');
257
+ return;
258
+ }
259
+ await plugin.updateIndexSettings();
260
+ await plugin.updateEmbedders();
261
+ await plugin.reindex(false);
262
+ };
263
+
264
+ // Keyword-search index settings only (ranking rules, stop words, typo tolerance,
265
+ // synonyms, pagination). Deliberately does NOT touch embedders - see updateEmbedders
266
+ // below for why that has to stay a separate, independently-gated call.
267
+ plugin.updateIndexSettings = async (data) => {
268
+ await ensureIndex(plugin, 'post', 'pid');
269
+ await ensureIndex(plugin, 'topic', 'tid');
270
+ await ensureIndex(plugin, 'chat_message', 'mid');
271
+ data = {
272
+ maxDocuments: parseInt(data?.maxDocuments || await settings.getOne(plugin.id, 'maxDocuments') || 500, 10),
273
+ rankingRules: (data?.rankingRules || await settings.getOne(plugin.id, 'rankingRules'))?.map(value => value.rule),
274
+ stopWords: (data?.stopWords || await settings.getOne(plugin.id, 'stopWords'))?.map(value => value.word),
275
+ typoTolerance: ['on', true].includes(
276
+ data?.typoTolerance || await settings.getOne(plugin.id, 'typoTolerance') || undefined,
277
+ ),
278
+ typoToleranceMinWordSizeOneTypo: parseInt(
279
+ data?.typoToleranceMinWordSizeOneTypo ||
280
+ await settings.getOne(plugin.id, 'typoToleranceMinWordSizeOneTypo') || 5,
281
+ 10,
282
+ ),
283
+ typoToleranceMinWordSizeTwoTypos: parseInt(
284
+ data?.typoToleranceMinWordSizeTwoTypos ||
285
+ await settings.getOne(plugin.id, 'typoToleranceMinWordSizeTwoTypos') || 9,
286
+ 10,
287
+ ),
288
+ typoToleranceDisableOnWords:
289
+ (data?.typoToleranceDisableOnWords || await settings.getOne(plugin.id, 'typoToleranceDisableOnWords') ||
290
+ undefined)?.map(value => value.word),
291
+ synonyms: Object.fromEntries(
292
+ (data?.synonyms || await settings.getOne(plugin.id, 'synonyms') || [])?.map((
293
+ { word, synonyms },
294
+ ) => [word, synonyms?.split(',').map(synonym => synonym.trim())]),
295
+ ),
296
+ };
297
+ const postTask = await plugin.client.index('post').updateSettings({
298
+ filterableAttributes: ['tid', 'cid', 'uid', 'timestamp'],
299
+ sortableAttributes: ['timestamp', 'cid'],
300
+ searchableAttributes: ['content'],
301
+ pagination: {
302
+ maxTotalHits: data.maxDocuments,
303
+ },
304
+ rankingRules: data.rankingRules,
305
+ stopWords: data.stopWords,
306
+ typoTolerance: {
307
+ enabled: data.typoTolerance,
308
+ minWordSizeForTypos: {
309
+ oneTypo: data.typoToleranceMinWordSizeOneTypo,
310
+ twoTypos: data.typoToleranceMinWordSizeTwoTypos,
311
+ },
312
+ disableOnWords: data.typoToleranceDisableOnWords,
313
+ },
314
+ synonyms: data.synonyms,
315
+ });
316
+ const topicTask = await plugin.client.index('topic').updateSettings({
317
+ filterableAttributes: ['cid', 'uid', 'timestamp'],
318
+ sortableAttributes: ['cid', 'title', 'timestamp'],
319
+ searchableAttributes: ['title'],
320
+ pagination: {
321
+ maxTotalHits: data.maxDocuments,
322
+ },
323
+ rankingRules: data.rankingRules,
324
+ stopWords: data.stopWords,
325
+ typoTolerance: {
326
+ enabled: data.typoTolerance,
327
+ minWordSizeForTypos: {
328
+ oneTypo: data.typoToleranceMinWordSizeOneTypo,
329
+ twoTypos: data.typoToleranceMinWordSizeTwoTypos,
330
+ },
331
+ disableOnWords: data.typoToleranceDisableOnWords,
332
+ },
333
+ synonyms: data.synonyms,
334
+ });
335
+ const messageTask = await plugin.client.index('chat_message').updateSettings({
336
+ filterableAttributes: ['roomId', 'uid', 'timestamp'],
337
+ sortableAttributes: ['timestamp'],
338
+ searchableAttributes: ['content'],
339
+ pagination: {
340
+ maxTotalHits: data.maxDocuments,
341
+ },
342
+ rankingRules: data.rankingRules,
343
+ stopWords: data.stopWords,
344
+ typoTolerance: {
345
+ enabled: data.typoTolerance,
346
+ minWordSizeForTypos: {
347
+ oneTypo: data.typoToleranceMinWordSizeOneTypo,
348
+ twoTypos: data.typoToleranceMinWordSizeTwoTypos,
349
+ },
350
+ disableOnWords: data.typoToleranceDisableOnWords,
351
+ },
352
+ synonyms: data.synonyms,
353
+ });
354
+ return [postTask, topicTask, messageTask];
355
+ };
356
+
357
+ // Pushes (or removes) the "default" embedder on all three indexes based on the
358
+ // semantic search ACP settings. Meilisearch re-embeds every existing document
359
+ // through the (often paid) embedding API whenever an index's embedder config
360
+ // actually changes, so this is intentionally its own call rather than something
361
+ // updateIndexSettings does automatically - it must only run when semantic settings
362
+ // were deliberately changed (see lib/settings.js), or as part of an explicit
363
+ // reindex/connect flow, never as a side effect of unrelated settings (ranking rules,
364
+ // stop words, ...) being saved. Skips the network call entirely when the resolved
365
+ // config is identical to what was last pushed, as a second line of defense against
366
+ // needless re-embedding regardless of caller. Older/self-hosted Meilisearch instances
367
+ // without embedder support will reject the call; that's caught here so it never
368
+ // blocks the rest of the settings-save flow.
369
+ plugin.updateEmbedders = async (rawData) => {
370
+ // Pre-flight validation: fail fast with an admin-facing alert instead of letting
371
+ // Meilisearch reject the task server-side (which can take up to 60s on slow/unreachable
372
+ // embedding endpoints, leaving the ACP Save button stuck). Client-side validation in
373
+ // static/lib/admin.js is the primary UX path; this is defense-in-depth for reindex,
374
+ // prepareSearch-on-host-change, and any programmatic caller.
375
+ const validationError = await validateEmbedderConfig(plugin, rawData);
376
+ if (validationError) {
377
+ winston.warn(`[plugin/meilisearch] Embedder config validation failed: ${validationError}`);
378
+ plugin.notifyAdmins('embedder:validation', {
379
+ type: 'danger',
380
+ titleKey: '[[meilisearch:admin.semanticConfigInvalid]]',
381
+ message: validationError,
382
+ });
383
+ return; // no fingerprint persisted, no task enqueued, save proceeds immediately
384
+ }
385
+ const indexNames = ['post', 'topic', 'chat_message'];
386
+ // Resolve configs + diff-against-applied first (read-only), THEN push and persist -
387
+ // keeps the three indexes from racing on the same shared "applied" bookkeeping object
388
+ // (parallel read-modify-write on one settings key would let one index's write silently
389
+ // clobber another's).
390
+ const plans = await Promise.all(indexNames.map(async (indexName) => {
391
+ const config = await buildEmbedderConfig(plugin, rawData, indexName);
392
+ const applied = await getAppliedConfig(plugin, indexName);
393
+ return { indexName, config, needsPush: applied === undefined || !deepEqual(config, applied) };
394
+ }));
395
+ const results = await Promise.all(plans.map(async (plan) => {
396
+ if (!plan.needsPush) return null;
397
+ try {
398
+ // updateEmbedders() only enqueues a Meilisearch task - a bad URL/model/response
399
+ // shape for "rest"/"openAi"-with-custom-url configs only surfaces once Meilisearch
400
+ // actually runs the task (e.g. its probe call to auto-detect dimensions), so this
401
+ // must wait for and check the task's real outcome, not just the enqueue response.
402
+ const enqueued = await plugin.client.index(plan.indexName).updateEmbedders({ [EMBEDDER_NAME]: plan.config });
403
+ try {
404
+ await waitForSucceededTask(plugin, enqueued.taskUid, {
405
+ timeout: EMBEDDER_TASK_TIMEOUT_MS,
406
+ interval: 200,
407
+ });
408
+ } catch (err) {
409
+ // Distinguish "task is still running" (poll timeout) from "task definitively failed"
410
+ // (Meili rejected the call, 4xx, malformed config, network error). Meilisearch SDK
411
+ // throws MeilisearchTaskTimeOutError (name property, see node_modules/meilisearch/dist/index.js:152)
412
+ // on poll timeout; this means the task is still running server-side and may yet succeed.
413
+ if (err.name === 'MeilisearchTaskTimeOutError') {
414
+ // Re-embedding every document legitimately exceeds 60s on a forum of any size.
415
+ // Optimistically persist the fingerprint so the next ACP save doesn't re-push
416
+ // identical config (which would double paid-API cost — the exact thing we're
417
+ // preventing). startEmbedderPoller verifies the real outcome.
418
+ winston.warn(`[plugin/meilisearch] Embedder task for "${plan.indexName}" still running after ${EMBEDDER_TASK_TIMEOUT_MS}ms; background-polling.`);
419
+ startEmbedderPoller(plugin, plan, enqueued.taskUid);
420
+ return { ...plan, taskUid: enqueued.taskUid }; // optimistic — taskUid for later reconciliation
421
+ }
422
+ throw err; // real failure — outer catch handles alerting + return null
423
+ }
424
+ return { ...plan, taskUid: null }; // sync-verified success — no reconciliation needed
425
+ } catch (err) {
426
+ const reason = err.message;
427
+ winston.error(`[plugin/meilisearch] Failed to apply "${plan.indexName}" embedder (semantic search): ${reason}`);
428
+ plugin.notifyAdmins(`embedder:${plan.indexName}`, {
429
+ type: 'danger',
430
+ titleKey: '[[meilisearch:admin.semanticEmbedderFailed]]',
431
+ message: `${plan.indexName}: ${reason}`,
432
+ });
433
+ return null;
434
+ }
435
+ }));
436
+ const succeeded = results.filter(Boolean);
437
+ if (succeeded.length) {
438
+ await setAppliedConfigs(plugin, succeeded);
439
+ }
440
+ };
441
+ };
442
+
443
+ // Exposed so other call sites that enqueue Meilisearch tasks (lib/reindex.js) get the same
444
+ // "actually check the task succeeded" behavior instead of trusting waitForTask() to throw.
445
+ module.exports.waitForSucceededTask = waitForSucceededTask;
@@ -0,0 +1,42 @@
1
+ 'use strict';
2
+
3
+ module.exports = {
4
+ REINDEX_BATCH_SIZE: 500,
5
+ PENDING_KEY: 'plugin:meilisearch:pending',
6
+ PENDING_MAX: 100000,
7
+ DRAIN_MAX: 500,
8
+ REINDEX_LOCK_KEY: 'plugin:meilisearch:reindex:lock',
9
+ REINDEX_LOCK_TTL: 3600,
10
+ LOCK_REFRESH_INTERVAL: 5 * 60 * 1000,
11
+ REPLAY_OPS: [
12
+ 'indexPost', 'deindexPost', 'indexTopic', 'deindexTopic', 'deindexPostsPurge',
13
+ 'deindexTopicsPurge', 'reindexTopicPosts', 'deindexTopicPosts', 'restoreTopic',
14
+ 'changePostOwner', 'changeTopicOwner', 'onTopicMerge', 'onScheduledPublish',
15
+ 'indexMessage', 'deindexMessage',
16
+ ],
17
+ GLOBAL_CHAT_SEARCH_LIMIT: 200,
18
+ GLOBAL_CHAT_SEARCH_FETCH: 300,
19
+ MEMBERSHIP_CHUNK_SIZE: 100,
20
+ DECORATION_BATCH_SIZE: 10,
21
+ // Sync wait window for plugin.updateEmbedders' Meilisearch task before falling back to
22
+ // background poller. 60s keeps ACP save responsive; if the embedder task is still
23
+ // running server-side (re-embedding every doc legitimately exceeds 60s on a forum of
24
+ // any size), updateEmbedders persists the fingerprint optimistically and startEmbedderPoller
25
+ // verifies the eventual outcome.
26
+ EMBEDDER_TASK_TIMEOUT_MS: 60000,
27
+ // Background poller ceiling (60 min) and interval (5 s) for verifying embedder tasks
28
+ // that exceeded the sync wait window. 60 min covers re-embedding up to ~1M posts at
29
+ // OpenAI Tier 1 rate limits (1M TPM). For forums larger than that, the poller gives up
30
+ // and relies on reattachPollers to reconcile the task outcome on the next restart
31
+ // (within Meilisearch's ~24h task retention window). No double-paid-API-cost risk —
32
+ // fingerprint is preserved, not cleared, on poller timeout (see startEmbedderPoller
33
+ // MeilisearchTaskTimeOutError handling in lib/client.js).
34
+ EMBEDDER_TASK_BG_TIMEOUT_MS: 3600000,
35
+ EMBEDDER_TASK_BG_INTERVAL_MS: 5000,
36
+ // Meilisearch SDK client-side HTTP timeout. Caps every fetch() the SDK makes (including
37
+ // the initial enqueue POST for embedder tasks) at 30s — without this, the SDK uses no
38
+ // AbortSignal (node_modules/meilisearch/dist/index.js:340), so an unreachable Meilisearch
39
+ // host can leave plugin.updateEmbedders (and therefore the ACP Save button) stuck
40
+ // indefinitely.
41
+ MEILI_HTTP_TIMEOUT_MS: 30000,
42
+ };
@@ -0,0 +1,73 @@
1
+ 'use strict';
2
+
3
+ const defaults = {
4
+ host: 'http://localhost:7700',
5
+ apiKey: undefined,
6
+ maxDocuments: undefined,
7
+ indexed: false,
8
+ rankingRules: [
9
+ { rule: 'words' },
10
+ { rule: 'typo' },
11
+ { rule: 'proximity' },
12
+ { rule: 'attribute' },
13
+ { rule: 'sort' },
14
+ { rule: 'exactness' },
15
+ ],
16
+ stopWords: [],
17
+ typoTolerance: 'on',
18
+ typoToleranceMinWordSizeOneTypo: 5,
19
+ typoToleranceMinWordSizeTwoTypos: 9,
20
+ typoToleranceDisableOnWords: [],
21
+ synonyms: [],
22
+ healthCheckInterval: 60,
23
+ searchMinTermLength: 2,
24
+ globalChatSearchEnabled: 'on',
25
+ semanticSearchEnabled: 'off',
26
+ semanticSearchProvider: 'openAi',
27
+ semanticSearchApiKey: undefined,
28
+ semanticSearchModel: undefined,
29
+ semanticSearchUrl: undefined,
30
+ semanticSearchDimensions: undefined,
31
+ semanticSearchRatio: 0.5,
32
+ semanticSearchScoreThreshold: 0.2,
33
+ semanticSearchRestRequest: undefined,
34
+ semanticSearchRestResponse: undefined,
35
+ // Internal bookkeeping (not an ACP field): last embedder config actually pushed to
36
+ // Meilisearch per index, so unrelated settings saves can't re-trigger paid re-embedding.
37
+ appliedEmbedders: {},
38
+ lastReindexResult: {
39
+ success: false,
40
+ finishedAt: null,
41
+ topic_progress: { current: null, total: null },
42
+ post_progress: { current: null, total: null },
43
+ message_progress: { current: null, total: null },
44
+ skippedDeletedTopics: 0,
45
+ skippedDeletedPosts: 0,
46
+ skippedDeletedMessages: 0,
47
+ skippedSystemMessages: 0,
48
+ skippedOrphanMessages: 0,
49
+ error: null,
50
+ },
51
+ };
52
+
53
+ const breakingSettings = [
54
+ 'maxDocuments',
55
+ 'rankingRules',
56
+ 'stopWords',
57
+ 'typoTolerance',
58
+ 'typoToleranceMinWordSizeOneTypo',
59
+ 'typoToleranceMinWordSizeTwoTypos',
60
+ 'typoToleranceDisableOnWords',
61
+ 'typoToleranceDisableOnAttributes',
62
+ 'synonyms',
63
+ 'semanticSearchEnabled',
64
+ 'semanticSearchProvider',
65
+ 'semanticSearchApiKey',
66
+ 'semanticSearchModel',
67
+ 'semanticSearchUrl',
68
+ 'semanticSearchDimensions',
69
+ 'semanticSearchRestRequest',
70
+ 'semanticSearchRestResponse',
71
+ ];
72
+
73
+ module.exports = { defaults, breakingSettings };