@jasonwch/nodebb-plugin-meilisearch-r 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -28
- package/lib/alerts.js +24 -0
- package/lib/chat-search-global.js +221 -0
- package/lib/client.js +445 -0
- package/lib/constants.js +42 -0
- package/lib/defaults.js +73 -0
- package/lib/embedder.js +297 -0
- package/lib/hooks/messages.js +49 -0
- package/lib/hooks/posts.js +180 -0
- package/lib/hooks/topics.js +210 -0
- package/lib/pending-queue.js +69 -0
- package/lib/reindex.js +311 -0
- package/lib/routes.js +129 -0
- package/lib/search.js +230 -0
- package/lib/settings-helpers.js +15 -0
- package/lib/settings.js +119 -0
- package/lib/state.js +69 -0
- package/library.js +17 -1555
- package/package.json +1 -1
- package/static/languages/en-GB/meilisearch.json +112 -79
- package/static/languages/he/meilisearch.json +113 -0
- package/static/languages/pl/meilisearch.json +34 -1
- package/static/languages/zh-CN/meilisearch.json +34 -1
- package/static/lib/admin.js +317 -2
- package/static/lib/client/quick-search.js +1 -1
- package/static/templates/admin/plugins/meilisearch.tpl +289 -222
package/lib/client.js
ADDED
|
@@ -0,0 +1,445 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const winston = nodebb.require('winston');
|
|
4
|
+
const settings = nodebb.require('./src/meta/settings');
|
|
5
|
+
const Topics = nodebb.require('./src/topics');
|
|
6
|
+
const { drainPending } = require('./pending-queue');
|
|
7
|
+
const {
|
|
8
|
+
EMBEDDER_NAME, buildEmbedderConfig, validateEmbedderConfig, deepEqual,
|
|
9
|
+
getAppliedConfig, getAppliedTaskUid, setAppliedConfigs, clearAppliedConfig,
|
|
10
|
+
} = require('./embedder');
|
|
11
|
+
const {
|
|
12
|
+
EMBEDDER_TASK_TIMEOUT_MS, EMBEDDER_TASK_BG_TIMEOUT_MS, EMBEDDER_TASK_BG_INTERVAL_MS,
|
|
13
|
+
MEILI_HTTP_TIMEOUT_MS,
|
|
14
|
+
} = require('./constants');
|
|
15
|
+
|
|
16
|
+
async function ensureIndex(plugin, uid, primaryKey) {
|
|
17
|
+
try {
|
|
18
|
+
await plugin.client.getIndex(uid);
|
|
19
|
+
} catch (e) {
|
|
20
|
+
await plugin.client.createIndex(uid, { primaryKey });
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// Meilisearch settings/document mutations are enqueued tasks - the initial call resolving
|
|
25
|
+
// just means Meilisearch accepted the request, not that it succeeded. waitForTask() itself
|
|
26
|
+
// only ever throws on ITS OWN polling timeout (confirmed against the installed
|
|
27
|
+
// meilisearch-js's TaskClient#waitForTask, node_modules/meilisearch/dist/index.js): a task
|
|
28
|
+
// that fails on the server resolves normally with status "failed" and an `error` object, so
|
|
29
|
+
// callers that don't check `.status` (as every waitForTask call in this plugin used to)
|
|
30
|
+
// silently treat a failed settings/document change as if it had gone through.
|
|
31
|
+
async function waitForSucceededTask(plugin, taskUidOrTask, options) {
|
|
32
|
+
const task = await plugin.client.tasks.waitForTask(taskUidOrTask, options);
|
|
33
|
+
if (task.status !== 'succeeded') {
|
|
34
|
+
throw new Error(task.error?.message || `Meilisearch task ${task.status}`);
|
|
35
|
+
}
|
|
36
|
+
return task;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// Module-level tracker for in-flight embedder background pollers. Keyed by taskUid so the
|
|
40
|
+
// same task can't be polled twice (e.g. if updateEmbedders is somehow re-entered during
|
|
41
|
+
// the 60s sync wait). Cleared on terminal status (success or failure) by the poller itself.
|
|
42
|
+
const embedderPollers = new Set();
|
|
43
|
+
|
|
44
|
+
// Background-verify an embedder task that exceeded the sync wait window. Per the Meilisearch
|
|
45
|
+
// SDK (node_modules/meilisearch/dist/index.js:152), a poll timeout throws
|
|
46
|
+
// MeilisearchTaskTimeOutError — task is still running server-side. Re-embedding every
|
|
47
|
+
// document legitimately exceeds 60s on a forum of any size; the task may yet succeed.
|
|
48
|
+
// Strategy: poll at a longer interval with a longer timeout; alert only on delayed failure.
|
|
49
|
+
// On delayed success: update the persisted taskUid to null (sync-verified) so reattachPollers
|
|
50
|
+
// skips it on future restarts instead of hitting a 404 when Meilisearch purges task history.
|
|
51
|
+
// On delayed failure: revert the optimistic fingerprint via clearAppliedConfig so the next
|
|
52
|
+
// save pushes fresh config, alert admins, flip plugin.healthy=false to defer subsequent
|
|
53
|
+
// writes to the pending queue.
|
|
54
|
+
// On poll timeout (MeilisearchTaskTimeOutError): task is STILL RUNNING — do NOT clear the
|
|
55
|
+
// fingerprint (would re-open the double-paid-API-cost loop). Log and let the task continue
|
|
56
|
+
// server-side; if it ultimately fails, the next restart's reattachPollers will catch it.
|
|
57
|
+
// On network error: task status is genuinely unknown — log warning but do NOT clear, since
|
|
58
|
+
// clearing on a transient blip would cause an unnecessary re-push. reattachPollers catches
|
|
59
|
+
// actual failures on the next restart.
|
|
60
|
+
function startEmbedderPoller(plugin, plan, taskUid) {
|
|
61
|
+
if (embedderPollers.has(taskUid)) return;
|
|
62
|
+
embedderPollers.add(taskUid);
|
|
63
|
+
const startTime = Date.now();
|
|
64
|
+
const poll = async () => {
|
|
65
|
+
try {
|
|
66
|
+
const task = await plugin.client.tasks.waitForTask(taskUid, {
|
|
67
|
+
timeout: EMBEDDER_TASK_BG_TIMEOUT_MS,
|
|
68
|
+
interval: EMBEDDER_TASK_BG_INTERVAL_MS,
|
|
69
|
+
});
|
|
70
|
+
embedderPollers.delete(taskUid);
|
|
71
|
+
if (task.status !== 'succeeded') {
|
|
72
|
+
// Delayed failure: fingerprint was optimistically persisted at enqueue time
|
|
73
|
+
// (see plugin.updateEmbedders catch-block). Clear it so the next save pushes
|
|
74
|
+
// fresh config rather than silently skipping as "already applied".
|
|
75
|
+
await clearAppliedConfig(plugin, plan.indexName);
|
|
76
|
+
plugin.healthy = false;
|
|
77
|
+
plugin.notifyAdmins(`embedder:${plan.indexName}`, {
|
|
78
|
+
type: 'danger',
|
|
79
|
+
titleKey: '[[meilisearch:admin.semanticEmbedderDelayedFailed]]',
|
|
80
|
+
message: `${plan.indexName}: ${task.error?.message || `task ${task.status}`}`,
|
|
81
|
+
});
|
|
82
|
+
} else {
|
|
83
|
+
// Delayed success: mark taskUid as null (sync-verified) so reattachPollers
|
|
84
|
+
// skips this index on future restarts instead of hitting a 404 when Meilisearch
|
|
85
|
+
// purges task history (~24h). Fingerprint config is already correct — no change.
|
|
86
|
+
winston.info(`[plugin/meilisearch] Embedder task for "${plan.indexName}" succeeded after ${Date.now() - startTime}ms (background-poll)`);
|
|
87
|
+
await setAppliedConfigs(plugin, [{ indexName: plan.indexName, config: plan.config, taskUid: null }]);
|
|
88
|
+
}
|
|
89
|
+
} catch (err) {
|
|
90
|
+
embedderPollers.delete(taskUid);
|
|
91
|
+
if (err.name === 'MeilisearchTaskTimeOutError') {
|
|
92
|
+
// Poller's own 30-min timeout fired — task is STILL RUNNING server-side.
|
|
93
|
+
// Do NOT clear the fingerprint: that would make the next save re-push identical
|
|
94
|
+
// config, re-opening the double-paid-API-cost loop this whole mechanism prevents.
|
|
95
|
+
// The task will eventually succeed or fail; if NodeBB restarts before that,
|
|
96
|
+
// reattachPollers will pick it up via the persisted taskUid.
|
|
97
|
+
winston.warn(`[plugin/meilisearch] Embedder task ${taskUid} for "${plan.indexName}" still running after ${EMBEDDER_TASK_BG_TIMEOUT_MS}ms; giving up poll. Task continues server-side — reattachPollers will reconcile on next restart.`);
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
// Network error or other non-timeout failure — task status is genuinely unknown.
|
|
101
|
+
// Don't clear the fingerprint on a transient blip (would cause unnecessary re-push).
|
|
102
|
+
// If the task actually failed, reattachPollers will catch it on the next restart
|
|
103
|
+
// via the persisted taskUid → getTask(uid) → status: "failed" → clearAppliedConfig.
|
|
104
|
+
winston.warn(`[plugin/meilisearch] Embedder poller for "${plan.indexName}" (task ${taskUid}) hit an error (non-fatal, fingerprint preserved): ${err.message}`);
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
poll().catch(err => winston.error(`[plugin/meilisearch] embedder poller crashed: ${err.message}`));
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// Recover embedder pollers after process restart. embedderPollers (line 42) is module-level
|
|
111
|
+
// in-memory; a NodeBB restart/crash/container-redeploy during the 30-min background-poller
|
|
112
|
+
// window (EMBEDDER_TASK_BG_TIMEOUT_MS) wipes it. This sweep reads the persisted taskUid
|
|
113
|
+
// (stored alongside each fingerprint in appliedEmbedders via setAppliedConfigs) and
|
|
114
|
+
// reconciles directly via plugin.client.tasks.getTask(uid):
|
|
115
|
+
// succeeded → no-op (fingerprint correct, taskUid was null from sync-success path)
|
|
116
|
+
// enqueued/processing → reattach startEmbedderPoller (task still running server-side)
|
|
117
|
+
// failed → clearAppliedConfig + alert (Scenario A: task failed during downtime)
|
|
118
|
+
// 404/network error → assume success, log warning (self-heals on next config change)
|
|
119
|
+
// For old-format fingerprints (no taskUid), falls through to query-based sweep as fallback.
|
|
120
|
+
async function reattachPollers(plugin) {
|
|
121
|
+
if (!plugin.client) return;
|
|
122
|
+
const ourIndexes = ['post', 'topic', 'chat_message'];
|
|
123
|
+
// Phase 1: Direct reconciliation via persisted taskUid (new-format fingerprints)
|
|
124
|
+
for (const indexName of ourIndexes) {
|
|
125
|
+
try {
|
|
126
|
+
const taskUid = await getAppliedTaskUid(plugin, indexName);
|
|
127
|
+
if (taskUid === undefined || taskUid === null) continue;
|
|
128
|
+
const task = await plugin.client.tasks.getTask(taskUid);
|
|
129
|
+
if (task.status === 'succeeded') continue;
|
|
130
|
+
if (task.status === 'enqueued' || task.status === 'processing') {
|
|
131
|
+
if (embedderPollers.has(taskUid)) continue;
|
|
132
|
+
const config = await buildEmbedderConfig(plugin, null, indexName);
|
|
133
|
+
winston.info(`[plugin/meilisearch] Reattaching embedder poller for "${indexName}" (task ${taskUid} still ${task.status})`);
|
|
134
|
+
startEmbedderPoller(plugin, { indexName, config }, taskUid);
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
// Task failed during downtime — clear stale fingerprint + alert
|
|
138
|
+
winston.warn(`[plugin/meilisearch] Embedder task ${taskUid} for "${indexName}" failed during downtime: ${task.error?.message || task.status}`);
|
|
139
|
+
await clearAppliedConfig(plugin, indexName);
|
|
140
|
+
plugin.healthy = false;
|
|
141
|
+
plugin.notifyAdmins(`embedder:${indexName}`, {
|
|
142
|
+
type: 'danger',
|
|
143
|
+
titleKey: '[[meilisearch:admin.semanticEmbedderDelayedFailed]]',
|
|
144
|
+
message: `${indexName}: ${task.error?.message || `task ${task.status}`}`,
|
|
145
|
+
});
|
|
146
|
+
} catch (err) {
|
|
147
|
+
// Non-fatal: taskUid may be 404 (purged from Meilisearch's ~24h history), or
|
|
148
|
+
// Meilisearch may be momentarily unreachable. Assume success — self-heals on next
|
|
149
|
+
// legitimate semantic-key change.
|
|
150
|
+
winston.warn(`[plugin/meilisearch] reattachPollers: could not reconcile taskUid for "${indexName}" (non-fatal): ${err.message}`);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
// Phase 2: Query-based fallback for old-format fingerprints (no taskUid persisted).
|
|
154
|
+
// Catches in-flight embedder tasks that have a persisted fingerprint but no taskUid
|
|
155
|
+
// (pre-Option-1 format). Will be a no-op once all fingerprints are migrated to new format.
|
|
156
|
+
try {
|
|
157
|
+
const tasksResp = await plugin.client.tasks.getTasks({
|
|
158
|
+
statuses: ['enqueued', 'processing'],
|
|
159
|
+
types: ['settingsUpdate'],
|
|
160
|
+
limit: 50,
|
|
161
|
+
});
|
|
162
|
+
const candidates = [];
|
|
163
|
+
for (const t of (tasksResp.results || [])) {
|
|
164
|
+
if (!ourIndexes.includes(t.indexUid)) continue;
|
|
165
|
+
if (embedderPollers.has(t.uid)) continue;
|
|
166
|
+
if (!(t.details && t.details.embedders)) continue;
|
|
167
|
+
const tu = await getAppliedTaskUid(plugin, t.indexUid);
|
|
168
|
+
if (tu !== undefined) continue; // new-format fingerprint — already reconciled in Phase 1
|
|
169
|
+
candidates.push(t);
|
|
170
|
+
}
|
|
171
|
+
if (!candidates.length) return;
|
|
172
|
+
winston.info(`[plugin/meilisearch] Query fallback: reattaching ${candidates.length} embedder poller(s) for old-format fingerprints`);
|
|
173
|
+
for (const task of candidates) {
|
|
174
|
+
const applied = await getAppliedConfig(plugin, task.indexUid);
|
|
175
|
+
if (applied === undefined) continue;
|
|
176
|
+
const config = await buildEmbedderConfig(plugin, null, task.indexUid);
|
|
177
|
+
startEmbedderPoller(plugin, { indexName: task.indexUid, config }, task.uid);
|
|
178
|
+
}
|
|
179
|
+
} catch (err) {
|
|
180
|
+
winston.warn(`[plugin/meilisearch] reattachPollers query fallback failed (non-fatal): ${err.message}`);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// MeiliSearch connection lifecycle: connecting, health checks, and index/settings sync.
|
|
185
|
+
module.exports = function attachClient(plugin) {
|
|
186
|
+
plugin.patchTopicsSearch = function () {
|
|
187
|
+
if (Topics.__meiliOriginalSearch) { return; }
|
|
188
|
+
Topics.__meiliOriginalSearch = Topics.search;
|
|
189
|
+
Topics.search = async function (tid, term) {
|
|
190
|
+
if (!tid || !term || !String(term).trim()) { return []; }
|
|
191
|
+
return Topics.__meiliOriginalSearch.call(Topics, tid, term);
|
|
192
|
+
};
|
|
193
|
+
};
|
|
194
|
+
|
|
195
|
+
plugin.checkHealth = async function () {
|
|
196
|
+
try {
|
|
197
|
+
plugin.healthy = await plugin.client.isHealthy();
|
|
198
|
+
if (!plugin.healthy) {
|
|
199
|
+
winston.warn('[plugin/meilisearch] MeiliSearch host is unhealthy');
|
|
200
|
+
} else {
|
|
201
|
+
drainPending(plugin).catch(err => winston.error(`[plugin/meilisearch] drain failed: ${err.message}`));
|
|
202
|
+
}
|
|
203
|
+
return plugin.healthy;
|
|
204
|
+
} catch (err) {
|
|
205
|
+
plugin.healthy = false;
|
|
206
|
+
winston.warn(`[plugin/meilisearch] ${err.message}`);
|
|
207
|
+
return false;
|
|
208
|
+
}
|
|
209
|
+
};
|
|
210
|
+
|
|
211
|
+
// Ensure the chat_message index + filterable attributes exist, independent of reindex state.
|
|
212
|
+
// prepareSearch() returns early when `indexed=true`, so updateIndexSettings (which configures
|
|
213
|
+
// chat_message) is skipped on normal restarts — but indexMessage auto-creates the index via
|
|
214
|
+
// document add, leaving filterableAttributes empty. Without roomId/uid filterable, searchMessages
|
|
215
|
+
// would error on its filter clause. Idempotent + cheap (2 calls, settings deduped by Meilisearch).
|
|
216
|
+
plugin.ensureChatMessageIndex = async function () {
|
|
217
|
+
if (!plugin.client) return;
|
|
218
|
+
try {
|
|
219
|
+
await ensureIndex(plugin, 'chat_message', 'mid');
|
|
220
|
+
await plugin.client.index('chat_message').updateFilterableAttributes(['roomId', 'uid', 'timestamp']);
|
|
221
|
+
} catch (err) {
|
|
222
|
+
winston.error(`[plugin/meilisearch] ensureChatMessageIndex failed: ${err.message}`);
|
|
223
|
+
plugin.healthy = false;
|
|
224
|
+
}
|
|
225
|
+
};
|
|
226
|
+
|
|
227
|
+
plugin.prepareSearch = async function (data, connectionChanged = false) {
|
|
228
|
+
winston.debug(`[plugin/meilisearch] Connecting to MeiliSearch host: ${data?.host || await settings.getOne(plugin.id, 'host')}`);
|
|
229
|
+
const { Meilisearch } = await import('meilisearch');
|
|
230
|
+
plugin.client = new Meilisearch({
|
|
231
|
+
host: data?.host || await settings.getOne(plugin.id, 'host'),
|
|
232
|
+
apiKey: data?.apiKey || await settings.getOne(plugin.id, 'apiKey') || undefined,
|
|
233
|
+
timeout: MEILI_HTTP_TIMEOUT_MS,
|
|
234
|
+
});
|
|
235
|
+
if (plugin.healthCheckTask) clearInterval(plugin.healthCheckTask);
|
|
236
|
+
const intervalRaw = parseInt(data?.healthCheckInterval || await settings.getOne(plugin.id, 'healthCheckInterval'), 10);
|
|
237
|
+
const intervalSeconds = Number.isFinite(intervalRaw) && intervalRaw > 0
|
|
238
|
+
? Math.min(Math.max(intervalRaw, 10), 3600)
|
|
239
|
+
: 60;
|
|
240
|
+
plugin.healthCheckTask = setInterval(plugin.checkHealth, intervalSeconds * 1000);
|
|
241
|
+
// #15: Always sync plugin.healthy on (re)connect — otherwise it stays false from
|
|
242
|
+
// initialization until the first setInterval tick, causing stale "MeiliSearch
|
|
243
|
+
// Unreachable" warnings on save during the startup window (MS online, flag stale).
|
|
244
|
+
await plugin.checkHealth();
|
|
245
|
+
// G1: Recover in-flight embedder pollers after process restart (runs once on startup,
|
|
246
|
+
// not on every health-check tick — the sweep is unnecessary after the first run since
|
|
247
|
+
// embedderPollers Set is populated and deduplicates subsequent calls).
|
|
248
|
+
reattachPollers(plugin).catch(err => winston.warn(`[plugin/meilisearch] reattachPollers failed: ${err.message}`));
|
|
249
|
+
// #14: Don't auto-reindex after a failed reindex unless connection settings changed.
|
|
250
|
+
const indexed = await settings.getOne(plugin.id, 'indexed');
|
|
251
|
+
if (indexed) return;
|
|
252
|
+
if (!plugin.healthy || plugin.initializingOnAnotherInstance) return;
|
|
253
|
+
const lastResult = await settings.getOne(plugin.id, 'lastReindexResult') || {};
|
|
254
|
+
const allowAutoReindex = connectionChanged || lastResult.success !== false;
|
|
255
|
+
if (!allowAutoReindex) {
|
|
256
|
+
winston.warn('[plugin/meilisearch] Skipping auto-reindex: last reindex failed. Trigger manually from ACP.');
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
await plugin.updateIndexSettings();
|
|
260
|
+
await plugin.updateEmbedders();
|
|
261
|
+
await plugin.reindex(false);
|
|
262
|
+
};
|
|
263
|
+
|
|
264
|
+
// Keyword-search index settings only (ranking rules, stop words, typo tolerance,
|
|
265
|
+
// synonyms, pagination). Deliberately does NOT touch embedders - see updateEmbedders
|
|
266
|
+
// below for why that has to stay a separate, independently-gated call.
|
|
267
|
+
plugin.updateIndexSettings = async (data) => {
|
|
268
|
+
await ensureIndex(plugin, 'post', 'pid');
|
|
269
|
+
await ensureIndex(plugin, 'topic', 'tid');
|
|
270
|
+
await ensureIndex(plugin, 'chat_message', 'mid');
|
|
271
|
+
data = {
|
|
272
|
+
maxDocuments: parseInt(data?.maxDocuments || await settings.getOne(plugin.id, 'maxDocuments') || 500, 10),
|
|
273
|
+
rankingRules: (data?.rankingRules || await settings.getOne(plugin.id, 'rankingRules'))?.map(value => value.rule),
|
|
274
|
+
stopWords: (data?.stopWords || await settings.getOne(plugin.id, 'stopWords'))?.map(value => value.word),
|
|
275
|
+
typoTolerance: ['on', true].includes(
|
|
276
|
+
data?.typoTolerance || await settings.getOne(plugin.id, 'typoTolerance') || undefined,
|
|
277
|
+
),
|
|
278
|
+
typoToleranceMinWordSizeOneTypo: parseInt(
|
|
279
|
+
data?.typoToleranceMinWordSizeOneTypo ||
|
|
280
|
+
await settings.getOne(plugin.id, 'typoToleranceMinWordSizeOneTypo') || 5,
|
|
281
|
+
10,
|
|
282
|
+
),
|
|
283
|
+
typoToleranceMinWordSizeTwoTypos: parseInt(
|
|
284
|
+
data?.typoToleranceMinWordSizeTwoTypos ||
|
|
285
|
+
await settings.getOne(plugin.id, 'typoToleranceMinWordSizeTwoTypos') || 9,
|
|
286
|
+
10,
|
|
287
|
+
),
|
|
288
|
+
typoToleranceDisableOnWords:
|
|
289
|
+
(data?.typoToleranceDisableOnWords || await settings.getOne(plugin.id, 'typoToleranceDisableOnWords') ||
|
|
290
|
+
undefined)?.map(value => value.word),
|
|
291
|
+
synonyms: Object.fromEntries(
|
|
292
|
+
(data?.synonyms || await settings.getOne(plugin.id, 'synonyms') || [])?.map((
|
|
293
|
+
{ word, synonyms },
|
|
294
|
+
) => [word, synonyms?.split(',').map(synonym => synonym.trim())]),
|
|
295
|
+
),
|
|
296
|
+
};
|
|
297
|
+
const postTask = await plugin.client.index('post').updateSettings({
|
|
298
|
+
filterableAttributes: ['tid', 'cid', 'uid', 'timestamp'],
|
|
299
|
+
sortableAttributes: ['timestamp', 'cid'],
|
|
300
|
+
searchableAttributes: ['content'],
|
|
301
|
+
pagination: {
|
|
302
|
+
maxTotalHits: data.maxDocuments,
|
|
303
|
+
},
|
|
304
|
+
rankingRules: data.rankingRules,
|
|
305
|
+
stopWords: data.stopWords,
|
|
306
|
+
typoTolerance: {
|
|
307
|
+
enabled: data.typoTolerance,
|
|
308
|
+
minWordSizeForTypos: {
|
|
309
|
+
oneTypo: data.typoToleranceMinWordSizeOneTypo,
|
|
310
|
+
twoTypos: data.typoToleranceMinWordSizeTwoTypos,
|
|
311
|
+
},
|
|
312
|
+
disableOnWords: data.typoToleranceDisableOnWords,
|
|
313
|
+
},
|
|
314
|
+
synonyms: data.synonyms,
|
|
315
|
+
});
|
|
316
|
+
const topicTask = await plugin.client.index('topic').updateSettings({
|
|
317
|
+
filterableAttributes: ['cid', 'uid', 'timestamp'],
|
|
318
|
+
sortableAttributes: ['cid', 'title', 'timestamp'],
|
|
319
|
+
searchableAttributes: ['title'],
|
|
320
|
+
pagination: {
|
|
321
|
+
maxTotalHits: data.maxDocuments,
|
|
322
|
+
},
|
|
323
|
+
rankingRules: data.rankingRules,
|
|
324
|
+
stopWords: data.stopWords,
|
|
325
|
+
typoTolerance: {
|
|
326
|
+
enabled: data.typoTolerance,
|
|
327
|
+
minWordSizeForTypos: {
|
|
328
|
+
oneTypo: data.typoToleranceMinWordSizeOneTypo,
|
|
329
|
+
twoTypos: data.typoToleranceMinWordSizeTwoTypos,
|
|
330
|
+
},
|
|
331
|
+
disableOnWords: data.typoToleranceDisableOnWords,
|
|
332
|
+
},
|
|
333
|
+
synonyms: data.synonyms,
|
|
334
|
+
});
|
|
335
|
+
const messageTask = await plugin.client.index('chat_message').updateSettings({
|
|
336
|
+
filterableAttributes: ['roomId', 'uid', 'timestamp'],
|
|
337
|
+
sortableAttributes: ['timestamp'],
|
|
338
|
+
searchableAttributes: ['content'],
|
|
339
|
+
pagination: {
|
|
340
|
+
maxTotalHits: data.maxDocuments,
|
|
341
|
+
},
|
|
342
|
+
rankingRules: data.rankingRules,
|
|
343
|
+
stopWords: data.stopWords,
|
|
344
|
+
typoTolerance: {
|
|
345
|
+
enabled: data.typoTolerance,
|
|
346
|
+
minWordSizeForTypos: {
|
|
347
|
+
oneTypo: data.typoToleranceMinWordSizeOneTypo,
|
|
348
|
+
twoTypos: data.typoToleranceMinWordSizeTwoTypos,
|
|
349
|
+
},
|
|
350
|
+
disableOnWords: data.typoToleranceDisableOnWords,
|
|
351
|
+
},
|
|
352
|
+
synonyms: data.synonyms,
|
|
353
|
+
});
|
|
354
|
+
return [postTask, topicTask, messageTask];
|
|
355
|
+
};
|
|
356
|
+
|
|
357
|
+
// Pushes (or removes) the "default" embedder on all three indexes based on the
|
|
358
|
+
// semantic search ACP settings. Meilisearch re-embeds every existing document
|
|
359
|
+
// through the (often paid) embedding API whenever an index's embedder config
|
|
360
|
+
// actually changes, so this is intentionally its own call rather than something
|
|
361
|
+
// updateIndexSettings does automatically - it must only run when semantic settings
|
|
362
|
+
// were deliberately changed (see lib/settings.js), or as part of an explicit
|
|
363
|
+
// reindex/connect flow, never as a side effect of unrelated settings (ranking rules,
|
|
364
|
+
// stop words, ...) being saved. Skips the network call entirely when the resolved
|
|
365
|
+
// config is identical to what was last pushed, as a second line of defense against
|
|
366
|
+
// needless re-embedding regardless of caller. Older/self-hosted Meilisearch instances
|
|
367
|
+
// without embedder support will reject the call; that's caught here so it never
|
|
368
|
+
// blocks the rest of the settings-save flow.
|
|
369
|
+
plugin.updateEmbedders = async (rawData) => {
|
|
370
|
+
// Pre-flight validation: fail fast with an admin-facing alert instead of letting
|
|
371
|
+
// Meilisearch reject the task server-side (which can take up to 60s on slow/unreachable
|
|
372
|
+
// embedding endpoints, leaving the ACP Save button stuck). Client-side validation in
|
|
373
|
+
// static/lib/admin.js is the primary UX path; this is defense-in-depth for reindex,
|
|
374
|
+
// prepareSearch-on-host-change, and any programmatic caller.
|
|
375
|
+
const validationError = await validateEmbedderConfig(plugin, rawData);
|
|
376
|
+
if (validationError) {
|
|
377
|
+
winston.warn(`[plugin/meilisearch] Embedder config validation failed: ${validationError}`);
|
|
378
|
+
plugin.notifyAdmins('embedder:validation', {
|
|
379
|
+
type: 'danger',
|
|
380
|
+
titleKey: '[[meilisearch:admin.semanticConfigInvalid]]',
|
|
381
|
+
message: validationError,
|
|
382
|
+
});
|
|
383
|
+
return; // no fingerprint persisted, no task enqueued, save proceeds immediately
|
|
384
|
+
}
|
|
385
|
+
const indexNames = ['post', 'topic', 'chat_message'];
|
|
386
|
+
// Resolve configs + diff-against-applied first (read-only), THEN push and persist -
|
|
387
|
+
// keeps the three indexes from racing on the same shared "applied" bookkeeping object
|
|
388
|
+
// (parallel read-modify-write on one settings key would let one index's write silently
|
|
389
|
+
// clobber another's).
|
|
390
|
+
const plans = await Promise.all(indexNames.map(async (indexName) => {
|
|
391
|
+
const config = await buildEmbedderConfig(plugin, rawData, indexName);
|
|
392
|
+
const applied = await getAppliedConfig(plugin, indexName);
|
|
393
|
+
return { indexName, config, needsPush: applied === undefined || !deepEqual(config, applied) };
|
|
394
|
+
}));
|
|
395
|
+
const results = await Promise.all(plans.map(async (plan) => {
|
|
396
|
+
if (!plan.needsPush) return null;
|
|
397
|
+
try {
|
|
398
|
+
// updateEmbedders() only enqueues a Meilisearch task - a bad URL/model/response
|
|
399
|
+
// shape for "rest"/"openAi"-with-custom-url configs only surfaces once Meilisearch
|
|
400
|
+
// actually runs the task (e.g. its probe call to auto-detect dimensions), so this
|
|
401
|
+
// must wait for and check the task's real outcome, not just the enqueue response.
|
|
402
|
+
const enqueued = await plugin.client.index(plan.indexName).updateEmbedders({ [EMBEDDER_NAME]: plan.config });
|
|
403
|
+
try {
|
|
404
|
+
await waitForSucceededTask(plugin, enqueued.taskUid, {
|
|
405
|
+
timeout: EMBEDDER_TASK_TIMEOUT_MS,
|
|
406
|
+
interval: 200,
|
|
407
|
+
});
|
|
408
|
+
} catch (err) {
|
|
409
|
+
// Distinguish "task is still running" (poll timeout) from "task definitively failed"
|
|
410
|
+
// (Meili rejected the call, 4xx, malformed config, network error). Meilisearch SDK
|
|
411
|
+
// throws MeilisearchTaskTimeOutError (name property, see node_modules/meilisearch/dist/index.js:152)
|
|
412
|
+
// on poll timeout; this means the task is still running server-side and may yet succeed.
|
|
413
|
+
if (err.name === 'MeilisearchTaskTimeOutError') {
|
|
414
|
+
// Re-embedding every document legitimately exceeds 60s on a forum of any size.
|
|
415
|
+
// Optimistically persist the fingerprint so the next ACP save doesn't re-push
|
|
416
|
+
// identical config (which would double paid-API cost — the exact thing we're
|
|
417
|
+
// preventing). startEmbedderPoller verifies the real outcome.
|
|
418
|
+
winston.warn(`[plugin/meilisearch] Embedder task for "${plan.indexName}" still running after ${EMBEDDER_TASK_TIMEOUT_MS}ms; background-polling.`);
|
|
419
|
+
startEmbedderPoller(plugin, plan, enqueued.taskUid);
|
|
420
|
+
return { ...plan, taskUid: enqueued.taskUid }; // optimistic — taskUid for later reconciliation
|
|
421
|
+
}
|
|
422
|
+
throw err; // real failure — outer catch handles alerting + return null
|
|
423
|
+
}
|
|
424
|
+
return { ...plan, taskUid: null }; // sync-verified success — no reconciliation needed
|
|
425
|
+
} catch (err) {
|
|
426
|
+
const reason = err.message;
|
|
427
|
+
winston.error(`[plugin/meilisearch] Failed to apply "${plan.indexName}" embedder (semantic search): ${reason}`);
|
|
428
|
+
plugin.notifyAdmins(`embedder:${plan.indexName}`, {
|
|
429
|
+
type: 'danger',
|
|
430
|
+
titleKey: '[[meilisearch:admin.semanticEmbedderFailed]]',
|
|
431
|
+
message: `${plan.indexName}: ${reason}`,
|
|
432
|
+
});
|
|
433
|
+
return null;
|
|
434
|
+
}
|
|
435
|
+
}));
|
|
436
|
+
const succeeded = results.filter(Boolean);
|
|
437
|
+
if (succeeded.length) {
|
|
438
|
+
await setAppliedConfigs(plugin, succeeded);
|
|
439
|
+
}
|
|
440
|
+
};
|
|
441
|
+
};
|
|
442
|
+
|
|
443
|
+
// Exposed so other call sites that enqueue Meilisearch tasks (lib/reindex.js) get the same
|
|
444
|
+
// "actually check the task succeeded" behavior instead of trusting waitForTask() to throw.
|
|
445
|
+
module.exports.waitForSucceededTask = waitForSucceededTask;
|
package/lib/constants.js
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
module.exports = {
|
|
4
|
+
REINDEX_BATCH_SIZE: 500,
|
|
5
|
+
PENDING_KEY: 'plugin:meilisearch:pending',
|
|
6
|
+
PENDING_MAX: 100000,
|
|
7
|
+
DRAIN_MAX: 500,
|
|
8
|
+
REINDEX_LOCK_KEY: 'plugin:meilisearch:reindex:lock',
|
|
9
|
+
REINDEX_LOCK_TTL: 3600,
|
|
10
|
+
LOCK_REFRESH_INTERVAL: 5 * 60 * 1000,
|
|
11
|
+
REPLAY_OPS: [
|
|
12
|
+
'indexPost', 'deindexPost', 'indexTopic', 'deindexTopic', 'deindexPostsPurge',
|
|
13
|
+
'deindexTopicsPurge', 'reindexTopicPosts', 'deindexTopicPosts', 'restoreTopic',
|
|
14
|
+
'changePostOwner', 'changeTopicOwner', 'onTopicMerge', 'onScheduledPublish',
|
|
15
|
+
'indexMessage', 'deindexMessage',
|
|
16
|
+
],
|
|
17
|
+
GLOBAL_CHAT_SEARCH_LIMIT: 200,
|
|
18
|
+
GLOBAL_CHAT_SEARCH_FETCH: 300,
|
|
19
|
+
MEMBERSHIP_CHUNK_SIZE: 100,
|
|
20
|
+
DECORATION_BATCH_SIZE: 10,
|
|
21
|
+
// Sync wait window for plugin.updateEmbedders' Meilisearch task before falling back to
|
|
22
|
+
// background poller. 60s keeps ACP save responsive; if the embedder task is still
|
|
23
|
+
// running server-side (re-embedding every doc legitimately exceeds 60s on a forum of
|
|
24
|
+
// any size), updateEmbedders persists the fingerprint optimistically and startEmbedderPoller
|
|
25
|
+
// verifies the eventual outcome.
|
|
26
|
+
EMBEDDER_TASK_TIMEOUT_MS: 60000,
|
|
27
|
+
// Background poller ceiling (60 min) and interval (5 s) for verifying embedder tasks
|
|
28
|
+
// that exceeded the sync wait window. 60 min covers re-embedding up to ~1M posts at
|
|
29
|
+
// OpenAI Tier 1 rate limits (1M TPM). For forums larger than that, the poller gives up
|
|
30
|
+
// and relies on reattachPollers to reconcile the task outcome on the next restart
|
|
31
|
+
// (within Meilisearch's ~24h task retention window). No double-paid-API-cost risk —
|
|
32
|
+
// fingerprint is preserved, not cleared, on poller timeout (see startEmbedderPoller
|
|
33
|
+
// MeilisearchTaskTimeOutError handling in lib/client.js).
|
|
34
|
+
EMBEDDER_TASK_BG_TIMEOUT_MS: 3600000,
|
|
35
|
+
EMBEDDER_TASK_BG_INTERVAL_MS: 5000,
|
|
36
|
+
// Meilisearch SDK client-side HTTP timeout. Caps every fetch() the SDK makes (including
|
|
37
|
+
// the initial enqueue POST for embedder tasks) at 30s — without this, the SDK uses no
|
|
38
|
+
// AbortSignal (node_modules/meilisearch/dist/index.js:340), so an unreachable Meilisearch
|
|
39
|
+
// host can leave plugin.updateEmbedders (and therefore the ACP Save button) stuck
|
|
40
|
+
// indefinitely.
|
|
41
|
+
MEILI_HTTP_TIMEOUT_MS: 30000,
|
|
42
|
+
};
|
package/lib/defaults.js
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const defaults = {
|
|
4
|
+
host: 'http://localhost:7700',
|
|
5
|
+
apiKey: undefined,
|
|
6
|
+
maxDocuments: undefined,
|
|
7
|
+
indexed: false,
|
|
8
|
+
rankingRules: [
|
|
9
|
+
{ rule: 'words' },
|
|
10
|
+
{ rule: 'typo' },
|
|
11
|
+
{ rule: 'proximity' },
|
|
12
|
+
{ rule: 'attribute' },
|
|
13
|
+
{ rule: 'sort' },
|
|
14
|
+
{ rule: 'exactness' },
|
|
15
|
+
],
|
|
16
|
+
stopWords: [],
|
|
17
|
+
typoTolerance: 'on',
|
|
18
|
+
typoToleranceMinWordSizeOneTypo: 5,
|
|
19
|
+
typoToleranceMinWordSizeTwoTypos: 9,
|
|
20
|
+
typoToleranceDisableOnWords: [],
|
|
21
|
+
synonyms: [],
|
|
22
|
+
healthCheckInterval: 60,
|
|
23
|
+
searchMinTermLength: 2,
|
|
24
|
+
globalChatSearchEnabled: 'on',
|
|
25
|
+
semanticSearchEnabled: 'off',
|
|
26
|
+
semanticSearchProvider: 'openAi',
|
|
27
|
+
semanticSearchApiKey: undefined,
|
|
28
|
+
semanticSearchModel: undefined,
|
|
29
|
+
semanticSearchUrl: undefined,
|
|
30
|
+
semanticSearchDimensions: undefined,
|
|
31
|
+
semanticSearchRatio: 0.5,
|
|
32
|
+
semanticSearchScoreThreshold: 0.2,
|
|
33
|
+
semanticSearchRestRequest: undefined,
|
|
34
|
+
semanticSearchRestResponse: undefined,
|
|
35
|
+
// Internal bookkeeping (not an ACP field): last embedder config actually pushed to
|
|
36
|
+
// Meilisearch per index, so unrelated settings saves can't re-trigger paid re-embedding.
|
|
37
|
+
appliedEmbedders: {},
|
|
38
|
+
lastReindexResult: {
|
|
39
|
+
success: false,
|
|
40
|
+
finishedAt: null,
|
|
41
|
+
topic_progress: { current: null, total: null },
|
|
42
|
+
post_progress: { current: null, total: null },
|
|
43
|
+
message_progress: { current: null, total: null },
|
|
44
|
+
skippedDeletedTopics: 0,
|
|
45
|
+
skippedDeletedPosts: 0,
|
|
46
|
+
skippedDeletedMessages: 0,
|
|
47
|
+
skippedSystemMessages: 0,
|
|
48
|
+
skippedOrphanMessages: 0,
|
|
49
|
+
error: null,
|
|
50
|
+
},
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
const breakingSettings = [
|
|
54
|
+
'maxDocuments',
|
|
55
|
+
'rankingRules',
|
|
56
|
+
'stopWords',
|
|
57
|
+
'typoTolerance',
|
|
58
|
+
'typoToleranceMinWordSizeOneTypo',
|
|
59
|
+
'typoToleranceMinWordSizeTwoTypos',
|
|
60
|
+
'typoToleranceDisableOnWords',
|
|
61
|
+
'typoToleranceDisableOnAttributes',
|
|
62
|
+
'synonyms',
|
|
63
|
+
'semanticSearchEnabled',
|
|
64
|
+
'semanticSearchProvider',
|
|
65
|
+
'semanticSearchApiKey',
|
|
66
|
+
'semanticSearchModel',
|
|
67
|
+
'semanticSearchUrl',
|
|
68
|
+
'semanticSearchDimensions',
|
|
69
|
+
'semanticSearchRestRequest',
|
|
70
|
+
'semanticSearchRestResponse',
|
|
71
|
+
];
|
|
72
|
+
|
|
73
|
+
module.exports = { defaults, breakingSettings };
|