@jasonwch/nodebb-plugin-meilisearch-r 1.1.2 → 1.2.1

package/lib/hooks/topics.js

@@ -0,0 +1,218 @@
+'use strict';
+
+const winston = nodebb.require('winston');
+const Topics = nodebb.require('./src/topics');
+const batch = nodebb.require('./src/batch');
+const { REINDEX_BATCH_SIZE } = require('../constants');
+const { maybeDefer, meiliWrite, enqueuePending } = require('../pending-queue');
+
+// action:topic.* hooks (post/save/edit/move/restore/delete/merge/changeOwner/purge)
+// plus scheduled-topic publish handling.
+module.exports = function attachTopicHooks(plugin) {
+	plugin.indexTopic = async function ({ topic, tid, fromCid, post }) {
+		const isMove = fromCid !== undefined;
+		if (await maybeDefer(plugin, 'indexTopic', { topic, tid, fromCid, post })) return;
+		// #2: On topic.move, no 'topic' key is provided — re-fetch by tid.
+		if (!topic && tid) {
+			topic = await Topics.getTopicData(tid);
+		}
+		if (!topic) {
+			winston.warn('[plugin/meilisearch] indexTopic: no topic data');
+			return;
+		}
+		// Issue 3: Skip deleted topics (admin editing/moving a deleted topic should not re-index it).
+		if (parseInt(topic.deleted, 10) === 1) {
+			winston.debug(`[plugin/meilisearch] Skipping deleted topic ${topic.tid}`);
+			return;
+		}
+		// Issue 5: Skip scheduled topics (timestamp in future) — indexed on publish via onScheduledPublish.
+		// action:topic.post fires AFTER action:post.save, so the main post may already be in Meili.
+		// Deindex it here to prevent full-search from finding scheduled content prematurely.
+		if (topic.scheduled || (topic.timestamp && topic.timestamp > Date.now())) {
+			const mainPid = post ? post.pid : (parseInt(topic.mainPid, 10) > 0 ? topic.mainPid : null);
+			if (mainPid) {
+				await plugin.client.index('post').deleteDocument(mainPid).catch(() => {});
+			}
+			winston.debug(`[plugin/meilisearch] Skipping scheduled topic ${topic.tid}, deindexed main post ${mainPid || 'N/A'}`);
+			return;
+		}
+		await meiliWrite(plugin, 'indexTopic', { topic, tid, fromCid, post }, () =>
+			plugin.client.index('topic').updateDocuments([{
+				tid: topic.tid,
+				cid: topic.cid,
+				uid: topic.uid,
+				mainPid: topic.mainPid,
+				title: topic.title,
+				timestamp: topic.timestamp,
+			}], { primaryKey: 'tid' }));
+		// #4: On topic move, re-index child posts' cids (fire-and-forget with fallback).
+		if (isMove && topic.tid) {
+			if (plugin.healthy) {
+				plugin.reindexTopicPosts({ tid: topic.tid }).catch(err =>
+					winston.error(`[plugin/meilisearch] reindexTopicPosts failed: ${err.message}`));
+			} else {
+				await enqueuePending('reindexTopicPosts', { tid: topic.tid });
+			}
+		}
+	};
+
+	plugin.deindexTopic = async function ({ topic, tid }) {
+		if (await maybeDefer(plugin, 'deindexTopic', { topic, tid })) return;
+		// Tolerate topic.move-style payloads (no 'topic' key) — extract tid.
+		const topicTid = topic ? topic.tid : tid;
+		if (!topicTid) {
+			winston.warn('[plugin/meilisearch] deindexTopic: no tid');
+			return;
+		}
+		await meiliWrite(plugin, 'deindexTopic', { topic, tid }, () =>
+			plugin.client.index('topic').deleteDocument(topicTid));
+		// Also deindex the topic's posts from the post index so admin /search
+		// (which queries both indexes and bypasses NodeBB's relevance-sort
+		// deleted-content filter for admins) can't surface deleted-topic content.
+		if (plugin.healthy) {
+			plugin.deindexTopicPosts({ tid: topicTid }).catch(err =>
+				winston.error(`[plugin/meilisearch] deindexTopicPosts failed: ${err.message}`));
+		} else {
+			await enqueuePending('deindexTopicPosts', { tid: topicTid });
+		}
+	};
+
+	plugin.restoreTopic = async function ({ topic, tid }) {
+		const topicTid = topic ? topic.tid : tid;
+		if (await maybeDefer(plugin, 'restoreTopic', { topic, tid })) return;
+		// Re-fetch topic if missing (defensive — same pattern as indexTopic).
+		if (!topic && topicTid) {
+			topic = await Topics.getTopicData(topicTid);
+		}
+		if (!topic) {
+			winston.warn('[plugin/meilisearch] restoreTopic: no topic data');
+			return;
+		}
+		// Re-index the topic document.
+		await meiliWrite(plugin, 'restoreTopic', { topic, tid }, () =>
+			plugin.client.index('topic').updateDocuments([{
+				tid: topic.tid,
+				cid: topic.cid,
+				uid: topic.uid,
+				mainPid: topic.mainPid,
+				title: topic.title,
+				timestamp: topic.timestamp,
+			}], { primaryKey: 'tid' }));
+		// Re-index the topic's posts (they were deindexed on delete).
+		if (plugin.healthy) {
+			plugin.reindexTopicPosts({ tid: topic.tid }).catch(err =>
+				winston.error(`[plugin/meilisearch] restoreTopic reindexTopicPosts failed: ${err.message}`));
+		} else {
+			await enqueuePending('reindexTopicPosts', { tid: topic.tid });
+		}
+	};
+
+	plugin.changeTopicOwner = async function ({ topics, toUid }) {
+		if (await maybeDefer(plugin, 'changeTopicOwner', { topics, toUid })) return;
+		// topics have OLD uid — re-fetch from DB to get fresh uid.
+		// Skip deleted or scheduled topics.
+		const validTids = (Array.isArray(topics) ? topics : [])
+			.filter(t => t && t.tid && parseInt(t.deleted, 10) !== 1)
+			.map(t => t.tid);
+		if (!validTids.length) return;
+		const freshTopics = await Topics.getTopicsFields(validTids, ['tid', 'cid', 'uid', 'mainPid', 'title', 'timestamp']);
+		// Fix 7: Skip scheduled topics (timestamp in future).
+		const activeTopics = freshTopics.filter(t => t && !(t.timestamp && t.timestamp > Date.now()));
+		if (!activeTopics.length) return;
+		const docs = activeTopics.map(topic => ({
+			tid: topic.tid,
+			cid: topic.cid,
+			uid: topic.uid,
+			mainPid: topic.mainPid,
+			title: topic.title,
+			timestamp: topic.timestamp,
+		}));
+		await meiliWrite(plugin, 'changeTopicOwner', { topics: activeTopics.map(t => ({ tid: t.tid })), toUid }, () =>
+			plugin.client.index('topic').updateDocuments(docs, { primaryKey: 'tid' }));
+	};
+
+	plugin.onTopicMerge = async function ({ otherTids, mergeIntoTid }) {
+		if (await maybeDefer(plugin, 'onTopicMerge', { otherTids, mergeIntoTid })) return;
+		const postIndex = plugin.client.index('post');
+		// Deindex merged-out topics + their posts (Topics.delete in merge fires no action:topic.delete).
+		for (const tid of otherTids) {
+			await plugin.deindexTopic({ tid });
+			// Merge zeros mainPid before Topics.delete, so deindexTopicPosts can't find the main post.
+			// Fallback: search Meili for any orphaned posts still carrying this tid.
+			try {
+				const result = await postIndex.search('', { filter: `tid = ${parseInt(tid, 10)}`, limit: 1000 });
+				if (result.hits && result.hits.length) {
+					const orphanPids = result.hits.map(h => h.pid);
+					await meiliWrite(plugin, 'deindexPostsPurge', { posts: orphanPids.map(pid => ({ pid })) }, () =>
+						postIndex.deleteDocuments(orphanPids));
+				}
+			} catch (err) {
+				// B2b: SDK write error path — may contain document content snippets from
+				// already-indexed posts (rare; only if Meilisearch rejects a batch quoting
+				// the bad doc). Accepted risk for low-frequency write failures; document
+				// content is already-in-DB data.
+				winston.error(`[plugin/meilisearch] onTopicMerge orphan cleanup for tid ${tid} failed: ${err.message}`);
+				plugin.healthy = false;
+				// Don't enqueue — orphans are cleaned by next reindex. Continue to next tid.
+			}
+		}
+		// Re-index the merge target (postcount/lastpost changed; posts already moved via action:post.move).
+		await plugin.indexTopic({ tid: mergeIntoTid });
+		if (plugin.healthy) {
+			plugin.reindexTopicPosts({ tid: mergeIntoTid }).catch(err =>
+				winston.error(`[plugin/meilisearch] onTopicMerge reindexTopicPosts failed: ${err.message}`));
+		}
+	};
+
+	plugin.onScheduledPublish = async function ({ topics }) {
+		const tids = (Array.isArray(topics) ? topics : []).filter(t => t && t.tid).map(t => t.tid);
+		if (!tids.length) return;
+		if (await maybeDefer(plugin, 'onScheduledPublish', { topics: tids.map(tid => ({ tid })) })) return;
+		// Re-index each published topic + its posts (DB deleted=0, timestamp in past now).
+		for (const tid of tids) {
+			await plugin.indexTopic({ tid });
+			if (plugin.healthy) {
+				plugin.reindexTopicPosts({ tid }).catch(err =>
+					winston.error(`[plugin/meilisearch] onScheduledPublish reindexTopicPosts failed: ${err.message}`));
+			}
+		}
+	};
+
+	plugin.deindexTopicsPurge = async function ({ topics }) {
+		if (await maybeDefer(plugin, 'deindexTopicsPurge', { topics })) return;
+		const topicsArr = Array.isArray(topics) ? topics.filter(t => t && t.tid) : [];
+		if (!topicsArr.length) {
+			return;
+		}
+		const tids = topicsArr.map(t => t.tid);
+		const mainPids = topicsArr.map(t => t.mainPid).filter(Boolean);
+		const postIndex = plugin.client.index('post');
+		const reducedPayload = { topics: topicsArr.map(t => ({ tid: t.tid, mainPid: t.mainPid })) };
+		try {
+			await Promise.all([
+				plugin.client.index('topic').deleteDocuments(tids),
+				mainPids.length ? postIndex.deleteDocuments(mainPids) : Promise.resolve(),
+			]);
+			await Promise.all(topicsArr.map(async (t) => {
+				await batch.processSortedSet(
+					`tid:${t.tid}:posts`,
+					async (pids) => {
+						if (!pids.length) {
+							return;
+						}
+						await postIndex.deleteDocuments(pids);
+					},
+					{ batch: REINDEX_BATCH_SIZE },
+				);
+			}));
+		} catch (err) {
+			// B2b: SDK write error path — may contain document content snippets from
+			// already-indexed posts (rare; only if Meilisearch rejects a batch quoting
+			// the bad doc). Accepted risk for low-frequency write failures; document
+			// content is already-in-DB data.
+			winston.error(`[plugin/meilisearch] deindexTopicsPurge failed: ${err.message}`);
+			plugin.healthy = false;
+			await enqueuePending('deindexTopicsPurge', reducedPayload);
+		}
+	};
+};

package/lib/pending-queue.js

@@ -0,0 +1,69 @@
+'use strict';
+
+const winston = nodebb.require('winston');
+const db = nodebb.require('./src/database');
+const { PENDING_KEY, PENDING_MAX, DRAIN_MAX, REPLAY_OPS } = require('./constants');
+
+// Persists a write that couldn't be applied to MeiliSearch yet, so it can be replayed later.
+async function enqueuePending(op, payload) {
+	const count = await db.sortedSetCard(PENDING_KEY);
+	if (count >= PENDING_MAX) {
+		const oldest = await db.getSortedSetRange(PENDING_KEY, 0, 0);
+		if (oldest.length) await db.sortedSetRemove(PENDING_KEY, ...oldest);
+	}
+	await db.sortedSetAdd(PENDING_KEY, Date.now(), JSON.stringify({ op, payload }));
+}
+
+// Decides whether a write hook should be deferred instead of applied immediately:
+// buffered in memory during a forced reindex, or queued to the DB while MeiliSearch is unhealthy.
+async function maybeDefer(plugin, op, payload) {
+	if (plugin.reindexingForced) {
+		if (plugin.pendingDuringReindex.length >= PENDING_MAX) {
+			plugin.pendingDuringReindex.shift();
+		}
+		plugin.pendingDuringReindex.push({ op, payload });
+		return true;
+	}
+	if (!plugin.healthy && !(await plugin.checkHealth())) {
+		await enqueuePending(op, payload);
+		return true;
+	}
+	return false;
+}
+
+// Runs a MeiliSearch write, falling back to the pending queue on failure.
+async function meiliWrite(plugin, op, payload, fn) {
+	try {
+		await fn();
+	} catch (err) {
+		winston.error(`[plugin/meilisearch] ${op} write failed: ${err.message}`);
+		plugin.healthy = false;
+		await enqueuePending(op, payload);
+	}
+}
+
+// Replays queued writes once MeiliSearch is healthy again.
+async function drainPending(plugin) {
+	if (!plugin.healthy || plugin.reindexingForced) return;
+	let drained = 0;
+	for (;;) {
+		if (!plugin.healthy) break;
+		const members = await db.getSortedSetRange(PENDING_KEY, 0, 49);
+		if (!members.length) break;
+		await db.sortedSetRemove(PENDING_KEY, ...members);
+		for (const member of members) {
+			try {
+				const { op, payload } = JSON.parse(member);
+				if (op && REPLAY_OPS.includes(op)) {
+					await plugin[op](payload);
+				}
+			} catch (err) {
+				winston.error(`[plugin/meilisearch] replay failed: ${err.message}`);
+			}
+			drained += 1;
+		}
+		if (drained >= DRAIN_MAX) break;
+	}
+}
+
+module.exports = { enqueuePending, maybeDefer, meiliWrite, drainPending };

package/lib/redact.js

@@ -0,0 +1,131 @@
+'use strict';
+
+// Credential redaction for log/alert paths.
+//
+// Background: M5 + C1 fix. Meilisearch task error messages and settings diff logs may
+// echo the configured apiKey (e.g. "Unauthorized: key sk-abc123" or nested in JSON).
+// Without redaction, these flow to:
+//   - winston transports (stdout, logs/output.log, /admin/developer/logs)
+//   - log aggregators (Loki/ELK/Datadog) which sit OUTSIDE the admin trust boundary
+//   - admin ACP toasts (15s transient — already mitigated by Issue 2 admin-room filter)
+//
+// Strategy: dual-layer redaction
+//   1. Pattern-based regex: catches known API key shapes (OpenAI sk-*, Bearer tokens)
+//      even when we can't read the configured value (e.g. message logged before init)
+//   2. Value-based substitution: reads semanticSearchApiKey from settings and string-
+//      replaces any literal occurrence — catches custom REST tokens, future OpenAI
+//      key formats, and exotic auth schemes that don't match the regex.
+//
+// Value-based redaction requires a settings.getOne DB read on first call; cached for
+// 5 min so throttled error paths (notifyAdmins throttles 60s/key) incur ≤1 DB read/min.
+
+const settings = nodebb.require('./src/meta/settings');
+
+// Fields whose VALUES must never appear in winston diff logs (lib/settings.js isBreaking).
+// Listed by ACP field name. semanticSearchUrl is intentionally NOT included — it's an
+// internal hostname, not a credential, and admins need it for debugging connectivity.
+const SECRET_SETTINGS = new Set([
+	'semanticSearchApiKey',
+	'semanticSearchRestRequest',
+	'semanticSearchRestResponse',
+]);
+
+const REDACTION_PLACEHOLDER = '<redacted>';
+
+// Patterns to catch common API key formats in free-form error strings.
+// OpenAI keys: sk-*, sk-proj-*, sk-org-*, sk-svcacct-* — all start with "sk-"
+// and are at least 20 chars long (real keys are 40+ but we redact aggressively at 20).
+const KEY_PATTERNS = [
+	/\bsk-[a-zA-Z0-9_-]{20,}\b/g,
+	/\bBearer\s+[a-zA-Z0-9_-]{8,}\b/gi,
+];
+
+// Min length for value-based redaction — avoids replacing short strings that might
+// legitimately appear in error messages (e.g. an apiKey of "ab" would corrupt many
+// error strings). 8 chars is safely above any plausible short substring.
+const MIN_VALUE_REDACT_LENGTH = 8;
+
+// Cached configured semanticSearchApiKey to avoid hitting DB on every redact call.
+// TTL ensures we pick up rotations within 5 min. Cached as the raw string value
+// (or '' when no key configured) so we can distinguish "cached empty" from "not cached".
+let _cachedApiKey = null;
+let _cachedAt = 0;
+const CACHE_TTL_MS = 5 * 60 * 1000;
+
+async function getCachedApiKey(plugin) {
+	const now = Date.now();
+	if (_cachedApiKey !== null && now - _cachedAt < CACHE_TTL_MS) {
+		return _cachedApiKey;
+	}
+	try {
+		// A1 fix: trim the cached value to match the on-wire key (embedder.js get() trims
+		// before sending to Meilisearch, so any echo in errors is the trimmed form).
+		// Without this, a padded stored key (" sk-abc ") would defeat value-based redaction
+		// when Meilisearch echoes the trimmed key ("sk-abc") in a 401 error.
+		_cachedApiKey = ((await settings.getOne(plugin.id, 'semanticSearchApiKey')) || '').trim();
+		_cachedAt = now;
+	} catch (e) {
+		// A2 fix: don't set _cachedAt on the failure path — let the next call retry immediately
+		// instead of cycling through the full 5-min TTL with a stale empty cache. Pattern-based
+		// redaction still runs, so sk-* and Bearer-shaped keys stay protected during the blip.
+		_cachedApiKey = '';
+	}
+	return _cachedApiKey;
+}
+
+// Test-only hooks (exported for the test sandbox; not consumed by production code paths).
+function _resetCache() {
+	_cachedApiKey = null;
+	_cachedAt = 0;
+}
+
+function _setCachedApiKey(value) {
+	_cachedApiKey = (value || '').trim();
+	_cachedAt = Date.now();
+}
+
+// Redact known secret patterns + the configured apiKey value from a free-form string.
+// Used by winston log calls and notifyAdmins payloads so credentials don't leak
+// when Meilisearch echoes them in error messages.
+//
+// Returns the redacted string. If `plugin` is omitted, only pattern-based redaction
+// runs (used in early-init paths before plugin object is fully wired).
+async function redactSecrets(message, plugin) {
+	if (typeof message !== 'string' || !message) return message;
+	let redacted = message;
+
+	// Layer 1: pattern-based — catches known formats even without DB access
+	for (const pattern of KEY_PATTERNS) {
+		redacted = redacted.replace(pattern, REDACTION_PLACEHOLDER);
+	}
+
+	// Layer 2: value-based — catches the exact configured apiKey regardless of format
+	if (plugin) {
+		const configuredKey = await getCachedApiKey(plugin);
+		if (configuredKey && configuredKey.length >= MIN_VALUE_REDACT_LENGTH) {
+			// Escape regex special chars in the key (keys can contain +, /, = for base64)
+			const escapedKey = configuredKey.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+			redacted = redacted.replace(new RegExp(escapedKey, 'g'), REDACTION_PLACEHOLDER);
+		}
+	}
+
+	return redacted;
+}
+
+// Whole-value redaction for winston diff logs (lib/settings.js isBreaking).
+// Returns '<redacted>' for credential-bearing fields, JSON.stringify for others.
+// Preserves the "did it change?" audit signal without leaking the credential value.
+function redactSettingForLog(setting, value) {
+	if (SECRET_SETTINGS.has(setting)) return REDACTION_PLACEHOLDER;
+	return JSON.stringify(value);
+}
+
+module.exports = {
+	redactSecrets,
+	redactSettingForLog,
+	SECRET_SETTINGS,
+	REDACTION_PLACEHOLDER,
+	// Test-only exports (not consumed by production code paths):
+	_resetCache,
+	_setCachedApiKey,
+};