nodebb-plugin-search-agent 0.0.934 → 0.0.936

package/library.js

@@ -30,13 +30,13 @@ plugin.init = async (params) => {
 
 	// Start initial embedding sync in the background — does not block NodeBB startup.
 	winston.info('[search-agent] Starting initial embedding sync…');
-	startSync().catch(err => winston.warn(`[search-agent] Initial sync failed: ${err.message}`));
+    startSync();
 
 	// Re-sync every 10 minutes to pick up new posts.
 	const RESYNC_INTERVAL_MS = 10 * 60 * 1000;
 	setInterval(() => {
 		winston.info('[search-agent] Running scheduled embedding re-sync…');
-		startSync().catch(err => winston.warn(`[search-agent] Scheduled re-sync failed: ${err.message}`));
+        startSync();
 	}, RESYNC_INTERVAL_MS).unref();
 
 	winston.info('[plugins/search-agent] Initialised.');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nodebb-plugin-search-agent",
-  "version": "0.0.934",
+  "version": "0.0.936",
   "description": "NodeBB plugin that adds a floating chat assistant to help users find relevant forum topics using TF-IDF text similarity",
   "main": "library.js",
   "author": "Racheli Bayfus",

package/services/embeddingService.js

@@ -1,3 +1,22 @@
+// Remove images, files, and non-text content from input
+function extractPureText(text) {
+	if (typeof text !== 'string') return '';
+	// Remove Markdown images: ![alt](url)
+	let cleaned = text.replace(/!\[[^\]]*\]\([^)]*\)/g, '');
+	// Remove HTML <img> tags
+	cleaned = cleaned.replace(/<img\b[^>]*>/gi, '');
+	// Remove links to files/images (common extensions)
+	cleaned = cleaned.replace(/https?:\/\/(\S+\.(jpg|jpeg|png|gif|bmp|svg|webp|pdf|docx?|xlsx?|pptx?|zip|rar|7z|tar|gz|mp3|mp4|avi|mov|wmv|flv|mkv|ogg|wav|exe|bin|apk|ipa|dmg|iso|csv|json|xml|yml|yaml|psd|ai|eps|ttf|otf|woff2?))(\?\S*)?/gi, '');
+	// Remove Markdown file links: [desc](url.ext)
+	cleaned = cleaned.replace(/\[[^\]]*\]\([^)]*\.(jpg|jpeg|png|gif|bmp|svg|webp|pdf|docx?|xlsx?|pptx?|zip|rar|7z|tar|gz|mp3|mp4|avi|mov|wmv|flv|mkv|ogg|wav|exe|bin|apk|ipa|dmg|iso|csv|json|xml|yml|yaml|psd|ai|eps|ttf|otf|woff2?)\)/gi, '');
+	// Remove any remaining <a href=...> tags to files
+	cleaned = cleaned.replace(/<a\b[^>]*href=["']?[^"'>]+\.(jpg|jpeg|png|gif|bmp|svg|webp|pdf|docx?|xlsx?|pptx?|zip|rar|7z|tar|gz|mp3|mp4|avi|mov|wmv|flv|mkv|ogg|wav|exe|bin|apk|ipa|dmg|iso|csv|json|xml|yml|yaml|psd|ai|eps|ttf|otf|woff2?)[^>]*>.*?<\/a>/gi, '');
+	// Remove any remaining HTML tags
+	cleaned = cleaned.replace(/<[^>]+>/g, ' ');
+	// Remove extra whitespace
+	cleaned = cleaned.replace(/[ \t]+/g, ' ').replace(/\n{2,}/g, '\n').trim();
+	return cleaned;
+}
 'use strict';
 
 const https = require('https');
@@ -16,7 +35,7 @@ const RETRY_DELAY_MS = 500;
 // Using 1.5 chars/token worst-case: 8000 tokens × 1.5 = 12 000 chars — gives a safe margin.
 
 const MAX_CHARS = 12000;
-const CHUNK_OVERLAP = 2000; // chars to overlap between chunks for context
+const CHUNK_OVERLAP = 200; // chars to overlap between chunks for context
 
 // Split a long string into chunks of maxLen, with optional overlap
 function splitIntoChunks(text, maxLen = MAX_CHARS, overlap = CHUNK_OVERLAP) {
@@ -136,36 +155,52 @@ async function embed(text) {
 		throw new Error('OPENAI_API_KEY environment variable is not set');
 	}
 
-	// Split into chunks if too long
-	const chunks = splitIntoChunks(text, MAX_CHARS, CHUNK_OVERLAP);
-	if (chunks.length === 1) {
-		const safe = truncate(text);
-		if (_embedCache.has(safe)) {
-			winston().verbose('[search-agent] embeddingService: embedding cache hit');
-			return _embedCache.get(safe);
-		}
-		winston().verbose(`[search-agent] embeddingService: generating embedding for text (${safe.length} chars)`);
-		const response = await withRetry(() => requestEmbeddings(apiKey, safe));
-		winston().verbose('[search-agent] embeddingService: embedding generated successfully');
-		const embedding = response.data[0].embedding;
-		if (_embedCache.size >= EMBED_CACHE_MAX) {
-			_embedCache.delete(_embedCache.keys().next().value);
-		}
-		_embedCache.set(safe, embedding);
-		return embedding;
-	} else {
-		// For multi-chunk, embed all and average
-		winston().verbose(`[search-agent] embeddingService: splitting long text into ${chunks.length} chunks for embedding`);
-		const vectors = await embedBatch(chunks);
-		const avg = averageVectors(vectors);
-		// Optionally cache the average for the full text
-		const safe = truncate(text);
-		if (_embedCache.size >= EMBED_CACHE_MAX) {
-			_embedCache.delete(_embedCache.keys().next().value);
-		}
-		_embedCache.set(safe, avg);
-		return avg;
-	}
+	       // Remove non-text content
+	       const pureText = extractPureText(text);
+	       if (!pureText) {
+		       throw new Error('embed() received no usable text after filtering');
+	       }
+	       // Split into chunks if too long
+	       const chunks = splitIntoChunks(pureText, MAX_CHARS, CHUNK_OVERLAP);
+	       // Estimate tokens (roughly 1.5 chars/token for non-ASCII, 4 chars/token for ASCII)
+	       const estimateTokens = (str) => {
+		       // If mostly ASCII, use 4 chars/token, else 1.5
+		       const ascii = /^[\x00-\x7F]*$/.test(str);
+		       return ascii ? Math.ceil(str.length / 4) : Math.ceil(str.length / 1.5);
+	       };
+	       if (chunks.length === 1) {
+		       const safe = truncate(text);
+		       if (_embedCache.has(safe)) {
+			       winston().verbose('[search-agent] embeddingService: embedding cache hit');
+			       return _embedCache.get(safe);
+		       }
+		       const tokenCount = estimateTokens(safe);
+		       winston().info(`[search-agent] embeddingService: generating embedding for text (${safe.length} chars, ~${tokenCount} tokens)`);
+		       const response = await withRetry(() => requestEmbeddings(apiKey, safe));
+		       winston().verbose('[search-agent] embeddingService: embedding generated successfully');
+		       const embedding = response.data[0].embedding;
+		       if (_embedCache.size >= EMBED_CACHE_MAX) {
+			       _embedCache.delete(_embedCache.keys().next().value);
+		       }
+		       _embedCache.set(safe, embedding);
+		       return embedding;
+	       } else {
+		       // For multi-chunk, embed all and average
+		       winston().info(`[search-agent] embeddingService: splitting long text into ${chunks.length} chunks for embedding`);
+		       chunks.forEach((chunk, i) => {
+			       const tokenCount = estimateTokens(chunk);
+			       winston().info(`[search-agent] embeddingService: chunk ${i+1}/${chunks.length} — ${chunk.length} chars, ~${tokenCount} tokens`);
+		       });
+		       const vectors = await embedBatch(chunks);
+		       const avg = averageVectors(vectors);
+		       // Optionally cache the average for the full text
+		       const safe = truncate(text);
+		       if (_embedCache.size >= EMBED_CACHE_MAX) {
+			       _embedCache.delete(_embedCache.keys().next().value);
+		       }
+		       _embedCache.set(safe, avg);
+		       return avg;
+	       }
 }
 
 /**
@@ -189,15 +224,39 @@ async function embedBatch(texts) {
 		throw new Error('OPENAI_API_KEY environment variable is not set');
 	}
 
-	// For each text, if too long, split and average embeddings
-	const allChunks = [];
-	const chunkMap = [];
-	for (const text of texts) {
-		const chunks = splitIntoChunks(text, MAX_CHARS, CHUNK_OVERLAP);
-		chunkMap.push({ count: chunks.length });
-		allChunks.push(...chunks);
+	       // For each text, filter to pure text, then split and average embeddings
+	       const allChunks = [];
+	       const chunkMap = [];
+	       // Estimate tokens (roughly 1.5 chars/token for non-ASCII, 4 chars/token for ASCII)
+	       const estimateTokens = (str) => {
+		       const ascii = /^[\x00-\x7F]*$/.test(str);
+		       return ascii ? Math.ceil(str.length / 4) : Math.ceil(str.length / 1.5);
+	       };
+	       for (const [textIdx, text] of texts.entries()) {
+		       const pureText = extractPureText(text);
+		       if (!pureText) {
+			       chunkMap.push({ count: 0 });
+			       continue;
+		       }
+		       const chunks = splitIntoChunks(pureText, MAX_CHARS, CHUNK_OVERLAP);
+		       chunkMap.push({ count: chunks.length });
+		       allChunks.push(...chunks);
+		       if (chunks.length === 1) {
+			       const tokenCount = estimateTokens(chunks[0]);
+			       winston().info(`[search-agent] embeddingService: batch input ${textIdx+1}/${texts.length} — 1 chunk, ${chunks[0].length} chars, ~${tokenCount} tokens`);
+		       } else {
+			       winston().info(`[search-agent] embeddingService: batch input ${textIdx+1}/${texts.length} — ${chunks.length} chunks`);
+			       chunks.forEach((chunk, i) => {
+				       const tokenCount = estimateTokens(chunk);
+				       winston().info(`[search-agent] embeddingService:   chunk ${i+1}/${chunks.length} — ${chunk.length} chars, ~${tokenCount} tokens`);
+			       });
+		       }
+	       }
+	       winston().verbose(`[search-agent] embeddingService: batch embedding ${allChunks.length} chunk(s) from ${texts.length} input(s)`);
+	if (allChunks.length === 0) {
+		// All texts were filtered out
+		return chunkMap.map(({ count }) => count === 0 ? [] : null);
 	}
-	winston().verbose(`[search-agent] embeddingService: batch embedding ${allChunks.length} chunk(s) from ${texts.length} input(s)`);
 	const safeChunks = allChunks.map(truncate);
 	const response = await withRetry(() => requestEmbeddings(apiKey, safeChunks));
 	winston().verbose(`[search-agent] embeddingService: batch embeddings generated successfully (${safeChunks.length} vector(s))`);
@@ -206,7 +265,9 @@ async function embedBatch(texts) {
 	const result = [];
 	let idx = 0;
 	for (const { count } of chunkMap) {
-		if (count === 1) {
+		if (count === 0) {
+			result.push([]); // No usable text
+		} else if (count === 1) {
 			result.push(vectors[idx]);
 			idx += 1;
 		} else {

package/services/vectorSearchService.js

@@ -13,7 +13,7 @@ const TOP_K = 50;
 // Absolute minimum cosine similarity — only filters pure noise (near-zero similarity).
 // Do NOT raise this: the relevant result often scores lower than irrelevant ones.
 // The AI re-ranker (which reads content) is the precision gate, not this floor.
-const MIN_SCORE = 0.10;
+const MIN_SCORE = 0.15;
 // Rebuild the Orama index after this interval (mirrors TF-IDF cache TTL)
 const INDEX_TTL_MS = 5 * 60 * 1000;