nodebb-plugin-search-agent 0.0.935 → 0.0.936

package/library.js

@@ -30,13 +30,13 @@ plugin.init = async (params) => {
 
 	// Start initial embedding sync in the background — does not block NodeBB startup.
 	winston.info('[search-agent] Starting initial embedding sync…');
-	startSync().catch(err => winston.warn(`[search-agent] Initial sync failed: ${err.message}`));
+    startSync();
 
 	// Re-sync every 10 minutes to pick up new posts.
 	const RESYNC_INTERVAL_MS = 10 * 60 * 1000;
 	setInterval(() => {
 		winston.info('[search-agent] Running scheduled embedding re-sync…');
-		startSync().catch(err => winston.warn(`[search-agent] Scheduled re-sync failed: ${err.message}`));
+        startSync();
 	}, RESYNC_INTERVAL_MS).unref();
 
 	winston.info('[plugins/search-agent] Initialised.');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nodebb-plugin-search-agent",
-  "version": "0.0.935",
+  "version": "0.0.936",
   "description": "NodeBB plugin that adds a floating chat assistant to help users find relevant forum topics using TF-IDF text similarity",
   "main": "library.js",
   "author": "Racheli Bayfus",

package/services/embeddingService.js

@@ -155,41 +155,52 @@ async function embed(text) {
 		throw new Error('OPENAI_API_KEY environment variable is not set');
 	}
 
-	// Remove non-text content
-	const pureText = extractPureText(text);
-	if (!pureText) {
-		throw new Error('embed() received no usable text after filtering');
-	}
-	// Split into chunks if too long
-	const chunks = splitIntoChunks(pureText, MAX_CHARS, CHUNK_OVERLAP);
-	if (chunks.length === 1) {
-		const safe = truncate(text);
-		if (_embedCache.has(safe)) {
-			winston().verbose('[search-agent] embeddingService: embedding cache hit');
-			return _embedCache.get(safe);
-		}
-		winston().verbose(`[search-agent] embeddingService: generating embedding for text (${safe.length} chars)`);
-		const response = await withRetry(() => requestEmbeddings(apiKey, safe));
-		winston().verbose('[search-agent] embeddingService: embedding generated successfully');
-		const embedding = response.data[0].embedding;
-		if (_embedCache.size >= EMBED_CACHE_MAX) {
-			_embedCache.delete(_embedCache.keys().next().value);
-		}
-		_embedCache.set(safe, embedding);
-		return embedding;
-	} else {
-		// For multi-chunk, embed all and average
-		winston().verbose(`[search-agent] embeddingService: splitting long text into ${chunks.length} chunks for embedding`);
-		const vectors = await embedBatch(chunks);
-		const avg = averageVectors(vectors);
-		// Optionally cache the average for the full text
-		const safe = truncate(text);
-		if (_embedCache.size >= EMBED_CACHE_MAX) {
-			_embedCache.delete(_embedCache.keys().next().value);
-		}
-		_embedCache.set(safe, avg);
-		return avg;
-	}
+	       // Remove non-text content
+	       const pureText = extractPureText(text);
+	       if (!pureText) {
+		       throw new Error('embed() received no usable text after filtering');
+	       }
+	       // Split into chunks if too long
+	       const chunks = splitIntoChunks(pureText, MAX_CHARS, CHUNK_OVERLAP);
+	       // Estimate tokens (roughly 1.5 chars/token for non-ASCII, 4 chars/token for ASCII)
+	       const estimateTokens = (str) => {
+		       // If mostly ASCII, use 4 chars/token, else 1.5
+		       const ascii = /^[\x00-\x7F]*$/.test(str);
+		       return ascii ? Math.ceil(str.length / 4) : Math.ceil(str.length / 1.5);
+	       };
+	       if (chunks.length === 1) {
+		       const safe = truncate(text);
+		       if (_embedCache.has(safe)) {
+			       winston().verbose('[search-agent] embeddingService: embedding cache hit');
+			       return _embedCache.get(safe);
+		       }
+		       const tokenCount = estimateTokens(safe);
+		       winston().info(`[search-agent] embeddingService: generating embedding for text (${safe.length} chars, ~${tokenCount} tokens)`);
+		       const response = await withRetry(() => requestEmbeddings(apiKey, safe));
+		       winston().verbose('[search-agent] embeddingService: embedding generated successfully');
+		       const embedding = response.data[0].embedding;
+		       if (_embedCache.size >= EMBED_CACHE_MAX) {
+			       _embedCache.delete(_embedCache.keys().next().value);
+		       }
+		       _embedCache.set(safe, embedding);
+		       return embedding;
+	       } else {
+		       // For multi-chunk, embed all and average
+		       winston().info(`[search-agent] embeddingService: splitting long text into ${chunks.length} chunks for embedding`);
+		       chunks.forEach((chunk, i) => {
+			       const tokenCount = estimateTokens(chunk);
+			       winston().info(`[search-agent] embeddingService: chunk ${i+1}/${chunks.length} — ${chunk.length} chars, ~${tokenCount} tokens`);
+		       });
+		       const vectors = await embedBatch(chunks);
+		       const avg = averageVectors(vectors);
+		       // Optionally cache the average for the full text
+		       const safe = truncate(text);
+		       if (_embedCache.size >= EMBED_CACHE_MAX) {
+			       _embedCache.delete(_embedCache.keys().next().value);
+		       }
+		       _embedCache.set(safe, avg);
+		       return avg;
+	       }
 }
 
 /**
@@ -213,20 +224,35 @@ async function embedBatch(texts) {
 		throw new Error('OPENAI_API_KEY environment variable is not set');
 	}
 
-	// For each text, filter to pure text, then split and average embeddings
-	const allChunks = [];
-	const chunkMap = [];
-	for (const text of texts) {
-		const pureText = extractPureText(text);
-		if (!pureText) {
-			chunkMap.push({ count: 0 });
-			continue;
-		}
-		const chunks = splitIntoChunks(pureText, MAX_CHARS, CHUNK_OVERLAP);
-		chunkMap.push({ count: chunks.length });
-		allChunks.push(...chunks);
-	}
-	winston().verbose(`[search-agent] embeddingService: batch embedding ${allChunks.length} chunk(s) from ${texts.length} input(s)`);
+	       // For each text, filter to pure text, then split and average embeddings
+	       const allChunks = [];
+	       const chunkMap = [];
+	       // Estimate tokens (roughly 1.5 chars/token for non-ASCII, 4 chars/token for ASCII)
+	       const estimateTokens = (str) => {
+		       const ascii = /^[\x00-\x7F]*$/.test(str);
+		       return ascii ? Math.ceil(str.length / 4) : Math.ceil(str.length / 1.5);
+	       };
+	       for (const [textIdx, text] of texts.entries()) {
+		       const pureText = extractPureText(text);
+		       if (!pureText) {
+			       chunkMap.push({ count: 0 });
+			       continue;
+		       }
+		       const chunks = splitIntoChunks(pureText, MAX_CHARS, CHUNK_OVERLAP);
+		       chunkMap.push({ count: chunks.length });
+		       allChunks.push(...chunks);
+		       if (chunks.length === 1) {
+			       const tokenCount = estimateTokens(chunks[0]);
+			       winston().info(`[search-agent] embeddingService: batch input ${textIdx+1}/${texts.length} — 1 chunk, ${chunks[0].length} chars, ~${tokenCount} tokens`);
+		       } else {
+			       winston().info(`[search-agent] embeddingService: batch input ${textIdx+1}/${texts.length} — ${chunks.length} chunks`);
+			       chunks.forEach((chunk, i) => {
+				       const tokenCount = estimateTokens(chunk);
+				       winston().info(`[search-agent] embeddingService:   chunk ${i+1}/${chunks.length} — ${chunk.length} chars, ~${tokenCount} tokens`);
+			       });
+		       }
+	       }
+	       winston().verbose(`[search-agent] embeddingService: batch embedding ${allChunks.length} chunk(s) from ${texts.length} input(s)`);
 	if (allChunks.length === 0) {
 		// All texts were filtered out
 		return chunkMap.map(({ count }) => count === 0 ? [] : null);

package/services/vectorSearchService.js

@@ -13,7 +13,7 @@ const TOP_K = 50;
 // Absolute minimum cosine similarity — only filters pure noise (near-zero similarity).
 // Do NOT raise this: the relevant result often scores lower than irrelevant ones.
 // The AI re-ranker (which reads content) is the precision gate, not this floor.
-const MIN_SCORE = 0.10;
+const MIN_SCORE = 0.15;
 // Rebuild the Orama index after this interval (mirrors TF-IDF cache TTL)
 const INDEX_TTL_MS = 5 * 60 * 1000;