nodebb-plugin-search-agent 0.0.938 → 0.0.941

package/lib/cosineSimilarity.js

@@ -0,0 +1,42 @@
+'use strict';
+
+/**
+ * Computes the cosine similarity between two numeric vectors.
+ * Handles mismatched lengths by using the shorter vector's dimension.
+ * Returns 0 if either vector has zero magnitude.
+ *
+ * @param {number[]} a
+ * @param {number[]} b
+ * @returns {number} similarity in [-1, 1]
+ */
+function cosineSimilarity(a, b) {
+	const len = Math.min(a.length, b.length);
+	let dot = 0;
+	let magA = 0;
+	let magB = 0;
+
+	for (let i = 0; i < len; i++) {
+		dot += a[i] * b[i];
+		magA += a[i] * a[i];
+		magB += b[i] * b[i];
+	}
+
+	const denom = Math.sqrt(magA) * Math.sqrt(magB);
+	return denom === 0 ? 0 : dot / denom;
+}
+
+/**
+ * Ranks items by cosine similarity to a query embedding.
+ * Each item must have an `embedding` property (number[]).
+ *
+ * @param {number[]} queryEmbedding
+ * @param {Array<{embedding: number[], [key: string]: any}>} items
+ * @returns {Array<{item: object, score: number}>} sorted descending by score
+ */
+function rankBySimilarity(queryEmbedding, items) {
+	return items
+		.map(item => ({ item, score: cosineSimilarity(queryEmbedding, item.embedding) }))
+		.sort((a, b) => b.score - a.score);
+}
+
+module.exports = { cosineSimilarity, rankBySimilarity };

package/lib/searchHandler.js

@@ -1,9 +1,3 @@
-// ─── Token estimation helper ───────────────────────────────────────────────
-function estimateTokens(str) {
-	// Roughly 4 chars/token for English, 2 for Hebrew/UTF-8, but 4 is safe for cost estimation
-	return Math.ceil(str.length / 4);
-}
-
 'use strict';
 
 const https = require('https');
@@ -181,18 +175,18 @@ function callOpenAI(apiKey, model, messages) {
  * @param {string} model
  * @returns {Promise<string>}
  */
-// async function expandQueryWithHyDE(queryText, apiKey, model) {
-// 	const response = await callOpenAI(apiKey, model, [
-// 		{
-// 			role: 'system',
-// 			content:
-// 				'אתה חבר בפורום. בהינתן שאלת חיפוש, כתוב פוסט תגובה קצר וריאליסטי בפורום שעונה ישירות על השאלה. ' +
-// 				'כתוב רק את תוכן הפוסט — ללא ברכות, הערות מטא, או שורת נושא.',
-// 		},
-// 		{ role: 'user', content: queryText },
-// 	]);
-// 	return (response.choices[0].message.content || '').trim() || queryText;
-// }
+async function expandQueryWithHyDE(queryText, apiKey, model) {
+	const response = await callOpenAI(apiKey, model, [
+		{
+			role: 'system',
+			content:
+				'אתה חבר בפורום. בהינתן שאלת חיפוש, כתוב פוסט תגובה קצר וריאליסטי בפורום שעונה ישירות על השאלה. ' +
+				'כתוב רק את תוכן הפוסט — ללא ברכות, הערות מטא, או שורת נושא.',
+		},
+		{ role: 'user', content: queryText },
+	]);
+	return (response.choices[0].message.content || '').trim() || queryText;
+}
 
 /**
  * Send candidates to OpenAI for independent per-topic relevance scoring.
@@ -204,38 +198,25 @@ function callOpenAI(apiKey, model, messages) {
 async function reRankWithAI(queryText, candidates, topicMap, apiKey, model, maxResults, snippetByTid = {}) {
 	console.log('Re-ranking with AI:', { queryText, candidates: candidates.map(c => ({ tid: c.tid, title: (topicMap[String(c.tid)] || {}).title })) });
 
-
-	// Embed the query and all candidate post snippets
-
-	const { embed, embedBatch } = require('../services/embeddingService');
-	const queryEmbedding = await embed(queryText);
-	const postSnippets = candidates.map((c) => {
-		const raw = (snippetByTid[String(c.tid)] || '').replace(/<[^>]*>/g, ' ').replace(/[ \t]+/g, ' ').trim();
-		return raw.slice(0, 1500);
-	});
-	const postEmbeddings = await embedBatch(postSnippets);
-
-	// Format: [tid:..., embedding: [v1, v2, ...]]
-	const candidateList = candidates.map((c, i) => {
-		return `[tid:${c.tid}]\nembedding: [${postEmbeddings[i].slice(0, 8).map(x => x.toFixed(4)).join(', ')} ...]`;
-	}).join('\n\n');
+	// Only send the embedded query and the matched post snippet for each candidate
+	const candidateList = candidates
+		.map((c) => {
+			const raw = (snippetByTid[String(c.tid)] || '').replace(/<[^>]*>/g, ' ').replace(/[ \t]+/g, ' ').trim();
+			// Only send the snippet, not the title
+			return `[tid:${c.tid}]\n${raw.slice(0, 1500)}`;
+		})
+		.join('\n\n');
 
 	const systemPrompt =
 		'אתה מסנן חיפוש פורום מחמיר. ' +
-		'לכל מועמד ברשימה, דרג את הרלוונטיות של embedding הפוסט לembedding של השאלה בסקלה 0-10: ' +
+		'לכל מועמד ברשימה, דרג את הרלוונטיות של קטע הפוסט לשאלת המשתמש בסקלה 0-10: ' +
 		'10 = עונה ישירות ובאופן מלא. 7-9 = עונה על חלק משמעותי. 0-6 = לא רלוונטי. ' +
 		'החזר אך ורק JSON תקני במבנה: {"tid": ציון, ...} — לדוגמה: {"42": 9, "15": 3}. ' +
-		'אין להוסיף הסברים, טקסט נוסף, או עיצוב מחוץ ל-JSON.';
-        
-	const userMessage =
-		`embedding של שאלת המשתמש: [${queryEmbedding.slice(0, 8).map(x => x.toFixed(4)).join(', ')} ...]\n\nפוסטים:\n${candidateList}`;
+		'אין להוסיף הסברים, טקסט נוסף, או עיצוב מחוץ ל-JSON.'+
+		'הוסף שדה נוסף "scoreExplanation" עם משפט קצר שמסביר למה קטע עם ציון נמוך לא רלוונטי.';
 
-	// --- Token count logging ---
-	const totalEmbeddingChars = queryText.length + postSnippets.reduce((sum, s) => sum + s.length, 0);
-	const embeddingTokens = estimateTokens(queryText) + postSnippets.reduce((sum, s) => sum + estimateTokens(s), 0);
-	const llmPromptTokens = estimateTokens(systemPrompt) + estimateTokens(userMessage);
-	const winston = require.main.require('winston');
-	winston.info(`[search-agent] Token usage: embedding API ≈ ${embeddingTokens} tokens, LLM prompt ≈ ${llmPromptTokens} tokens (for this search)`);
+	const userMessage =
+		`שאלת המשתמש (מוטמעת): "${queryText}"\n\nפוסטים:\n${candidateList}`;
 
 	const response = await callOpenAI(apiKey, model, [
 		{ role: 'system', content: systemPrompt },
@@ -255,25 +236,12 @@ async function reRankWithAI(queryText, candidates, topicMap, apiKey, model, maxR
 	const scores = JSON.parse(match[0]);
 	const candidateByTid = Object.fromEntries(candidates.map(c => [String(c.tid), c]));
 
-	       let filtered = Object.entries(scores)
-		       .filter(([, score]) => Number(score) >= 7)
-		       .sort(([, a], [, b]) => Number(b) - Number(a))
-		       .slice(0, maxResults)
-		       .map(([tid]) => candidateByTid[tid])
-		       .filter(Boolean);
-
-	       // If nothing passed the threshold, return the top scoring candidate (if any)
-	       if (filtered.length === 0 && candidates.length > 0) {
-		       // Find the tid with the highest score
-		       const sortedAll = Object.entries(scores)
-			       .sort(([, a], [, b]) => Number(b) - Number(a));
-		       if (sortedAll.length > 0) {
-			       const [topTid] = sortedAll[0];
-			       const topCandidate = candidateByTid[topTid];
-			       if (topCandidate) filtered = [topCandidate];
-		       }
-	       }
-	       return filtered;
+	return Object.entries(scores)
+		.filter(([, score]) => Number(score) >= 7)
+		.sort(([, a], [, b]) => Number(b) - Number(a))
+		.slice(0, maxResults)
+		.map(([tid]) => candidateByTid[tid])
+		.filter(Boolean);
 }
 
 // ─── Public API ───────────────────────────────────────────────────────────────
@@ -305,16 +273,16 @@ async function searchTopics(queryText) {
 		// HyDE: replace the short raw query with a hypothetical answer so the
 		// embedding matches post content more closely.
 		let embeddingQuery = queryText;
-		// if (useAI && settings.hydeEnabled) {
-		// 	try {
-		// 		embeddingQuery = await expandQueryWithHyDE(
-		// 			queryText, settings.openaiApiKey, settings.openaiModel
-		// 		);
-		// 		winston.verbose(`[search-agent] HyDE expanded query (${embeddingQuery.length} chars)`);
-		// 	} catch (hydeErr) {
-		// 		winston.warn(`[search-agent] HyDE expansion failed, using raw query: ${hydeErr.message}`);
-		// 	}
-		// }
+		if (useAI && settings.hydeEnabled) {
+			try {
+				embeddingQuery = await expandQueryWithHyDE(
+					queryText, settings.openaiApiKey, settings.openaiModel
+				);
+				winston.verbose(`[search-agent] HyDE expanded query (${embeddingQuery.length} chars)`);
+			} catch (hydeErr) {
+				winston.warn(`[search-agent] HyDE expansion failed, using raw query: ${hydeErr.message}`);
+			}
+		}
 
 		// Request more candidates when AI will re-rank them.
 		const vectorLimit = useAI ? settings.aiCandidates : settings.maxResults;

package/library.js

@@ -30,13 +30,13 @@ plugin.init = async (params) => {
 
 	// Start initial embedding sync in the background — does not block NodeBB startup.
 	winston.info('[search-agent] Starting initial embedding sync…');
-    startSync();
+	startSync().catch(err => winston.warn(`[search-agent] Initial sync failed: ${err.message}`));
 
 	// Re-sync every 10 minutes to pick up new posts.
 	const RESYNC_INTERVAL_MS = 10 * 60 * 1000;
 	setInterval(() => {
 		winston.info('[search-agent] Running scheduled embedding re-sync…');
-        startSync();
+		startSync().catch(err => winston.warn(`[search-agent] Scheduled re-sync failed: ${err.message}`));
 	}, RESYNC_INTERVAL_MS).unref();
 
 	winston.info('[plugins/search-agent] Initialised.');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nodebb-plugin-search-agent",
-  "version": "0.0.938",
+  "version": "0.0.941",
   "description": "NodeBB plugin that adds a floating chat assistant to help users find relevant forum topics using TF-IDF text similarity",
   "main": "library.js",
   "author": "Racheli Bayfus",

package/services/embeddingService.js

@@ -3,7 +3,7 @@
 const https = require('https');
 
 function winston() {
-    return require.main.require('winston');
+	return require.main.require('winston');
 }
 
 const OPENAI_EMBEDDINGS_HOSTNAME = 'api.openai.com';
@@ -11,236 +11,21 @@ const OPENAI_EMBEDDINGS_PATH = '/v1/embeddings';
 const EMBEDDING_MODEL = 'text-embedding-3-small';
 const MAX_RETRIES = 3;
 const RETRY_DELAY_MS = 500;
-
-// text-embedding-3-small supports 8,192 tokens.
-// Conservative char limits help avoid token overflows, especially for non-ASCII text.
-const MAX_CHARS = 10000;
-const CHUNK_OVERLAP = 300;
-const MIN_CHUNK_CHARS = 500;
-const TARGET_CHUNK_CHARS = 2200;
-
-// ─── Text cleanup ─────────────────────────────────────────────────────────────
-
-function extractPureText(text) {
-    if (typeof text !== 'string') return '';
-
-    // Remove Markdown images: ![alt](url)
-    let cleaned = text.replace(/!\[[^\]]*\]\([^)]*\)/g, '');
-
-    // Remove HTML <img> tags
-    cleaned = cleaned.replace(/<img\b[^>]*>/gi, '');
-
-    // Remove links to files/images (common extensions)
-    cleaned = cleaned.replace(
-        /https?:\/\/(\S+\.(jpg|jpeg|png|gif|bmp|svg|webp|pdf|docx?|xlsx?|pptx?|zip|rar|7z|tar|gz|mp3|mp4|avi|mov|wmv|flv|mkv|ogg|wav|exe|bin|apk|ipa|dmg|iso|csv|json|xml|yml|yaml|psd|ai|eps|ttf|otf|woff2?))(\?\S*)?/gi,
-        ''
-    );
-
-    // Remove Markdown file links: [desc](url.ext)
-    cleaned = cleaned.replace(
-        /\[[^\]]*\]\([^)]*\.(jpg|jpeg|png|gif|bmp|svg|webp|pdf|docx?|xlsx?|pptx?|zip|rar|7z|tar|gz|mp3|mp4|avi|mov|wmv|flv|mkv|ogg|wav|exe|bin|apk|ipa|dmg|iso|csv|json|xml|yml|yaml|psd|ai|eps|ttf|otf|woff2?)\)/gi,
-        ''
-    );
-
-    // Remove any remaining <a href=...> tags to files
-    cleaned = cleaned.replace(
-        /<a\b[^>]*href=["']?[^"'>]+\.(jpg|jpeg|png|gif|bmp|svg|webp|pdf|docx?|xlsx?|pptx?|zip|rar|7z|tar|gz|mp3|mp4|avi|mov|wmv|flv|mkv|ogg|wav|exe|bin|apk|ipa|dmg|iso|csv|json|xml|yml|yaml|psd|ai|eps|ttf|otf|woff2?)[^>]*>.*?<\/a>/gi,
-        ''
-    );
-
-    // Remove any remaining HTML tags
-    cleaned = cleaned.replace(/<[^>]+>/g, ' ');
-
-    // Remove extra whitespace
-    cleaned = cleaned
-        .replace(/[ \t]+/g, ' ')
-        .replace(/\n{2,}/g, '\n')
-        .trim();
-
-    return cleaned;
-}
-
-function normalizeWhitespace(text) {
-    return text
-        .replace(/\r\n/g, '\n')
-        .replace(/[ \t]+/g, ' ')
-        .replace(/\n{3,}/g, '\n\n')
-        .trim();
-}
-
-function splitIntoBlocks(text) {
-    const normalized = normalizeWhitespace(text);
-
-    const rawBlocks = normalized
-        .split(/\n{2,}|(?=^#{1,6}\s)|(?=^\s*[-*+]\s)|(?=^\s*\d+\.\s)|(?=^\s*>\s)|(?=^```)/gm)
-        .map(block => block.trim())
-        .filter(Boolean);
-
-    return rawBlocks;
-}
-
-function splitLargeBlock(block, maxLen = MAX_CHARS) {
-    if (block.length <= maxLen) return [block];
-
-    const sentences = block.match(/[^.!?\n]+[.!?\n]+|[^.!?\n]+$/g) || [block];
-    const chunks = [];
-    let current = '';
-
-    for (const sentence of sentences) {
-        const s = sentence.trim();
-        if (!s) continue;
-
-        if ((current + ' ' + s).trim().length <= maxLen) {
-            current = current ? `${current} ${s}` : s;
-            continue;
-        }
-
-        if (current) {
-            chunks.push(current);
-            current = '';
-        }
-
-        // Fallback for very long sentence
-        if (s.length > maxLen) {
-            let i = 0;
-            while (i < s.length) {
-                chunks.push(s.slice(i, i + maxLen).trim());
-                i += maxLen;
-            }
-        } else {
-            current = s;
-        }
-    }
-
-    if (current) chunks.push(current);
-
-    return chunks;
-}
-
-function buildOverlapPrefix(prevChunk, overlapChars = CHUNK_OVERLAP) {
-    if (!prevChunk) return '';
-    return prevChunk.slice(Math.max(0, prevChunk.length - overlapChars)).trim();
-}
-
-function splitIntoSemanticChunks(
-    text,
-    {
-        maxLen = MAX_CHARS,
-        targetLen = TARGET_CHUNK_CHARS,
-        minLen = MIN_CHUNK_CHARS,
-        overlap = CHUNK_OVERLAP,
-    } = {}
-) {
-    if (!text) return [];
-    if (text.length <= maxLen) return [text];
-
-    const blocks = splitIntoBlocks(text).flatMap(block => splitLargeBlock(block, maxLen));
-
-    const chunks = [];
-    let current = '';
-
-    for (const block of blocks) {
-        const next = current ? `${current}\n\n${block}` : block;
-
-        if (next.length <= targetLen || current.length < minLen) {
-            if (next.length <= maxLen) {
-                current = next;
-                continue;
-            }
-        }
-
-        if (current) {
-            chunks.push(current.trim());
-        }
-
-        current = block;
-    }
-
-    if (current) {
-        chunks.push(current.trim());
-    }
-
-    const withOverlap = chunks.map((chunk, i) => {
-        if (i === 0) return chunk;
-
-        const prefix = buildOverlapPrefix(chunks[i - 1], overlap);
-        const merged = prefix ? `${prefix}\n\n${chunk}` : chunk;
-
-        return merged.length <= maxLen ? merged : merged.slice(merged.length - maxLen);
-    });
-
-    const finalChunks = [];
-    for (const chunk of withOverlap) {
-        if (
-            finalChunks.length > 0 &&
-            chunk.length < minLen &&
-            finalChunks[finalChunks.length - 1].length + 2 + chunk.length <= maxLen
-        ) {
-            finalChunks[finalChunks.length - 1] += `\n\n${chunk}`;
-        } else {
-            finalChunks.push(chunk);
-        }
-    }
-
-    return finalChunks;
-}
-
-// ─── Vector helpers ───────────────────────────────────────────────────────────
-
-function averageVectors(vectors) {
-    if (!vectors.length) return [];
-    const len = vectors[0].length;
-    const sum = new Array(len).fill(0);
-
-    for (const v of vectors) {
-        for (let i = 0; i < len; i++) {
-            sum[i] += v[i];
-        }
-    }
-
-    return sum.map(x => x / vectors.length);
-}
-
-function truncate(text) {
-    return text.length > MAX_CHARS ? text.slice(0, MAX_CHARS) : text;
-}
-
-function estimateTokens(str) {
-    const ascii = /^[\x00-\x7F]*$/.test(str);
-    return ascii ? Math.ceil(str.length / 4) : Math.ceil(str.length / 1.5);
-}
+// text-embedding-3-small supports 8 192 tokens.
+// Hebrew/non-ASCII text tokenizes at ~1.5–2 chars/token (UTF-8 multibyte).
+// Using 1.5 chars/token worst-case: 8000 tokens × 1.5 = 12 000 chars — gives a safe margin.
+const MAX_CHARS = 12000;
 
 // ─── Embedding cache ──────────────────────────────────────────────────────────
-
+// Avoids calling the embeddings API for the same text within a session.
+// HyDE output varies, so the biggest wins come from repeated identical queries.
 const _embedCache = new Map();
 const EMBED_CACHE_MAX = 500;
 
-function getCachedEmbedding(key) {
-    if (!_embedCache.has(key)) return null;
-
-    const value = _embedCache.get(key);
-
-    // Refresh LRU-ish order
-    _embedCache.delete(key);
-    _embedCache.set(key, value);
-
-    return value;
-}
-
-function setCachedEmbedding(key, value) {
-    if (_embedCache.has(key)) {
-        _embedCache.delete(key);
-    }
-
-    while (_embedCache.size >= EMBED_CACHE_MAX) {
-        _embedCache.delete(_embedCache.keys().next().value);
-    }
-
-    _embedCache.set(key, value);
+function truncate(text) {
+	return text.length > MAX_CHARS ? text.slice(0, MAX_CHARS) : text;
 }
 
-// ─── OpenAI request helpers ───────────────────────────────────────────────────
-
 /**
  * Performs an HTTPS POST request to the OpenAI embeddings endpoint.
  * @param {string} apiKey
@@ -248,246 +33,132 @@ function setCachedEmbedding(key, value) {
  * @returns {Promise<object>} Parsed JSON response body
  */
 function requestEmbeddings(apiKey, input) {
-    return new Promise((resolve, reject) => {
-        const body = JSON.stringify({ model: EMBEDDING_MODEL, input });
-
-        const options = {
-            hostname: OPENAI_EMBEDDINGS_HOSTNAME,
-            path: OPENAI_EMBEDDINGS_PATH,
-            method: 'POST',
-            headers: {
-                'Content-Type': 'application/json',
-                'Authorization': `Bearer ${apiKey}`,
-                'Content-Length': Buffer.byteLength(body),
-            },
-        };
-
-        const req = https.request(options, res => {
-            const chunks = [];
-
-            res.on('data', chunk => chunks.push(chunk));
-
-            res.on('end', () => {
-                let parsed;
-                try {
-                    parsed = JSON.parse(Buffer.concat(chunks).toString('utf8'));
-                } catch (e) {
-                    return reject(new Error(`Failed to parse OpenAI response: ${e.message}`));
-                }
-
-                if (res.statusCode >= 400) {
-                    const message = (parsed.error && parsed.error.message) || `HTTP ${res.statusCode}`;
-                    return reject(new Error(`OpenAI API error: ${message}`));
-                }
-
-                resolve(parsed);
-            });
-        });
-
-        req.on('error', err => reject(new Error(`Network error calling OpenAI: ${err.message}`)));
-        req.write(body);
-        req.end();
-    });
+	return new Promise((resolve, reject) => {
+		const body = JSON.stringify({ model: EMBEDDING_MODEL, input });
+		const options = {
+			hostname: OPENAI_EMBEDDINGS_HOSTNAME,
+			path: OPENAI_EMBEDDINGS_PATH,
+			method: 'POST',
+			headers: {
+				'Content-Type': 'application/json',
+				'Authorization': `Bearer ${apiKey}`,
+				'Content-Length': Buffer.byteLength(body),
+			},
+		};
+
+		const req = https.request(options, (res) => {
+			const chunks = [];
+			res.on('data', chunk => chunks.push(chunk));
+			res.on('end', () => {
+				let parsed;
+				try {
+					parsed = JSON.parse(Buffer.concat(chunks).toString('utf8'));
+				} catch (e) {
+					return reject(new Error(`Failed to parse OpenAI response: ${e.message}`));
+				}
+
+				if (res.statusCode >= 400) {
+					const message = (parsed.error && parsed.error.message) || `HTTP ${res.statusCode}`;
+					return reject(new Error(`OpenAI API error: ${message}`));
+				}
+
+				resolve(parsed);
+			});
+		});
+
+		req.on('error', err => reject(new Error(`Network error calling OpenAI: ${err.message}`)));
+		req.write(body);
+		req.end();
+	});
 }
 
 /**
  * Retries an async operation up to maxRetries times with exponential back-off.
- * @param {Function} fn
+ * @param {Function} fn - Async function to retry
  * @param {number} retries
  * @returns {Promise<*>}
  */
 async function withRetry(fn, retries = MAX_RETRIES) {
-    let lastError;
-
-    for (let attempt = 1; attempt <= retries; attempt++) {
-        try {
-            return await fn();
-        } catch (err) {
-            lastError = err;
-
-            if (attempt < retries) {
-                winston().warn(
-                    `[search-agent] embeddingService: attempt ${attempt} failed (${err.message}), retrying in ${RETRY_DELAY_MS * attempt} ms...`
-                );
-                await new Promise(resolve => setTimeout(resolve, RETRY_DELAY_MS * attempt));
-            }
-        }
-    }
-
-    throw lastError;
+	let lastError;
+	for (let attempt = 1; attempt <= retries; attempt++) {
+		try {
+			return await fn();
+		} catch (err) {
+			lastError = err;
+			if (attempt < retries) {
+				winston().warn(`[search-agent] embeddingService: attempt ${attempt} failed (${err.message}), retrying in ${RETRY_DELAY_MS * attempt} ms…`);
+				await new Promise(resolve => setTimeout(resolve, RETRY_DELAY_MS * attempt));
+			}
+		}
+	}
+	throw lastError;
 }
 
-// ─── Public API ───────────────────────────────────────────────────────────────
-
 /**
  * Converts a single text string into an embedding vector.
  * @param {string} text
  * @returns {Promise<number[]>}
  */
 async function embed(text) {
-    if (typeof text !== 'string' || text.trim() === '') {
-        throw new Error('embed() requires a non-empty string');
-    }
-
-    const apiKey = process.env.OPENAI_API_KEY;
-    if (!apiKey) {
-        throw new Error('OPENAI_API_KEY environment variable is not set');
-    }
-
-    const pureText = extractPureText(text);
-    if (!pureText) {
-        throw new Error('embed() received no usable text after filtering');
-    }
+	if (typeof text !== 'string' || text.trim() === '') {
+		throw new Error('embed() requires a non-empty string');
+	}
 
-    const chunks = splitIntoSemanticChunks(pureText, {
-        maxLen: MAX_CHARS,
-        targetLen: TARGET_CHUNK_CHARS,
-        minLen: MIN_CHUNK_CHARS,
-        overlap: CHUNK_OVERLAP,
-    });
+	const apiKey = process.env.OPENAI_API_KEY;
+	if (!apiKey) {
+		throw new Error('OPENAI_API_KEY environment variable is not set');
+	}
 
-    if (chunks.length === 1) {
-        const safe = truncate(pureText);
-        const cached = getCachedEmbedding(safe);
-        if (cached) {
-            winston().verbose('[search-agent] embeddingService: embedding cache hit');
-            return cached;
-        }
+	const safe = truncate(text);
 
-        const tokenCount = estimateTokens(safe);
-        winston().info(
-            `[search-agent] embeddingService: generating embedding for text (${safe.length} chars, ~${tokenCount} tokens)`
-        );
+	if (_embedCache.has(safe)) {
+		winston().verbose('[search-agent] embeddingService: embedding cache hit');
+		return _embedCache.get(safe);
+	}
 
-        const response = await withRetry(() => requestEmbeddings(apiKey, safe));
-        const embedding = response.data[0].embedding;
+	winston().verbose(`[search-agent] embeddingService: generating embedding for text (${safe.length} chars)`);
+	const response = await withRetry(() => requestEmbeddings(apiKey, safe));
+	winston().verbose('[search-agent] embeddingService: embedding generated successfully');
+	const embedding = response.data[0].embedding;
 
-        winston().verbose('[search-agent] embeddingService: embedding generated successfully');
-        setCachedEmbedding(safe, embedding);
+	if (_embedCache.size >= EMBED_CACHE_MAX) {
+		_embedCache.delete(_embedCache.keys().next().value);
+	}
+	_embedCache.set(safe, embedding);
 
-        return embedding;
-    }
-
-    winston().info(
-        `[search-agent] embeddingService: splitting long text into ${chunks.length} semantic chunks for embedding`
-    );
-
-    chunks.forEach((chunk, i) => {
-        const tokenCount = estimateTokens(chunk);
-        winston().info(
-            `[search-agent] embeddingService: chunk ${i + 1}/${chunks.length} - ${chunk.length} chars, ~${tokenCount} tokens`
-        );
-    });
-
-    const vectors = await embedBatch(chunks);
-    const avg = averageVectors(vectors);
-    const safe = truncate(pureText);
-
-    setCachedEmbedding(safe, avg);
-
-    return avg;
+	return embedding;
 }
 
 /**
  * Converts an array of text strings into an array of embedding vectors.
- * Texts are sent in batched API requests after semantic chunking.
+ * Texts are sent in a single batched API request.
  * @param {string[]} texts
  * @returns {Promise<number[][]>}
  */
 async function embedBatch(texts) {
-    if (!Array.isArray(texts) || texts.length === 0) {
-        throw new Error('embedBatch() requires a non-empty array of strings');
-    }
-
-    const invalid = texts.findIndex(t => typeof t !== 'string' || t.trim() === '');
-    if (invalid !== -1) {
-        throw new Error(`embedBatch() received an empty or non-string value at index ${invalid}`);
-    }
-
-    const apiKey = process.env.OPENAI_API_KEY;
-    if (!apiKey) {
-        throw new Error('OPENAI_API_KEY environment variable is not set');
-    }
-
-    const allChunks = [];
-    const chunkMap = [];
-
-    for (const [textIdx, text] of texts.entries()) {
-        const pureText = extractPureText(text);
-
-        if (!pureText) {
-            chunkMap.push({ count: 0 });
-            continue;
-        }
-
-        const chunks = splitIntoSemanticChunks(pureText, {
-            maxLen: MAX_CHARS,
-            targetLen: TARGET_CHUNK_CHARS,
-            minLen: MIN_CHUNK_CHARS,
-            overlap: CHUNK_OVERLAP,
-        });
-
-        chunkMap.push({ count: chunks.length });
-        allChunks.push(...chunks);
-
-        if (chunks.length === 1) {
-            const tokenCount = estimateTokens(chunks[0]);
-            winston().info(
-                `[search-agent] embeddingService: batch input ${textIdx + 1}/${texts.length} - 1 chunk, ${chunks[0].length} chars, ~${tokenCount} tokens`
-            );
-        } else {
-            winston().info(
-                `[search-agent] embeddingService: batch input ${textIdx + 1}/${texts.length} - ${chunks.length} chunks`
-            );
-            chunks.forEach((chunk, i) => {
-                const tokenCount = estimateTokens(chunk);
-                winston().info(
-                    `[search-agent] embeddingService:   chunk ${i + 1}/${chunks.length} - ${chunk.length} chars, ~${tokenCount} tokens`
-                );
-            });
-        }
-    }
-
-    winston().verbose(
-        `[search-agent] embeddingService: batch embedding ${allChunks.length} chunk(s) from ${texts.length} input(s)`
-    );
-
-    if (allChunks.length === 0) {
-        return chunkMap.map(({ count }) => (count === 0 ? [] : null));
-    }
-
-    const safeChunks = allChunks.map(chunk => truncate(chunk));
-    const response = await withRetry(() => requestEmbeddings(apiKey, safeChunks));
-
-    winston().verbose(
-        `[search-agent] embeddingService: batch embeddings generated successfully (${safeChunks.length} vector(s))`
-    );
+	if (!Array.isArray(texts) || texts.length === 0) {
+		throw new Error('embedBatch() requires a non-empty array of strings');
+	}
 
-    const vectors = response.data
-        .sort((a, b) => a.index - b.index)
-        .map(item => item.embedding);
+	const invalid = texts.findIndex(t => typeof t !== 'string' || t.trim() === '');
+	if (invalid !== -1) {
+		throw new Error(`embedBatch() received an empty or non-string value at index ${invalid}`);
+	}
 
-    const result = [];
-    let idx = 0;
+	const apiKey = process.env.OPENAI_API_KEY;
+	if (!apiKey) {
+		throw new Error('OPENAI_API_KEY environment variable is not set');
+	}
 
-    for (const { count } of chunkMap) {
-        if (count === 0) {
-            result.push([]);
-        } else if (count === 1) {
-            result.push(vectors[idx]);
-            idx += 1;
-        } else {
-            result.push(averageVectors(vectors.slice(idx, idx + count)));
-            idx += count;
-        }
-    }
+	const safeTexts = texts.map(truncate);
+	winston().verbose(`[search-agent] embeddingService: generating batch embeddings for ${safeTexts.length} text(s)`);
+	const response = await withRetry(() => requestEmbeddings(apiKey, safeTexts));
+	winston().verbose(`[search-agent] embeddingService: batch embeddings generated successfully (${texts.length} vector(s))`);
 
-    return result;
+	// OpenAI returns items sorted by index field, but sort explicitly to be safe
+	return response.data
+		.sort((a, b) => a.index - b.index)
+		.map(item => item.embedding);
 }
 
-module.exports = {
-    embed,
-    embedBatch,
-    extractPureText,
-    splitIntoSemanticChunks,
-};
+module.exports = { embed, embedBatch };

package/services/vectorSearchService.js

@@ -5,409 +5,123 @@ const { embed } = require('./embeddingService');
 const { getAllEmbeddings } = require('./vectorStore');
 
 function winston() {
-    return require.main.require('winston');
+	return require.main.require('winston');
 }
 
 // Fetch this many candidates from Orama — cast a wide net so the AI has enough to choose from
 const TOP_K = 50;
-
-// Absolute minimum similarity — only filters pure noise.
-// Keep this low; the later ranking layers should handle precision.
-const MIN_SCORE = 0.15;
-
-// Rebuild the Orama index after this interval
+// Absolute minimum cosine similarity — only filters pure noise (near-zero similarity).
+// Do NOT raise this: the relevant result often scores lower than irrelevant ones.
+// The AI re-ranker (which reads content) is the precision gate, not this floor.
+const MIN_SCORE = 0.10;
+// Rebuild the Orama index after this interval (mirrors TF-IDF cache TTL)
 const INDEX_TTL_MS = 5 * 60 * 1000;
 
-// Hybrid search configuration
-const VECTOR_SIMILARITY = 0.1;
-const SEARCH_PROPERTIES = ['title', 'category', 'tags', 'content', 'parent_content'];
-const FIELD_BOOSTS = {
-    title: 3.5,
-    tags: 2.8,
-    category: 2.2,
-    content: 1.0,
-    parent_content: 0.8,
-};
-
 let _db = null;
 let _dbTs = 0;
 let _buildPromise = null;
 
-// Finance-heavy Hebrew forum query expansion.
-// These are intentionally conservative: good recall lift without flooding the query.
-const QUERY_EXPANSIONS = {
-    // General finance
-    'מניה': ['מניות', 'נייר ערך', 'שוק ההון', 'בורסה'],
-    'מניות': ['מניה', 'ניירות ערך', 'שוק ההון', 'בורסה'],
-    'אגח': ['אג"ח', 'איגרת חוב', 'איגרות חוב', 'חוב'],
-    'אג"ח': ['אגח', 'איגרת חוב', 'איגרות חוב', 'חוב'],
-    'קרן': ['קרנות', 'קרן נאמנות', 'קרן סל'],
-    'קרנות': ['קרן', 'קרן נאמנות', 'קרן סל'],
-    'קרן סל': ['etf', 'תעודת סל', 'קרן מחקה'],
-    'תעודת סל': ['etf', 'קרן סל', 'קרן מחקה'],
-    'etf': ['קרן סל', 'תעודת סל', 'קרן מחקה'],
-    'מדד': ['מדדים', 'מדד מניות', 'תשואת מדד'],
-    'מדדים': ['מדד', 'מדד מניות', 'תשואת מדד'],
-    'תיק': ['תיק השקעות', 'פיזור', 'החזקות'],
-    'השקעה': ['השקעות', 'להשקיע', 'תיק השקעות'],
-    'השקעות': ['השקעה', 'להשקיע', 'תיק השקעות'],
-    'להשקיע': ['השקעה', 'השקעות', 'תיק השקעות'],
-    'תשואה': ['רווח', 'תשואות', 'רווחיות'],
-    'רווח': ['רווחים', 'תשואה', 'רווחיות'],
-    'הפסד': ['הפסדים', 'ירידה', 'מינוס'],
-    'דיבידנד': ['דיבידנדים', 'חלוקת רווחים'],
-    'מכפיל': ['מכפיל רווח', 'pe', 'p/e'],
-    'pe': ['מכפיל', 'מכפיל רווח', 'p/e'],
-    'p/e': ['מכפיל', 'מכפיל רווח', 'pe'],
-    'מינוף': ['ממונף', 'הלוואה', 'מרגין', 'margin'],
-    'מרגין': ['margin', 'מינוף'],
-    'margin': ['מרגין', 'מינוף'],
-    'סיכון': ['סיכונים', 'תנודתיות', 'חשיפה'],
-    'נזילות': ['נזיל', 'מזומן', 'סחירות'],
-    'סחירות': ['נזילות', 'נזיל'],
-
-    // Tax / regulation
-    'מס': ['מיסוי', 'מסים', 'רשות המסים'],
-    'מיסוי': ['מס', 'מסים', 'רשות המסים'],
-    'מסים': ['מס', 'מיסוי', 'רשות המסים'],
-    'קיזוז': ['קיזוז הפסדים', 'מגן מס'],
-    'דוח': ['דו"ח', 'דיווח', 'טופס'],
-    'דו"ח': ['דוח', 'דיווח', 'טופס'],
-
-    // Savings / pension
-    'פנסיה': ['קרן פנסיה', 'חיסכון פנסיוני', 'קצבה'],
-    'גמל': ['קופת גמל', 'קופ"ג'],
-    'קופג': ['קופת גמל', 'קופ"ג', 'גמל'],
-    'קופ"ג': ['קופת גמל', 'קופג', 'גמל'],
-    'השתלמות': ['קרן השתלמות'],
-    'משכנתא': ['משכנתאות', 'ריבית', 'הלוואת דיור'],
-    'הלוואה': ['הלוואות', 'אשראי', 'מימון'],
-    'אשראי': ['הלוואה', 'הלוואות', 'מימון'],
-
-    // Trading / technical
-    'מסחר': ['טריידינג', 'קניה', 'מכירה', 'פקודה'],
-    'טריידינג': ['מסחר', 'מסחר יומי', 'קניה', 'מכירה'],
-    'שורט': ['short', 'מכירה בחסר'],
-    'short': ['שורט', 'מכירה בחסר'],
-    'לונג': ['long', 'החזקה'],
-    'long': ['לונג', 'החזקה'],
-    'פקודה': ['פקודות', 'לימיט', 'מרקט'],
-    'לימיט': ['limit', 'פקודת לימיט'],
-    'limit': ['לימיט', 'פקודת לימיט'],
-    'מרקט': ['market', 'פקודת שוק'],
-    'market': ['מרקט', 'פקודת שוק'],
-
-    // Crypto
-    'ביטקוין': ['btc', 'קריפטו', 'מטבע דיגיטלי'],
-    'btc': ['ביטקוין', 'קריפטו', 'מטבע דיגיטלי'],
-    'אתריום': ['eth', 'קריפטו', 'מטבע דיגיטלי'],
-    'eth': ['אתריום', 'קריפטו', 'מטבע דיגיטלי'],
-    'קריפטו': ['מטבע דיגיטלי', 'ביטקוין', 'אתריום', 'בלוקציין'],
-    'בלוקציין': ['קריפטו', 'מטבע דיגיטלי'],
-
-    // Hebrew forum / advice intent
-    'מומלץ': ['כדאי', 'המלצה', 'עדיף'],
-    'כדאי': ['מומלץ', 'המלצה', 'עדיף'],
-    'המלצה': ['מומלץ', 'כדאי', 'עדיף'],
-    'בעיה': ['תקלה', 'קושי', 'לא עובד'],
-    'תקלה': ['בעיה', 'לא עובד', 'שגיאה'],
-    'שגיאה': ['תקלה', 'בעיה', 'לא עובד'],
-};
-
-// Common phrase-level expansions that are better handled before token expansion.
-const PHRASE_EXPANSIONS = [
-    {
-        pattern: /\b(?:קרן\s+סל|תעודת\s+סל|קרן\s+מחקה)\b/gi,
-        terms: ['etf', 'קרן סל', 'תעודת סל', 'קרן מחקה'],
-    },
-    {
-        pattern: /\b(?:איגרת\s+חוב|איגרות\s+חוב|אג["׳׳]?\s?ח)\b/gi,
-        terms: ['אגח', 'אג"ח', 'איגרת חוב', 'איגרות חוב'],
-    },
-    {
-        pattern: /\b(?:קופת\s+גמל|קופ["׳׳]?\s?ג)\b/gi,
-        terms: ['קופת גמל', 'קופג', 'קופ"ג', 'גמל'],
-    },
-    {
-        pattern: /\b(?:מכפיל\s+רווח|p\/e|pe)\b/gi,
-        terms: ['מכפיל', 'מכפיל רווח', 'pe', 'p/e'],
-    },
-    {
-        pattern: /\b(?:מכירה\s+בחסר|short)\b/gi,
-        terms: ['שורט', 'short', 'מכירה בחסר'],
-    },
-];
-
-// Generic filler words to ignore for lexical expansion
-const STOP_WORDS = new Set([
-    'של', 'על', 'עם', 'בלי', 'גם', 'או', 'אם', 'אבל', 'כי', 'זה', 'זאת', 'זו',
-    'יש', 'אין', 'אני', 'אתה', 'את', 'הוא', 'היא', 'הם', 'הן', 'אנחנו', 'מה',
-    'איך', 'למה', 'מתי', 'איפה', 'האם', 'כל', 'עוד', 'כמו', 'רק', 'מאוד', 'פחות',
-    'יותר', 'אחרי', 'לפני', 'תוך', 'דרך', 'לגבי', 'בנוגע', 'בשביל', 'מול',
-]);
-
-function normalizeHebrew(text) {
-	if(!text) return text;
-
-	// Remove common prefixes
-	const prefixes = ['ה', 'ו', 'ב', 'ל', 'מ', 'ש', 'כ'];
-    for (const prefix of prefixes) {
-        if (text.startsWith(prefix) && text.length > 3) {
-            text = text.slice(1);
-            break;
-        }
-    }
-
-    // Remove common plural suffixes
-    const pluralSuffixes = ['ים', 'ות'];
-    for (const suffix of pluralSuffixes) {
-        if (text.endsWith(suffix) && text.length > 3) {
-            text = text.slice(0, -suffix.length);
-            break;
-        }
-    }
-
-    return String(text || '')
-        // remove niqqud / cantillation
-        .replace(/[\u0591-\u05C7]/g, '')
-        // normalize Hebrew punctuation variants
-        .replace(/[׳']/g, '\'')
-        .replace(/[״"]/g, '"')
-        // collapse whitespace
-        .replace(/\s+/g, ' ')
-        .trim();
-}
-
-function normalizeToken(token) {
-    return normalizeHebrew(token)
-        .toLowerCase()
-        .replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, '');
-}
-
-function uniqueTerms(terms, maxTerms = 24) {
-    const seen = new Set();
-    const out = [];
-
-    for (const raw of terms) {
-        const term = normalizeHebrew(raw).trim();
-        if (!term) continue;
-
-        const key = term.toLowerCase();
-        if (seen.has(key)) continue;
-
-        seen.add(key);
-        out.push(term);
-
-        if (out.length >= maxTerms) break;
-    }
-
-    return out;
-}
-
-function expandQuery(query) {
-    const normalized = normalizeHebrew(query);
-    const expanded = [normalized];
-
-    for (const phraseRule of PHRASE_EXPANSIONS) {
-        if (phraseRule.pattern.test(normalized)) {
-            expanded.push(...phraseRule.terms);
-        }
-        phraseRule.pattern.lastIndex = 0;
-    }
-
-    const tokens = normalized
-        .split(/[\s,/|()[\]{}:;!?]+/)
-        .map(normalizeToken)
-        .filter(Boolean)
-        .filter(token => !STOP_WORDS.has(token));
-
-    for (const token of tokens) {
-        expanded.push(token);
-
-        const synonyms = QUERY_EXPANSIONS[token];
-        if (synonyms) {
-            expanded.push(...synonyms);
-        }
-
-        // A little morphology help for Hebrew singular/plural and abbreviations
-        if (token.endsWith('ים') && token.length > 3) {
-            expanded.push(token.slice(0, -2));
-        }
-        if (token.endsWith('ות') && token.length > 3) {
-            expanded.push(token.slice(0, -2));
-        }
-        if (token.endsWith('ה') && token.length > 2) {
-            expanded.push(token.slice(0, -1));
-        }
-    }
-
-    const terms = uniqueTerms(expanded, 24);
-
-    return {
-        original: query,
-        normalized,
-        terms,
-        // Orama lexical search receives one expanded term string
-        term: terms.join(' '),
-    };
-}
-
-function coerceString(value) {
-    if (value == null) return '';
-
-    if (Array.isArray(value)) {
-        return value
-            .map(v => coerceString(v))
-            .filter(Boolean)
-            .join(', ');
-    }
-
-    return String(value).trim();
-}
-
-function buildDocument(row) {
-    return {
-        id: String(row.post_id),
-        post_id: row.post_id,
-        topic_id: row.topic_id,
-        title: coerceString(row.title),
-        category: coerceString(row.category),
-        tags: coerceString(row.tags),
-        parent_content: coerceString(row.parent_content),
-        content: coerceString(row.content),
-        embedding: row.embedding,
-    };
-}
-
 async function buildIndex() {
-    const storedEmbeddings = await getAllEmbeddings();
-
-    // Detect dimension from data; fall back to 1536 (text-embedding-3-small default)
-    const dimensions = storedEmbeddings.length > 0
-        ? storedEmbeddings[0].embedding.length
-        : 1536;
-
-    const db = await create({
-        schema: {
-            post_id: 'number',
-            topic_id: 'number',
-            title: 'string',
-            category: 'string',
-            tags: 'string',
-            parent_content: 'string',
-            content: 'string',
-            embedding: `vector[${dimensions}]`,
-        },
-    });
-
-    if (storedEmbeddings.length > 0) {
-        await insertMultiple(db, storedEmbeddings.map(buildDocument));
-    }
-
-    winston().info(
-        `[search-agent] vectorSearchService: Orama index built with ${storedEmbeddings.length} document(s)`
-    );
-
-    return db;
+	const storedEmbeddings = await getAllEmbeddings();
+
+	// Detect dimension from data; fall back to 1536 (text-embedding-3-small default)
+	const dimensions = storedEmbeddings.length > 0
+		? storedEmbeddings[0].embedding.length
+		: 1536;
+
+	const db = await create({
+		schema: {
+			post_id: 'number',
+			topic_id: 'number',
+			content: 'string',
+			embedding: `vector[${dimensions}]`,
+		},
+	});
+
+	if (storedEmbeddings.length > 0) {
+		await insertMultiple(db, storedEmbeddings.map(e => ({
+			id: String(e.post_id),
+			post_id: e.post_id,
+			topic_id: e.topic_id,
+			content: e.content,
+			embedding: e.embedding,
+		})));
+	}
+
+	winston().info(`[search-agent] vectorSearchService: Orama index built with ${storedEmbeddings.length} document(s)`);
+	return db;
 }
 
 async function getDb() {
-    const now = Date.now();
-    if (_db && (now - _dbTs) < INDEX_TTL_MS) {
-        return _db;
-    }
-
-    if (_buildPromise) {
-        return _buildPromise;
-    }
-
-    _buildPromise = buildIndex()
-        .then((db) => {
-            _db = db;
-            _dbTs = Date.now();
-            _buildPromise = null;
-            return db;
-        })
-        .catch((err) => {
-            _buildPromise = null;
-            throw err;
-        });
-
-    return _buildPromise;
+	const now = Date.now();
+	if (_db && (now - _dbTs) < INDEX_TTL_MS) {
+		return _db;
+	}
+
+	if (_buildPromise) {
+		return _buildPromise;
+	}
+
+	_buildPromise = buildIndex().then((db) => {
+		_db = db;
+		_dbTs = Date.now();
+		_buildPromise = null;
+		return db;
+	}).catch((err) => {
+		_buildPromise = null;
+		throw err;
+	});
+
+	return _buildPromise;
 }
 
 /** Invalidate the in-memory Orama index (e.g. after new embeddings are saved). */
 function invalidateIndex() {
-    _db = null;
-    _dbTs = 0;
-    winston().info('[search-agent] vectorSearchService: Orama index invalidated');
+	_db = null;
+	_dbTs = 0;
+	winston().info('[search-agent] vectorSearchService: Orama index invalidated');
 }
 
 /**
- * Performs hybrid search against stored post embeddings using:
- * 1. vector similarity on the original query embedding
- * 2. lexical search on an expanded Hebrew query
- * 3. field boosts to favor title/tags/category matches
+ * Performs semantic search against stored post embeddings using Orama vector search.
  *
  * @param {string} query - The search query string.
- * @param {number} limit - Max results to return.
- * @returns {Promise<Array<{ topic_id: number, post_id: number, title: string, category: string, tags: string, content: string, score: number }>>}
+ * @returns {Promise<Array<{ topic_id: number, post_id: number, content: string, score: number }>>}
+ *   Top results sorted by cosine similarity descending.
  */
 async function search(query, limit = TOP_K) {
-    if (typeof query !== 'string' || query.trim() === '') {
-        throw new Error('search() requires a non-empty query string');
-    }
-
-    const trimmed = query.trim();
-    const expanded = expandQuery(trimmed);
-
-    winston().verbose(
-        `[search-agent] vectorSearchService: running Orama hybrid search for "${trimmed}" (expanded="${expanded.term}")`
-    );
-
-    const [queryEmbedding, db] = await Promise.all([
-        // Keep the embedding on the original query only.
-        // Expansion is mainly for lexical recall, especially in Hebrew forum language.
-        embed(trimmed),
-        getDb(),
-    ]);
-
-    const results = await oramaSearch(db, {
-        mode: 'hybrid',
-        term: expanded.term,
-        properties: SEARCH_PROPERTIES,
-        boost: FIELD_BOOSTS,
-        vector: {
-            value: queryEmbedding,
-            property: 'embedding',
-        },
-        limit,
-        similarity: VECTOR_SIMILARITY,
-        includeVectors: false,
-    });
-
-    const hits = Array.isArray(results && results.hits) ? results.hits : [];
-
-    winston().verbose(`[search-agent] vectorSearchService: Orama returned ${hits.length} hit(s)`);
-
-    const filtered = hits.filter(hit => typeof hit.score === 'number' && hit.score >= MIN_SCORE);
-
-    winston().verbose(
-        `[search-agent] vectorSearchService: ${filtered.length}/${hits.length} hit(s) passed noise floor (MIN_SCORE=${MIN_SCORE})`
-    );
-
-    return filtered.map(hit => ({
-        topic_id: hit.document.topic_id,
-        post_id: hit.document.post_id,
-        title: hit.document.title || '',
-        category: hit.document.category || '',
-        tags: hit.document.tags || '',
-        content: hit.document.content,
-        score: hit.score,
-    }));
+	if (typeof query !== 'string' || query.trim() === '') {
+		throw new Error('search() requires a non-empty query string');
+	}
+
+	winston().verbose(`[search-agent] vectorSearchService: running Orama vector search for "${query.trim()}"`);
+
+	const [queryEmbedding, db] = await Promise.all([
+		embed(query),
+		getDb(),
+	]);
+
+	const results = await oramaSearch(db, {
+		mode: 'vector',
+		vector: { value: queryEmbedding, property: 'embedding' },
+		limit,
+		similarity: 0.1, 
+		includeVectors: false,
+	});
+
+	winston().verbose(`[search-agent] vectorSearchService: Orama returned ${results.hits.length} hit(s)`);
+
+	const filtered = results.hits.filter(hit => hit.score >= MIN_SCORE);
+	winston().verbose(
+		`[search-agent] vectorSearchService: ${filtered.length}/${results.hits.length} hit(s) passed noise floor (MIN_SCORE=${MIN_SCORE})`
+	);
+
+	return filtered.map(hit => ({
+		topic_id: hit.document.topic_id,
+		post_id: hit.document.post_id,
+		content: hit.document.content,
+		score: hit.score,
+	}));
 }
 
-module.exports = {
-    search,
-    invalidateIndex,
-};
+module.exports = { search, invalidateIndex };

package/test/testCosine.js

@@ -0,0 +1,15 @@
+const { cosineSimilarity } = require('../lib/cosineSimilarity');
+
+function testCosine() {
+  console.log("Testing cosine similarity...");
+
+  const a = cosineSimilarity([1, 0], [1, 0]);
+  const b = cosineSimilarity([1, 0], [0, 1]);
+
+  if (a < 0.9) throw new Error("Expected high similarity");
+  if (b > 0.1) throw new Error("Expected low similarity");
+
+  console.log("✅ Cosine OK");
+}
+
+module.exports = testCosine;