nodebb-plugin-search-agent 0.0.937 → 0.0.939

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nodebb-plugin-search-agent",
-  "version": "0.0.937",
+  "version": "0.0.939",
   "description": "NodeBB plugin that adds a floating chat assistant to help users find relevant forum topics using TF-IDF text similarity",
   "main": "library.js",
   "author": "Racheli Bayfus",

package/services/embeddingService.js

@@ -1,28 +1,9 @@
-// Remove images, files, and non-text content from input
-function extractPureText(text) {
-	if (typeof text !== 'string') return '';
-	// Remove Markdown images: ![alt](url)
-	let cleaned = text.replace(/!\[[^\]]*\]\([^)]*\)/g, '');
-	// Remove HTML <img> tags
-	cleaned = cleaned.replace(/<img\b[^>]*>/gi, '');
-	// Remove links to files/images (common extensions)
-	cleaned = cleaned.replace(/https?:\/\/(\S+\.(jpg|jpeg|png|gif|bmp|svg|webp|pdf|docx?|xlsx?|pptx?|zip|rar|7z|tar|gz|mp3|mp4|avi|mov|wmv|flv|mkv|ogg|wav|exe|bin|apk|ipa|dmg|iso|csv|json|xml|yml|yaml|psd|ai|eps|ttf|otf|woff2?))(\?\S*)?/gi, '');
-	// Remove Markdown file links: [desc](url.ext)
-	cleaned = cleaned.replace(/\[[^\]]*\]\([^)]*\.(jpg|jpeg|png|gif|bmp|svg|webp|pdf|docx?|xlsx?|pptx?|zip|rar|7z|tar|gz|mp3|mp4|avi|mov|wmv|flv|mkv|ogg|wav|exe|bin|apk|ipa|dmg|iso|csv|json|xml|yml|yaml|psd|ai|eps|ttf|otf|woff2?)\)/gi, '');
-	// Remove any remaining <a href=...> tags to files
-	cleaned = cleaned.replace(/<a\b[^>]*href=["']?[^"'>]+\.(jpg|jpeg|png|gif|bmp|svg|webp|pdf|docx?|xlsx?|pptx?|zip|rar|7z|tar|gz|mp3|mp4|avi|mov|wmv|flv|mkv|ogg|wav|exe|bin|apk|ipa|dmg|iso|csv|json|xml|yml|yaml|psd|ai|eps|ttf|otf|woff2?)[^>]*>.*?<\/a>/gi, '');
-	// Remove any remaining HTML tags
-	cleaned = cleaned.replace(/<[^>]+>/g, ' ');
-	// Remove extra whitespace
-	cleaned = cleaned.replace(/[ \t]+/g, ' ').replace(/\n{2,}/g, '\n').trim();
-	return cleaned;
-}
 'use strict';
 
 const https = require('https');
 
 function winston() {
-	return require.main.require('winston');
+    return require.main.require('winston');
 }
 
 const OPENAI_EMBEDDINGS_HOSTNAME = 'api.openai.com';
@@ -30,48 +11,236 @@ const OPENAI_EMBEDDINGS_PATH = '/v1/embeddings';
 const EMBEDDING_MODEL = 'text-embedding-3-small';
 const MAX_RETRIES = 3;
 const RETRY_DELAY_MS = 500;
-// text-embedding-3-small supports 8 192 tokens.
-// Hebrew/non-ASCII text tokenizes at ~1.5–2 chars/token (UTF-8 multibyte).
-// Using 1.5 chars/token worst-case: 8000 tokens × 1.5 = 12 000 chars — gives a safe margin.
 
+// text-embedding-3-small supports 8,192 tokens.
+// Conservative char limits help avoid token overflows, especially for non-ASCII text.
 const MAX_CHARS = 10000;
-const CHUNK_OVERLAP = 200; // chars to overlap between chunks for context
-
-// Split a long string into chunks of maxLen, with optional overlap
-function splitIntoChunks(text, maxLen = MAX_CHARS, overlap = CHUNK_OVERLAP) {
-	if (text.length <= maxLen) return [text];
-	const chunks = [];
-	let i = 0;
-	while (i < text.length) {
-		const chunk = text.slice(i, i + maxLen);
-		chunks.push(chunk);
-		if (i + maxLen >= text.length) break;
-		i += maxLen - overlap;
-	}
-	return chunks;
+const CHUNK_OVERLAP = 300;
+const MIN_CHUNK_CHARS = 500;
+const TARGET_CHUNK_CHARS = 2200;
+
+// ─── Text cleanup ─────────────────────────────────────────────────────────────
+
+function extractPureText(text) {
+    if (typeof text !== 'string') return '';
+
+    // Remove Markdown images: ![alt](url)
+    let cleaned = text.replace(/!\[[^\]]*\]\([^)]*\)/g, '');
+
+    // Remove HTML <img> tags
+    cleaned = cleaned.replace(/<img\b[^>]*>/gi, '');
+
+    // Remove links to files/images (common extensions)
+    cleaned = cleaned.replace(
+        /https?:\/\/(\S+\.(jpg|jpeg|png|gif|bmp|svg|webp|pdf|docx?|xlsx?|pptx?|zip|rar|7z|tar|gz|mp3|mp4|avi|mov|wmv|flv|mkv|ogg|wav|exe|bin|apk|ipa|dmg|iso|csv|json|xml|yml|yaml|psd|ai|eps|ttf|otf|woff2?))(\?\S*)?/gi,
+        ''
+    );
+
+    // Remove Markdown file links: [desc](url.ext)
+    cleaned = cleaned.replace(
+        /\[[^\]]*\]\([^)]*\.(jpg|jpeg|png|gif|bmp|svg|webp|pdf|docx?|xlsx?|pptx?|zip|rar|7z|tar|gz|mp3|mp4|avi|mov|wmv|flv|mkv|ogg|wav|exe|bin|apk|ipa|dmg|iso|csv|json|xml|yml|yaml|psd|ai|eps|ttf|otf|woff2?)\)/gi,
+        ''
+    );
+
+    // Remove any remaining <a href=...> tags to files
+    cleaned = cleaned.replace(
+        /<a\b[^>]*href=["']?[^"'>]+\.(jpg|jpeg|png|gif|bmp|svg|webp|pdf|docx?|xlsx?|pptx?|zip|rar|7z|tar|gz|mp3|mp4|avi|mov|wmv|flv|mkv|ogg|wav|exe|bin|apk|ipa|dmg|iso|csv|json|xml|yml|yaml|psd|ai|eps|ttf|otf|woff2?)[^>]*>.*?<\/a>/gi,
+        ''
+    );
+
+    // Remove any remaining HTML tags
+    cleaned = cleaned.replace(/<[^>]+>/g, ' ');
+
+    // Remove extra whitespace
+    cleaned = cleaned
+        .replace(/[ \t]+/g, ' ')
+        .replace(/\n{2,}/g, '\n')
+        .trim();
+
+    return cleaned;
 }
 
-// Average a list of vectors (arrays of numbers)
+function normalizeWhitespace(text) {
+    return text
+        .replace(/\r\n/g, '\n')
+        .replace(/[ \t]+/g, ' ')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+}
+
+function splitIntoBlocks(text) {
+    const normalized = normalizeWhitespace(text);
+
+    const rawBlocks = normalized
+        .split(/\n{2,}|(?=^#{1,6}\s)|(?=^\s*[-*+]\s)|(?=^\s*\d+\.\s)|(?=^\s*>\s)|(?=^```)/gm)
+        .map(block => block.trim())
+        .filter(Boolean);
+
+    return rawBlocks;
+}
+
+function splitLargeBlock(block, maxLen = MAX_CHARS) {
+    if (block.length <= maxLen) return [block];
+
+    const sentences = block.match(/[^.!?\n]+[.!?\n]+|[^.!?\n]+$/g) || [block];
+    const chunks = [];
+    let current = '';
+
+    for (const sentence of sentences) {
+        const s = sentence.trim();
+        if (!s) continue;
+
+        if ((current + ' ' + s).trim().length <= maxLen) {
+            current = current ? `${current} ${s}` : s;
+            continue;
+        }
+
+        if (current) {
+            chunks.push(current);
+            current = '';
+        }
+
+        // Fallback for very long sentence
+        if (s.length > maxLen) {
+            let i = 0;
+            while (i < s.length) {
+                chunks.push(s.slice(i, i + maxLen).trim());
+                i += maxLen;
+            }
+        } else {
+            current = s;
+        }
+    }
+
+    if (current) chunks.push(current);
+
+    return chunks;
+}
+
+function buildOverlapPrefix(prevChunk, overlapChars = CHUNK_OVERLAP) {
+    if (!prevChunk) return '';
+    return prevChunk.slice(Math.max(0, prevChunk.length - overlapChars)).trim();
+}
+
+function splitIntoSemanticChunks(
+    text,
+    {
+        maxLen = MAX_CHARS,
+        targetLen = TARGET_CHUNK_CHARS,
+        minLen = MIN_CHUNK_CHARS,
+        overlap = CHUNK_OVERLAP,
+    } = {}
+) {
+    if (!text) return [];
+    if (text.length <= maxLen) return [text];
+
+    const blocks = splitIntoBlocks(text).flatMap(block => splitLargeBlock(block, maxLen));
+
+    const chunks = [];
+    let current = '';
+
+    for (const block of blocks) {
+        const next = current ? `${current}\n\n${block}` : block;
+
+        if (next.length <= targetLen || current.length < minLen) {
+            if (next.length <= maxLen) {
+                current = next;
+                continue;
+            }
+        }
+
+        if (current) {
+            chunks.push(current.trim());
+        }
+
+        current = block;
+    }
+
+    if (current) {
+        chunks.push(current.trim());
+    }
+
+    const withOverlap = chunks.map((chunk, i) => {
+        if (i === 0) return chunk;
+
+        const prefix = buildOverlapPrefix(chunks[i - 1], overlap);
+        const merged = prefix ? `${prefix}\n\n${chunk}` : chunk;
+
+        return merged.length <= maxLen ? merged : merged.slice(merged.length - maxLen);
+    });
+
+    const finalChunks = [];
+    for (const chunk of withOverlap) {
+        if (
+            finalChunks.length > 0 &&
+            chunk.length < minLen &&
+            finalChunks[finalChunks.length - 1].length + 2 + chunk.length <= maxLen
+        ) {
+            finalChunks[finalChunks.length - 1] += `\n\n${chunk}`;
+        } else {
+            finalChunks.push(chunk);
+        }
+    }
+
+    return finalChunks;
+}
+
+// ─── Vector helpers ───────────────────────────────────────────────────────────
+
 function averageVectors(vectors) {
-	if (!vectors.length) return [];
-	const len = vectors[0].length;
-	const sum = new Array(len).fill(0);
-	for (const v of vectors) {
-		for (let i = 0; i < len; i++) sum[i] += v[i];
-	}
-	return sum.map(x => x / vectors.length);
+    if (!vectors.length) return [];
+    const len = vectors[0].length;
+    const sum = new Array(len).fill(0);
+
+    for (const v of vectors) {
+        for (let i = 0; i < len; i++) {
+            sum[i] += v[i];
+        }
+    }
+
+    return sum.map(x => x / vectors.length);
+}
+
+function truncate(text) {
+    return text.length > MAX_CHARS ? text.slice(0, MAX_CHARS) : text;
+}
+
+function estimateTokens(str) {
+    const ascii = /^[\x00-\x7F]*$/.test(str);
+    return ascii ? Math.ceil(str.length / 4) : Math.ceil(str.length / 1.5);
 }
 
 // ─── Embedding cache ──────────────────────────────────────────────────────────
-// Avoids calling the embeddings API for the same text within a session.
-// HyDE output varies, so the biggest wins come from repeated identical queries.
+
 const _embedCache = new Map();
 const EMBED_CACHE_MAX = 500;
 
-function truncate(text) {
-	return text.length > MAX_CHARS ? text.slice(0, MAX_CHARS) : text;
+function getCachedEmbedding(key) {
+    if (!_embedCache.has(key)) return null;
+
+    const value = _embedCache.get(key);
+
+    // Refresh LRU-ish order
+    _embedCache.delete(key);
+    _embedCache.set(key, value);
+
+    return value;
 }
 
+function setCachedEmbedding(key, value) {
+    if (_embedCache.has(key)) {
+        _embedCache.delete(key);
+    }
+
+    while (_embedCache.size >= EMBED_CACHE_MAX) {
+        _embedCache.delete(_embedCache.keys().next().value);
+    }
+
+    _embedCache.set(key, value);
+}
+
+// ─── OpenAI request helpers ───────────────────────────────────────────────────
+
 /**
  * Performs an HTTPS POST request to the OpenAI embeddings endpoint.
  * @param {string} apiKey
@@ -79,203 +248,246 @@ function truncate(text) {
  * @returns {Promise<object>} Parsed JSON response body
  */
 function requestEmbeddings(apiKey, input) {
-	return new Promise((resolve, reject) => {
-		const body = JSON.stringify({ model: EMBEDDING_MODEL, input });
-		const options = {
-			hostname: OPENAI_EMBEDDINGS_HOSTNAME,
-			path: OPENAI_EMBEDDINGS_PATH,
-			method: 'POST',
-			headers: {
-				'Content-Type': 'application/json',
-				'Authorization': `Bearer ${apiKey}`,
-				'Content-Length': Buffer.byteLength(body),
-			},
-		};
-
-		const req = https.request(options, (res) => {
-			const chunks = [];
-			res.on('data', chunk => chunks.push(chunk));
-			res.on('end', () => {
-				let parsed;
-				try {
-					parsed = JSON.parse(Buffer.concat(chunks).toString('utf8'));
-				} catch (e) {
-					return reject(new Error(`Failed to parse OpenAI response: ${e.message}`));
-				}
-
-				if (res.statusCode >= 400) {
-					const message = (parsed.error && parsed.error.message) || `HTTP ${res.statusCode}`;
-					return reject(new Error(`OpenAI API error: ${message}`));
-				}
-
-				resolve(parsed);
-			});
-		});
-
-		req.on('error', err => reject(new Error(`Network error calling OpenAI: ${err.message}`)));
-		req.write(body);
-		req.end();
-	});
+    return new Promise((resolve, reject) => {
+        const body = JSON.stringify({ model: EMBEDDING_MODEL, input });
+
+        const options = {
+            hostname: OPENAI_EMBEDDINGS_HOSTNAME,
+            path: OPENAI_EMBEDDINGS_PATH,
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${apiKey}`,
+                'Content-Length': Buffer.byteLength(body),
+            },
+        };
+
+        const req = https.request(options, res => {
+            const chunks = [];
+
+            res.on('data', chunk => chunks.push(chunk));
+
+            res.on('end', () => {
+                let parsed;
+                try {
+                    parsed = JSON.parse(Buffer.concat(chunks).toString('utf8'));
+                } catch (e) {
+                    return reject(new Error(`Failed to parse OpenAI response: ${e.message}`));
+                }
+
+                if (res.statusCode >= 400) {
+                    const message = (parsed.error && parsed.error.message) || `HTTP ${res.statusCode}`;
+                    return reject(new Error(`OpenAI API error: ${message}`));
+                }
+
+                resolve(parsed);
+            });
+        });
+
+        req.on('error', err => reject(new Error(`Network error calling OpenAI: ${err.message}`)));
+        req.write(body);
+        req.end();
+    });
 }
 
 /**
  * Retries an async operation up to maxRetries times with exponential back-off.
- * @param {Function} fn - Async function to retry
+ * @param {Function} fn
  * @param {number} retries
  * @returns {Promise<*>}
  */
 async function withRetry(fn, retries = MAX_RETRIES) {
-	let lastError;
-	for (let attempt = 1; attempt <= retries; attempt++) {
-		try {
-			return await fn();
-		} catch (err) {
-			lastError = err;
-			if (attempt < retries) {
-				winston().warn(`[search-agent] embeddingService: attempt ${attempt} failed (${err.message}), retrying in ${RETRY_DELAY_MS * attempt} ms…`);
-				await new Promise(resolve => setTimeout(resolve, RETRY_DELAY_MS * attempt));
-			}
-		}
-	}
-	throw lastError;
+    let lastError;
+
+    for (let attempt = 1; attempt <= retries; attempt++) {
+        try {
+            return await fn();
+        } catch (err) {
+            lastError = err;
+
+            if (attempt < retries) {
+                winston().warn(
+                    `[search-agent] embeddingService: attempt ${attempt} failed (${err.message}), retrying in ${RETRY_DELAY_MS * attempt} ms...`
+                );
+                await new Promise(resolve => setTimeout(resolve, RETRY_DELAY_MS * attempt));
+            }
+        }
+    }
+
+    throw lastError;
 }
 
+// ─── Public API ───────────────────────────────────────────────────────────────
+
 /**
  * Converts a single text string into an embedding vector.
  * @param {string} text
  * @returns {Promise<number[]>}
  */
 async function embed(text) {
-	if (typeof text !== 'string' || text.trim() === '') {
-		throw new Error('embed() requires a non-empty string');
-	}
-
-	const apiKey = process.env.OPENAI_API_KEY;
-	if (!apiKey) {
-		throw new Error('OPENAI_API_KEY environment variable is not set');
-	}
-
-	       // Remove non-text content
-	       const pureText = extractPureText(text);
-	       if (!pureText) {
-		       throw new Error('embed() received no usable text after filtering');
-	       }
-	       // Split into chunks if too long
-	       const chunks = splitIntoChunks(pureText, MAX_CHARS, CHUNK_OVERLAP);
-	       // Estimate tokens (roughly 1.5 chars/token for non-ASCII, 4 chars/token for ASCII)
-	       const estimateTokens = (str) => {
-		       // If mostly ASCII, use 4 chars/token, else 1.5
-		       const ascii = /^[\x00-\x7F]*$/.test(str);
-		       return ascii ? Math.ceil(str.length / 4) : Math.ceil(str.length / 1.5);
-	       };
-	       if (chunks.length === 1) {
-		       const safe = truncate(text);
-		       if (_embedCache.has(safe)) {
-			       winston().verbose('[search-agent] embeddingService: embedding cache hit');
-			       return _embedCache.get(safe);
-		       }
-		       const tokenCount = estimateTokens(safe);
-		       winston().info(`[search-agent] embeddingService: generating embedding for text (${safe.length} chars, ~${tokenCount} tokens)`);
-		       const response = await withRetry(() => requestEmbeddings(apiKey, safe));
-		       winston().verbose('[search-agent] embeddingService: embedding generated successfully');
-		       const embedding = response.data[0].embedding;
-		       if (_embedCache.size >= EMBED_CACHE_MAX) {
-			       _embedCache.delete(_embedCache.keys().next().value);
-		       }
-		       _embedCache.set(safe, embedding);
-		       return embedding;
-	       } else {
-		       // For multi-chunk, embed all and average
-		       winston().info(`[search-agent] embeddingService: splitting long text into ${chunks.length} chunks for embedding`);
-		       chunks.forEach((chunk, i) => {
-			       const tokenCount = estimateTokens(chunk);
-			       winston().info(`[search-agent] embeddingService: chunk ${i+1}/${chunks.length} — ${chunk.length} chars, ~${tokenCount} tokens`);
-		       });
-		       const vectors = await embedBatch(chunks);
-		       const avg = averageVectors(vectors);
-		       // Optionally cache the average for the full text
-		       const safe = truncate(text);
-		       if (_embedCache.size >= EMBED_CACHE_MAX) {
-			       _embedCache.delete(_embedCache.keys().next().value);
-		       }
-		       _embedCache.set(safe, avg);
-		       return avg;
-	       }
+    if (typeof text !== 'string' || text.trim() === '') {
+        throw new Error('embed() requires a non-empty string');
+    }
+
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+        throw new Error('OPENAI_API_KEY environment variable is not set');
+    }
+
+    const pureText = extractPureText(text);
+    if (!pureText) {
+        throw new Error('embed() received no usable text after filtering');
+    }
+
+    const chunks = splitIntoSemanticChunks(pureText, {
+        maxLen: MAX_CHARS,
+        targetLen: TARGET_CHUNK_CHARS,
+        minLen: MIN_CHUNK_CHARS,
+        overlap: CHUNK_OVERLAP,
+    });
+
+    if (chunks.length === 1) {
+        const safe = truncate(pureText);
+        const cached = getCachedEmbedding(safe);
+        if (cached) {
+            winston().verbose('[search-agent] embeddingService: embedding cache hit');
+            return cached;
+        }
+
+        const tokenCount = estimateTokens(safe);
+        winston().info(
+            `[search-agent] embeddingService: generating embedding for text (${safe.length} chars, ~${tokenCount} tokens)`
+        );
+
+        const response = await withRetry(() => requestEmbeddings(apiKey, safe));
+        const embedding = response.data[0].embedding;
+
+        winston().verbose('[search-agent] embeddingService: embedding generated successfully');
+        setCachedEmbedding(safe, embedding);
+
+        return embedding;
+    }
+
+    winston().info(
+        `[search-agent] embeddingService: splitting long text into ${chunks.length} semantic chunks for embedding`
+    );
+
+    chunks.forEach((chunk, i) => {
+        const tokenCount = estimateTokens(chunk);
+        winston().info(
+            `[search-agent] embeddingService: chunk ${i + 1}/${chunks.length} - ${chunk.length} chars, ~${tokenCount} tokens`
+        );
+    });
+
+    const vectors = await embedBatch(chunks);
+    const avg = averageVectors(vectors);
+    const safe = truncate(pureText);
+
+    setCachedEmbedding(safe, avg);
+
+    return avg;
 }
 
 /**
  * Converts an array of text strings into an array of embedding vectors.
- * Texts are sent in a single batched API request.
+ * Texts are sent in batched API requests after semantic chunking.
  * @param {string[]} texts
  * @returns {Promise<number[][]>}
  */
 async function embedBatch(texts) {
-	if (!Array.isArray(texts) || texts.length === 0) {
-		throw new Error('embedBatch() requires a non-empty array of strings');
-	}
-
-	const invalid = texts.findIndex(t => typeof t !== 'string' || t.trim() === '');
-	if (invalid !== -1) {
-		throw new Error(`embedBatch() received an empty or non-string value at index ${invalid}`);
-	}
-
-	const apiKey = process.env.OPENAI_API_KEY;
-	if (!apiKey) {
-		throw new Error('OPENAI_API_KEY environment variable is not set');
-	}
-
-	       // For each text, filter to pure text, then split and average embeddings
-	       const allChunks = [];
-	       const chunkMap = [];
-	       // Estimate tokens (roughly 1.5 chars/token for non-ASCII, 4 chars/token for ASCII)
-	       const estimateTokens = (str) => {
-		       const ascii = /^[\x00-\x7F]*$/.test(str);
-		       return ascii ? Math.ceil(str.length / 4) : Math.ceil(str.length / 1.5);
-	       };
-	       for (const [textIdx, text] of texts.entries()) {
-		       const pureText = extractPureText(text);
-		       if (!pureText) {
-			       chunkMap.push({ count: 0 });
-			       continue;
-		       }
-		       const chunks = splitIntoChunks(pureText, MAX_CHARS, CHUNK_OVERLAP);
-		       chunkMap.push({ count: chunks.length });
-		       allChunks.push(...chunks);
-		       if (chunks.length === 1) {
-			       const tokenCount = estimateTokens(chunks[0]);
-			       winston().info(`[search-agent] embeddingService: batch input ${textIdx+1}/${texts.length} — 1 chunk, ${chunks[0].length} chars, ~${tokenCount} tokens`);
-		       } else {
-			       winston().info(`[search-agent] embeddingService: batch input ${textIdx+1}/${texts.length} — ${chunks.length} chunks`);
-			       chunks.forEach((chunk, i) => {
-				       const tokenCount = estimateTokens(chunk);
-				       winston().info(`[search-agent] embeddingService:   chunk ${i+1}/${chunks.length} — ${chunk.length} chars, ~${tokenCount} tokens`);
-			       });
-		       }
-	       }
-	       winston().verbose(`[search-agent] embeddingService: batch embedding ${allChunks.length} chunk(s) from ${texts.length} input(s)`);
-	if (allChunks.length === 0) {
-		// All texts were filtered out
-		return chunkMap.map(({ count }) => count === 0 ? [] : null);
-	}
-	const safeChunks = allChunks.map(truncate);
-	const response = await withRetry(() => requestEmbeddings(apiKey, safeChunks));
-	winston().verbose(`[search-agent] embeddingService: batch embeddings generated successfully (${safeChunks.length} vector(s))`);
-	const vectors = response.data.sort((a, b) => a.index - b.index).map(item => item.embedding);
-	// Recombine chunk embeddings for each original text
-	const result = [];
-	let idx = 0;
-	for (const { count } of chunkMap) {
-		if (count === 0) {
-			result.push([]); // No usable text
-		} else if (count === 1) {
-			result.push(vectors[idx]);
-			idx += 1;
-		} else {
-			result.push(averageVectors(vectors.slice(idx, idx + count)));
-			idx += count;
-		}
-	}
-	return result;
+    if (!Array.isArray(texts) || texts.length === 0) {
+        throw new Error('embedBatch() requires a non-empty array of strings');
+    }
+
+    const invalid = texts.findIndex(t => typeof t !== 'string' || t.trim() === '');
+    if (invalid !== -1) {
+        throw new Error(`embedBatch() received an empty or non-string value at index ${invalid}`);
+    }
+
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+        throw new Error('OPENAI_API_KEY environment variable is not set');
+    }
+
+    const allChunks = [];
+    const chunkMap = [];
+
+    for (const [textIdx, text] of texts.entries()) {
+        const pureText = extractPureText(text);
+
+        if (!pureText) {
+            chunkMap.push({ count: 0 });
+            continue;
+        }
+
+        const chunks = splitIntoSemanticChunks(pureText, {
+            maxLen: MAX_CHARS,
+            targetLen: TARGET_CHUNK_CHARS,
+            minLen: MIN_CHUNK_CHARS,
+            overlap: CHUNK_OVERLAP,
+        });
+
+        chunkMap.push({ count: chunks.length });
+        allChunks.push(...chunks);
+
+        if (chunks.length === 1) {
+            const tokenCount = estimateTokens(chunks[0]);
+            winston().info(
+                `[search-agent] embeddingService: batch input ${textIdx + 1}/${texts.length} - 1 chunk, ${chunks[0].length} chars, ~${tokenCount} tokens`
+            );
+        } else {
+            winston().info(
+                `[search-agent] embeddingService: batch input ${textIdx + 1}/${texts.length} - ${chunks.length} chunks`
+            );
+            chunks.forEach((chunk, i) => {
+                const tokenCount = estimateTokens(chunk);
+                winston().info(
+                    `[search-agent] embeddingService:   chunk ${i + 1}/${chunks.length} - ${chunk.length} chars, ~${tokenCount} tokens`
+                );
+            });
+        }
+    }
+
+    winston().verbose(
+        `[search-agent] embeddingService: batch embedding ${allChunks.length} chunk(s) from ${texts.length} input(s)`
+    );
+
+    if (allChunks.length === 0) {
+        return chunkMap.map(({ count }) => (count === 0 ? [] : null));
+    }
+
+    const safeChunks = allChunks.map(chunk => truncate(chunk));
+    const response = await withRetry(() => requestEmbeddings(apiKey, safeChunks));
+
+    winston().verbose(
+        `[search-agent] embeddingService: batch embeddings generated successfully (${safeChunks.length} vector(s))`
+    );
+
+    const vectors = response.data
+        .sort((a, b) => a.index - b.index)
+        .map(item => item.embedding);
+
+    const result = [];
+    let idx = 0;
+
+    for (const { count } of chunkMap) {
+        if (count === 0) {
+            result.push([]);
+        } else if (count === 1) {
+            result.push(vectors[idx]);
+            idx += 1;
+        } else {
+            result.push(averageVectors(vectors.slice(idx, idx + count)));
+            idx += count;
+        }
+    }
+
+    return result;
 }
 
-module.exports = { embed, embedBatch };
+module.exports = {
+    embed,
+    embedBatch,
+    extractPureText,
+    splitIntoSemanticChunks,
+};

package/services/vectorSearchService.js

@@ -5,123 +5,409 @@ const { embed } = require('./embeddingService');
 const { getAllEmbeddings } = require('./vectorStore');
 
 function winston() {
-	return require.main.require('winston');
+    return require.main.require('winston');
 }
 
 // Fetch this many candidates from Orama — cast a wide net so the AI has enough to choose from
 const TOP_K = 50;
-// Absolute minimum cosine similarity — only filters pure noise (near-zero similarity).
-// Do NOT raise this: the relevant result often scores lower than irrelevant ones.
-// The AI re-ranker (which reads content) is the precision gate, not this floor.
+
+// Absolute minimum similarity — only filters pure noise.
+// Keep this low; the later ranking layers should handle precision.
 const MIN_SCORE = 0.15;
-// Rebuild the Orama index after this interval (mirrors TF-IDF cache TTL)
+
+// Rebuild the Orama index after this interval
 const INDEX_TTL_MS = 5 * 60 * 1000;
 
+// Hybrid search configuration
+const VECTOR_SIMILARITY = 0.1;
+const SEARCH_PROPERTIES = ['title', 'category', 'tags', 'content', 'parent_content'];
+const FIELD_BOOSTS = {
+    title: 3.5,
+    tags: 2.8,
+    category: 2.2,
+    content: 1.0,
+    parent_content: 0.8,
+};
+
 let _db = null;
 let _dbTs = 0;
 let _buildPromise = null;
 
+// Finance-heavy Hebrew forum query expansion.
+// These are intentionally conservative: good recall lift without flooding the query.
+const QUERY_EXPANSIONS = {
+    // General finance
+    'מניה': ['מניות', 'נייר ערך', 'שוק ההון', 'בורסה'],
+    'מניות': ['מניה', 'ניירות ערך', 'שוק ההון', 'בורסה'],
+    'אגח': ['אג"ח', 'איגרת חוב', 'איגרות חוב', 'חוב'],
+    'אג"ח': ['אגח', 'איגרת חוב', 'איגרות חוב', 'חוב'],
+    'קרן': ['קרנות', 'קרן נאמנות', 'קרן סל'],
+    'קרנות': ['קרן', 'קרן נאמנות', 'קרן סל'],
+    'קרן סל': ['etf', 'תעודת סל', 'קרן מחקה'],
+    'תעודת סל': ['etf', 'קרן סל', 'קרן מחקה'],
+    'etf': ['קרן סל', 'תעודת סל', 'קרן מחקה'],
+    'מדד': ['מדדים', 'מדד מניות', 'תשואת מדד'],
+    'מדדים': ['מדד', 'מדד מניות', 'תשואת מדד'],
+    'תיק': ['תיק השקעות', 'פיזור', 'החזקות'],
+    'השקעה': ['השקעות', 'להשקיע', 'תיק השקעות'],
+    'השקעות': ['השקעה', 'להשקיע', 'תיק השקעות'],
+    'להשקיע': ['השקעה', 'השקעות', 'תיק השקעות'],
+    'תשואה': ['רווח', 'תשואות', 'רווחיות'],
+    'רווח': ['רווחים', 'תשואה', 'רווחיות'],
+    'הפסד': ['הפסדים', 'ירידה', 'מינוס'],
+    'דיבידנד': ['דיבידנדים', 'חלוקת רווחים'],
+    'מכפיל': ['מכפיל רווח', 'pe', 'p/e'],
+    'pe': ['מכפיל', 'מכפיל רווח', 'p/e'],
+    'p/e': ['מכפיל', 'מכפיל רווח', 'pe'],
+    'מינוף': ['ממונף', 'הלוואה', 'מרגין', 'margin'],
+    'מרגין': ['margin', 'מינוף'],
+    'margin': ['מרגין', 'מינוף'],
+    'סיכון': ['סיכונים', 'תנודתיות', 'חשיפה'],
+    'נזילות': ['נזיל', 'מזומן', 'סחירות'],
+    'סחירות': ['נזילות', 'נזיל'],
+
+    // Tax / regulation
+    'מס': ['מיסוי', 'מסים', 'רשות המסים'],
+    'מיסוי': ['מס', 'מסים', 'רשות המסים'],
+    'מסים': ['מס', 'מיסוי', 'רשות המסים'],
+    'קיזוז': ['קיזוז הפסדים', 'מגן מס'],
+    'דוח': ['דו"ח', 'דיווח', 'טופס'],
+    'דו"ח': ['דוח', 'דיווח', 'טופס'],
+
+    // Savings / pension
+    'פנסיה': ['קרן פנסיה', 'חיסכון פנסיוני', 'קצבה'],
+    'גמל': ['קופת גמל', 'קופ"ג'],
+    'קופג': ['קופת גמל', 'קופ"ג', 'גמל'],
+    'קופ"ג': ['קופת גמל', 'קופג', 'גמל'],
+    'השתלמות': ['קרן השתלמות'],
+    'משכנתא': ['משכנתאות', 'ריבית', 'הלוואת דיור'],
+    'הלוואה': ['הלוואות', 'אשראי', 'מימון'],
+    'אשראי': ['הלוואה', 'הלוואות', 'מימון'],
+
+    // Trading / technical
+    'מסחר': ['טריידינג', 'קניה', 'מכירה', 'פקודה'],
+    'טריידינג': ['מסחר', 'מסחר יומי', 'קניה', 'מכירה'],
+    'שורט': ['short', 'מכירה בחסר'],
+    'short': ['שורט', 'מכירה בחסר'],
+    'לונג': ['long', 'החזקה'],
+    'long': ['לונג', 'החזקה'],
+    'פקודה': ['פקודות', 'לימיט', 'מרקט'],
+    'לימיט': ['limit', 'פקודת לימיט'],
+    'limit': ['לימיט', 'פקודת לימיט'],
+    'מרקט': ['market', 'פקודת שוק'],
+    'market': ['מרקט', 'פקודת שוק'],
+
+    // Crypto
+    'ביטקוין': ['btc', 'קריפטו', 'מטבע דיגיטלי'],
+    'btc': ['ביטקוין', 'קריפטו', 'מטבע דיגיטלי'],
+    'אתריום': ['eth', 'קריפטו', 'מטבע דיגיטלי'],
+    'eth': ['אתריום', 'קריפטו', 'מטבע דיגיטלי'],
+    'קריפטו': ['מטבע דיגיטלי', 'ביטקוין', 'אתריום', 'בלוקציין'],
+    'בלוקציין': ['קריפטו', 'מטבע דיגיטלי'],
+
+    // Hebrew forum / advice intent
+    'מומלץ': ['כדאי', 'המלצה', 'עדיף'],
+    'כדאי': ['מומלץ', 'המלצה', 'עדיף'],
+    'המלצה': ['מומלץ', 'כדאי', 'עדיף'],
+    'בעיה': ['תקלה', 'קושי', 'לא עובד'],
+    'תקלה': ['בעיה', 'לא עובד', 'שגיאה'],
+    'שגיאה': ['תקלה', 'בעיה', 'לא עובד'],
+};
+
+// Common phrase-level expansions that are better handled before token expansion.
+const PHRASE_EXPANSIONS = [
+    {
+        pattern: /\b(?:קרן\s+סל|תעודת\s+סל|קרן\s+מחקה)\b/gi,
+        terms: ['etf', 'קרן סל', 'תעודת סל', 'קרן מחקה'],
+    },
+    {
+        pattern: /\b(?:איגרת\s+חוב|איגרות\s+חוב|אג["׳׳]?\s?ח)\b/gi,
+        terms: ['אגח', 'אג"ח', 'איגרת חוב', 'איגרות חוב'],
+    },
+    {
+        pattern: /\b(?:קופת\s+גמל|קופ["׳׳]?\s?ג)\b/gi,
+        terms: ['קופת גמל', 'קופג', 'קופ"ג', 'גמל'],
+    },
+    {
+        pattern: /\b(?:מכפיל\s+רווח|p\/e|pe)\b/gi,
+        terms: ['מכפיל', 'מכפיל רווח', 'pe', 'p/e'],
+    },
+    {
+        pattern: /\b(?:מכירה\s+בחסר|short)\b/gi,
+        terms: ['שורט', 'short', 'מכירה בחסר'],
+    },
+];
+
+// Generic filler words to ignore for lexical expansion
+const STOP_WORDS = new Set([
+    'של', 'על', 'עם', 'בלי', 'גם', 'או', 'אם', 'אבל', 'כי', 'זה', 'זאת', 'זו',
+    'יש', 'אין', 'אני', 'אתה', 'את', 'הוא', 'היא', 'הם', 'הן', 'אנחנו', 'מה',
+    'איך', 'למה', 'מתי', 'איפה', 'האם', 'כל', 'עוד', 'כמו', 'רק', 'מאוד', 'פחות',
+    'יותר', 'אחרי', 'לפני', 'תוך', 'דרך', 'לגבי', 'בנוגע', 'בשביל', 'מול',
+]);
+
+function normalizeHebrew(text) {
+	if(!text) return text;
+
+	// Remove common prefixes
+	const prefixes = ['ה', 'ו', 'ב', 'ל', 'מ', 'ש', 'כ'];
+    for (const prefix of prefixes) {
+        if (text.startsWith(prefix) && text.length > 3) {
+            text = text.slice(1);
+            break;
+        }
+    }
+
+    // Remove common plural suffixes
+    const pluralSuffixes = ['ים', 'ות'];
+    for (const suffix of pluralSuffixes) {
+        if (text.endsWith(suffix) && text.length > 3) {
+            text = text.slice(0, -suffix.length);
+            break;
+        }
+    }
+
+    return String(text || '')
+        // remove niqqud / cantillation
+        .replace(/[\u0591-\u05C7]/g, '')
+        // normalize Hebrew punctuation variants
+        .replace(/[׳']/g, '\'')
+        .replace(/[״"]/g, '"')
+        // collapse whitespace
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+
+function normalizeToken(token) {
+    return normalizeHebrew(token)
+        .toLowerCase()
+        .replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, '');
+}
+
+function uniqueTerms(terms, maxTerms = 24) {
+    const seen = new Set();
+    const out = [];
+
+    for (const raw of terms) {
+        const term = normalizeHebrew(raw).trim();
+        if (!term) continue;
+
+        const key = term.toLowerCase();
+        if (seen.has(key)) continue;
+
+        seen.add(key);
+        out.push(term);
+
+        if (out.length >= maxTerms) break;
+    }
+
+    return out;
+}
+
+function expandQuery(query) {
+    const normalized = normalizeHebrew(query);
+    const expanded = [normalized];
+
+    for (const phraseRule of PHRASE_EXPANSIONS) {
+        if (phraseRule.pattern.test(normalized)) {
+            expanded.push(...phraseRule.terms);
+        }
+        phraseRule.pattern.lastIndex = 0;
+    }
+
+    const tokens = normalized
+        .split(/[\s,/|()[\]{}:;!?]+/)
+        .map(normalizeToken)
+        .filter(Boolean)
+        .filter(token => !STOP_WORDS.has(token));
+
+    for (const token of tokens) {
+        expanded.push(token);
+
+        const synonyms = QUERY_EXPANSIONS[token];
+        if (synonyms) {
+            expanded.push(...synonyms);
+        }
+
+        // A little morphology help for Hebrew singular/plural and abbreviations
+        if (token.endsWith('ים') && token.length > 3) {
+            expanded.push(token.slice(0, -2));
+        }
+        if (token.endsWith('ות') && token.length > 3) {
+            expanded.push(token.slice(0, -2));
+        }
+        if (token.endsWith('ה') && token.length > 2) {
+            expanded.push(token.slice(0, -1));
+        }
+    }
+
+    const terms = uniqueTerms(expanded, 24);
+
+    return {
+        original: query,
+        normalized,
+        terms,
+        // Orama lexical search receives one expanded term string
+        term: terms.join(' '),
+    };
+}
+
+function coerceString(value) {
+    if (value == null) return '';
+
+    if (Array.isArray(value)) {
+        return value
+            .map(v => coerceString(v))
+            .filter(Boolean)
+            .join(', ');
+    }
+
+    return String(value).trim();
+}
+
+function buildDocument(row) {
+    return {
+        id: String(row.post_id),
+        post_id: row.post_id,
+        topic_id: row.topic_id,
+        title: coerceString(row.title),
+        category: coerceString(row.category),
+        tags: coerceString(row.tags),
+        parent_content: coerceString(row.parent_content),
+        content: coerceString(row.content),
+        embedding: row.embedding,
+    };
+}
+
 async function buildIndex() {
-	const storedEmbeddings = await getAllEmbeddings();
-
-	// Detect dimension from data; fall back to 1536 (text-embedding-3-small default)
-	const dimensions = storedEmbeddings.length > 0
-		? storedEmbeddings[0].embedding.length
-		: 1536;
-
-	const db = await create({
-		schema: {
-			post_id: 'number',
-			topic_id: 'number',
-			content: 'string',
-			embedding: `vector[${dimensions}]`,
-		},
-	});
-
-	if (storedEmbeddings.length > 0) {
-		await insertMultiple(db, storedEmbeddings.map(e => ({
-			id: String(e.post_id),
-			post_id: e.post_id,
-			topic_id: e.topic_id,
-			content: e.content,
-			embedding: e.embedding,
-		})));
-	}
-
-	winston().info(`[search-agent] vectorSearchService: Orama index built with ${storedEmbeddings.length} document(s)`);
-	return db;
+    const storedEmbeddings = await getAllEmbeddings();
+
+    // Detect dimension from data; fall back to 1536 (text-embedding-3-small default)
+    const dimensions = storedEmbeddings.length > 0
+        ? storedEmbeddings[0].embedding.length
+        : 1536;
+
+    const db = await create({
+        schema: {
+            post_id: 'number',
+            topic_id: 'number',
+            title: 'string',
+            category: 'string',
+            tags: 'string',
+            parent_content: 'string',
+            content: 'string',
+            embedding: `vector[${dimensions}]`,
+        },
+    });
+
+    if (storedEmbeddings.length > 0) {
+        await insertMultiple(db, storedEmbeddings.map(buildDocument));
+    }
+
+    winston().info(
+        `[search-agent] vectorSearchService: Orama index built with ${storedEmbeddings.length} document(s)`
+    );
+
+    return db;
 }
 
 async function getDb() {
-	const now = Date.now();
-	if (_db && (now - _dbTs) < INDEX_TTL_MS) {
-		return _db;
-	}
-
-	if (_buildPromise) {
-		return _buildPromise;
-	}
-
-	_buildPromise = buildIndex().then((db) => {
-		_db = db;
-		_dbTs = Date.now();
-		_buildPromise = null;
-		return db;
-	}).catch((err) => {
-		_buildPromise = null;
-		throw err;
-	});
-
-	return _buildPromise;
+    const now = Date.now();
+    if (_db && (now - _dbTs) < INDEX_TTL_MS) {
+        return _db;
+    }
+
+    if (_buildPromise) {
+        return _buildPromise;
+    }
+
+    _buildPromise = buildIndex()
+        .then((db) => {
+            _db = db;
+            _dbTs = Date.now();
+            _buildPromise = null;
+            return db;
+        })
+        .catch((err) => {
+            _buildPromise = null;
+            throw err;
+        });
+
+    return _buildPromise;
 }
 
 /** Invalidate the in-memory Orama index (e.g. after new embeddings are saved). */
 function invalidateIndex() {
-	_db = null;
-	_dbTs = 0;
-	winston().info('[search-agent] vectorSearchService: Orama index invalidated');
+    _db = null;
+    _dbTs = 0;
+    winston().info('[search-agent] vectorSearchService: Orama index invalidated');
 }
 
 /**
- * Performs semantic search against stored post embeddings using Orama vector search.
+ * Performs hybrid search against stored post embeddings using:
+ * 1. vector similarity on the original query embedding
+ * 2. lexical search on an expanded Hebrew query
+ * 3. field boosts to favor title/tags/category matches
  *
  * @param {string} query - The search query string.
- * @returns {Promise<Array<{ topic_id: number, post_id: number, content: string, score: number }>>}
- *   Top results sorted by cosine similarity descending.
+ * @param {number} limit - Max results to return.
+ * @returns {Promise<Array<{ topic_id: number, post_id: number, title: string, category: string, tags: string, content: string, score: number }>>}
  */
 async function search(query, limit = TOP_K) {
-	if (typeof query !== 'string' || query.trim() === '') {
-		throw new Error('search() requires a non-empty query string');
-	}
-
-	winston().verbose(`[search-agent] vectorSearchService: running Orama vector search for "${query.trim()}"`);
-
-	const [queryEmbedding, db] = await Promise.all([
-		embed(query),
-		getDb(),
-	]);
-
-	const results = await oramaSearch(db, {
-		mode: 'vector',
-		vector: { value: queryEmbedding, property: 'embedding' },
-		limit,
-		similarity: 0.1, 
-		includeVectors: false,
-	});
-
-	winston().verbose(`[search-agent] vectorSearchService: Orama returned ${results.hits.length} hit(s)`);
-
-	const filtered = results.hits.filter(hit => hit.score >= MIN_SCORE);
-	winston().verbose(
-		`[search-agent] vectorSearchService: ${filtered.length}/${results.hits.length} hit(s) passed noise floor (MIN_SCORE=${MIN_SCORE})`
-	);
-
-	return filtered.map(hit => ({
-		topic_id: hit.document.topic_id,
-		post_id: hit.document.post_id,
-		content: hit.document.content,
-		score: hit.score,
-	}));
+    if (typeof query !== 'string' || query.trim() === '') {
+        throw new Error('search() requires a non-empty query string');
+    }
+
+    const trimmed = query.trim();
+    const expanded = expandQuery(trimmed);
+
+    winston().verbose(
+        `[search-agent] vectorSearchService: running Orama hybrid search for "${trimmed}" (expanded="${expanded.term}")`
+    );
+
+    const [queryEmbedding, db] = await Promise.all([
+        // Keep the embedding on the original query only.
+        // Expansion is mainly for lexical recall, especially in Hebrew forum language.
+        embed(trimmed),
+        getDb(),
+    ]);
+
+    const results = await oramaSearch(db, {
+        // mode: 'hybrid',
+        query: expanded.term,
+        properties: SEARCH_PROPERTIES,
+        boost: FIELD_BOOSTS,
+        vector: {
+            value: queryEmbedding,
+            property: 'embedding',
+        },
+        limit,
+        similarity: VECTOR_SIMILARITY,
+        includeVectors: false,
+    });
+
+    const hits = Array.isArray(results && results.hits) ? results.hits : [];
+
+    winston().verbose(`[search-agent] vectorSearchService: Orama returned ${hits.length} hit(s)`);
+
+    const filtered = hits.filter(hit => typeof hit.score === 'number' && hit.score >= MIN_SCORE);
+
+    winston().verbose(
+        `[search-agent] vectorSearchService: ${filtered.length}/${hits.length} hit(s) passed noise floor (MIN_SCORE=${MIN_SCORE})`
+    );
+
+    return filtered.map(hit => ({
+        topic_id: hit.document.topic_id,
+        post_id: hit.document.post_id,
+        title: hit.document.title || '',
+        category: hit.document.category || '',
+        tags: hit.document.tags || '',
+        content: hit.document.content,
+        score: hit.score,
+    }));
 }
 
-module.exports = { search, invalidateIndex };
+module.exports = {
+    search,
+    invalidateIndex,
+};