nodebb-plugin-search-agent 0.0.92 → 0.0.94

package/services/vectorSearchService.js

@@ -5,123 +5,409 @@ const { embed } = require('./embeddingService');
 const { getAllEmbeddings } = require('./vectorStore');
 
 function winston() {
-	return require.main.require('winston');
+    return require.main.require('winston');
 }
 
 // Fetch this many candidates from Orama — cast a wide net so the AI has enough to choose from
 const TOP_K = 50;
-// Absolute minimum cosine similarity — only filters pure noise (near-zero similarity).
-// Do NOT raise this: the relevant result often scores lower than irrelevant ones.
-// The AI re-ranker (which reads content) is the precision gate, not this floor.
-const MIN_SCORE = 0.10;
-// Rebuild the Orama index after this interval (mirrors TF-IDF cache TTL)
+
+// Absolute minimum similarity — only filters pure noise.
+// Keep this low; the later ranking layers should handle precision.
+const MIN_SCORE = 0.15;
+
+// Rebuild the Orama index after this interval
 const INDEX_TTL_MS = 5 * 60 * 1000;
 
+// Hybrid search configuration
+const VECTOR_SIMILARITY = 0.1;
+const SEARCH_PROPERTIES = ['title', 'category', 'tags', 'content', 'parent_content'];
+const FIELD_BOOSTS = {
+    title: 3.5,
+    tags: 2.8,
+    category: 2.2,
+    content: 1.0,
+    parent_content: 0.8,
+};
+
 let _db = null;
 let _dbTs = 0;
 let _buildPromise = null;
 
+// Finance-heavy Hebrew forum query expansion.
+// These are intentionally conservative: good recall lift without flooding the query.
+const QUERY_EXPANSIONS = {
+    // General finance
+    'מניה': ['מניות', 'נייר ערך', 'שוק ההון', 'בורסה'],
+    'מניות': ['מניה', 'ניירות ערך', 'שוק ההון', 'בורסה'],
+    'אגח': ['אג"ח', 'איגרת חוב', 'איגרות חוב', 'חוב'],
+    'אג"ח': ['אגח', 'איגרת חוב', 'איגרות חוב', 'חוב'],
+    'קרן': ['קרנות', 'קרן נאמנות', 'קרן סל'],
+    'קרנות': ['קרן', 'קרן נאמנות', 'קרן סל'],
+    'קרן סל': ['etf', 'תעודת סל', 'קרן מחקה'],
+    'תעודת סל': ['etf', 'קרן סל', 'קרן מחקה'],
+    'etf': ['קרן סל', 'תעודת סל', 'קרן מחקה'],
+    'מדד': ['מדדים', 'מדד מניות', 'תשואת מדד'],
+    'מדדים': ['מדד', 'מדד מניות', 'תשואת מדד'],
+    'תיק': ['תיק השקעות', 'פיזור', 'החזקות'],
+    'השקעה': ['השקעות', 'להשקיע', 'תיק השקעות'],
+    'השקעות': ['השקעה', 'להשקיע', 'תיק השקעות'],
+    'להשקיע': ['השקעה', 'השקעות', 'תיק השקעות'],
+    'תשואה': ['רווח', 'תשואות', 'רווחיות'],
+    'רווח': ['רווחים', 'תשואה', 'רווחיות'],
+    'הפסד': ['הפסדים', 'ירידה', 'מינוס'],
+    'דיבידנד': ['דיבידנדים', 'חלוקת רווחים'],
+    'מכפיל': ['מכפיל רווח', 'pe', 'p/e'],
+    'pe': ['מכפיל', 'מכפיל רווח', 'p/e'],
+    'p/e': ['מכפיל', 'מכפיל רווח', 'pe'],
+    'מינוף': ['ממונף', 'הלוואה', 'מרגין', 'margin'],
+    'מרגין': ['margin', 'מינוף'],
+    'margin': ['מרגין', 'מינוף'],
+    'סיכון': ['סיכונים', 'תנודתיות', 'חשיפה'],
+    'נזילות': ['נזיל', 'מזומן', 'סחירות'],
+    'סחירות': ['נזילות', 'נזיל'],
+
+    // Tax / regulation
+    'מס': ['מיסוי', 'מסים', 'רשות המסים'],
+    'מיסוי': ['מס', 'מסים', 'רשות המסים'],
+    'מסים': ['מס', 'מיסוי', 'רשות המסים'],
+    'קיזוז': ['קיזוז הפסדים', 'מגן מס'],
+    'דוח': ['דו"ח', 'דיווח', 'טופס'],
+    'דו"ח': ['דוח', 'דיווח', 'טופס'],
+
+    // Savings / pension
+    'פנסיה': ['קרן פנסיה', 'חיסכון פנסיוני', 'קצבה'],
+    'גמל': ['קופת גמל', 'קופ"ג'],
+    'קופג': ['קופת גמל', 'קופ"ג', 'גמל'],
+    'קופ"ג': ['קופת גמל', 'קופג', 'גמל'],
+    'השתלמות': ['קרן השתלמות'],
+    'משכנתא': ['משכנתאות', 'ריבית', 'הלוואת דיור'],
+    'הלוואה': ['הלוואות', 'אשראי', 'מימון'],
+    'אשראי': ['הלוואה', 'הלוואות', 'מימון'],
+
+    // Trading / technical
+    'מסחר': ['טריידינג', 'קניה', 'מכירה', 'פקודה'],
+    'טריידינג': ['מסחר', 'מסחר יומי', 'קניה', 'מכירה'],
+    'שורט': ['short', 'מכירה בחסר'],
+    'short': ['שורט', 'מכירה בחסר'],
+    'לונג': ['long', 'החזקה'],
+    'long': ['לונג', 'החזקה'],
+    'פקודה': ['פקודות', 'לימיט', 'מרקט'],
+    'לימיט': ['limit', 'פקודת לימיט'],
+    'limit': ['לימיט', 'פקודת לימיט'],
+    'מרקט': ['market', 'פקודת שוק'],
+    'market': ['מרקט', 'פקודת שוק'],
+
+    // Crypto
+    'ביטקוין': ['btc', 'קריפטו', 'מטבע דיגיטלי'],
+    'btc': ['ביטקוין', 'קריפטו', 'מטבע דיגיטלי'],
+    'אתריום': ['eth', 'קריפטו', 'מטבע דיגיטלי'],
+    'eth': ['אתריום', 'קריפטו', 'מטבע דיגיטלי'],
+    'קריפטו': ['מטבע דיגיטלי', 'ביטקוין', 'אתריום', 'בלוקציין'],
+    'בלוקציין': ['קריפטו', 'מטבע דיגיטלי'],
+
+    // Hebrew forum / advice intent
+    'מומלץ': ['כדאי', 'המלצה', 'עדיף'],
+    'כדאי': ['מומלץ', 'המלצה', 'עדיף'],
+    'המלצה': ['מומלץ', 'כדאי', 'עדיף'],
+    'בעיה': ['תקלה', 'קושי', 'לא עובד'],
+    'תקלה': ['בעיה', 'לא עובד', 'שגיאה'],
+    'שגיאה': ['תקלה', 'בעיה', 'לא עובד'],
+};
+
+// Common phrase-level expansions that are better handled before token expansion.
+const PHRASE_EXPANSIONS = [
+    {
+        pattern: /\b(?:קרן\s+סל|תעודת\s+סל|קרן\s+מחקה)\b/gi,
+        terms: ['etf', 'קרן סל', 'תעודת סל', 'קרן מחקה'],
+    },
+    {
+        pattern: /\b(?:איגרת\s+חוב|איגרות\s+חוב|אג["׳׳]?\s?ח)\b/gi,
+        terms: ['אגח', 'אג"ח', 'איגרת חוב', 'איגרות חוב'],
+    },
+    {
+        pattern: /\b(?:קופת\s+גמל|קופ["׳׳]?\s?ג)\b/gi,
+        terms: ['קופת גמל', 'קופג', 'קופ"ג', 'גמל'],
+    },
+    {
+        pattern: /\b(?:מכפיל\s+רווח|p\/e|pe)\b/gi,
+        terms: ['מכפיל', 'מכפיל רווח', 'pe', 'p/e'],
+    },
+    {
+        pattern: /\b(?:מכירה\s+בחסר|short)\b/gi,
+        terms: ['שורט', 'short', 'מכירה בחסר'],
+    },
+];
+
+// Generic filler words to ignore for lexical expansion
+const STOP_WORDS = new Set([
+    'של', 'על', 'עם', 'בלי', 'גם', 'או', 'אם', 'אבל', 'כי', 'זה', 'זאת', 'זו',
+    'יש', 'אין', 'אני', 'אתה', 'את', 'הוא', 'היא', 'הם', 'הן', 'אנחנו', 'מה',
+    'איך', 'למה', 'מתי', 'איפה', 'האם', 'כל', 'עוד', 'כמו', 'רק', 'מאוד', 'פחות',
+    'יותר', 'אחרי', 'לפני', 'תוך', 'דרך', 'לגבי', 'בנוגע', 'בשביל', 'מול',
+]);
+
+function normalizeHebrew(text) {
+	if(!text) return text;
+
+	// Remove common prefixes
+	const prefixes = ['ה', 'ו', 'ב', 'ל', 'מ', 'ש', 'כ'];
+    for (const prefix of prefixes) {
+        if (text.startsWith(prefix) && text.length > 3) {
+            text = text.slice(1);
+            break;
+        }
+    }
+
+    // Remove common plural suffixes
+    const pluralSuffixes = ['ים', 'ות'];
+    for (const suffix of pluralSuffixes) {
+        if (text.endsWith(suffix) && text.length > 3) {
+            text = text.slice(0, -suffix.length);
+            break;
+        }
+    }
+
+    return String(text || '')
+        // remove niqqud / cantillation
+        .replace(/[\u0591-\u05C7]/g, '')
+        // normalize Hebrew punctuation variants
+        .replace(/[׳']/g, '\'')
+        .replace(/[״"]/g, '"')
+        // collapse whitespace
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+
+function normalizeToken(token) {
+    return normalizeHebrew(token)
+        .toLowerCase()
+        .replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, '');
+}
+
+function uniqueTerms(terms, maxTerms = 24) {
+    const seen = new Set();
+    const out = [];
+
+    for (const raw of terms) {
+        const term = normalizeHebrew(raw).trim();
+        if (!term) continue;
+
+        const key = term.toLowerCase();
+        if (seen.has(key)) continue;
+
+        seen.add(key);
+        out.push(term);
+
+        if (out.length >= maxTerms) break;
+    }
+
+    return out;
+}
+
+function expandQuery(query) {
+    const normalized = normalizeHebrew(query);
+    const expanded = [normalized];
+
+    for (const phraseRule of PHRASE_EXPANSIONS) {
+        if (phraseRule.pattern.test(normalized)) {
+            expanded.push(...phraseRule.terms);
+        }
+        phraseRule.pattern.lastIndex = 0;
+    }
+
+    const tokens = normalized
+        .split(/[\s,/|()[\]{}:;!?]+/)
+        .map(normalizeToken)
+        .filter(Boolean)
+        .filter(token => !STOP_WORDS.has(token));
+
+    for (const token of tokens) {
+        expanded.push(token);
+
+        const synonyms = QUERY_EXPANSIONS[token];
+        if (synonyms) {
+            expanded.push(...synonyms);
+        }
+
+        // A little morphology help for Hebrew singular/plural and abbreviations
+        if (token.endsWith('ים') && token.length > 3) {
+            expanded.push(token.slice(0, -2));
+        }
+        if (token.endsWith('ות') && token.length > 3) {
+            expanded.push(token.slice(0, -2));
+        }
+        if (token.endsWith('ה') && token.length > 2) {
+            expanded.push(token.slice(0, -1));
+        }
+    }
+
+    const terms = uniqueTerms(expanded, 24);
+
+    return {
+        original: query,
+        normalized,
+        terms,
+        // Orama lexical search receives one expanded term string
+        term: terms.join(' '),
+    };
+}
+
+function coerceString(value) {
+    if (value == null) return '';
+
+    if (Array.isArray(value)) {
+        return value
+            .map(v => coerceString(v))
+            .filter(Boolean)
+            .join(', ');
+    }
+
+    return String(value).trim();
+}
+
+function buildDocument(row) {
+    return {
+        id: String(row.post_id),
+        post_id: row.post_id,
+        topic_id: row.topic_id,
+        title: coerceString(row.title),
+        category: coerceString(row.category),
+        tags: coerceString(row.tags),
+        parent_content: coerceString(row.parent_content),
+        content: coerceString(row.content),
+        embedding: row.embedding,
+    };
+}
+
 async function buildIndex() {
-	const storedEmbeddings = await getAllEmbeddings();
-
-	// Detect dimension from data; fall back to 1536 (text-embedding-3-small default)
-	const dimensions = storedEmbeddings.length > 0
-		? storedEmbeddings[0].embedding.length
-		: 1536;
-
-	const db = await create({
-		schema: {
-			post_id: 'number',
-			topic_id: 'number',
-			content: 'string',
-			embedding: `vector[${dimensions}]`,
-		},
-	});
-
-	if (storedEmbeddings.length > 0) {
-		await insertMultiple(db, storedEmbeddings.map(e => ({
-			id: String(e.post_id),
-			post_id: e.post_id,
-			topic_id: e.topic_id,
-			content: e.content,
-			embedding: e.embedding,
-		})));
-	}
-
-	winston().info(`[search-agent] vectorSearchService: Orama index built with ${storedEmbeddings.length} document(s)`);
-	return db;
+    const storedEmbeddings = await getAllEmbeddings();
+
+    // Detect dimension from data; fall back to 1536 (text-embedding-3-small default)
+    const dimensions = storedEmbeddings.length > 0
+        ? storedEmbeddings[0].embedding.length
+        : 1536;
+
+    const db = await create({
+        schema: {
+            post_id: 'number',
+            topic_id: 'number',
+            title: 'string',
+            category: 'string',
+            tags: 'string',
+            parent_content: 'string',
+            content: 'string',
+            embedding: `vector[${dimensions}]`,
+        },
+    });
+
+    if (storedEmbeddings.length > 0) {
+        await insertMultiple(db, storedEmbeddings.map(buildDocument));
+    }
+
+    winston().info(
+        `[search-agent] vectorSearchService: Orama index built with ${storedEmbeddings.length} document(s)`
+    );
+
+    return db;
 }
 
 async function getDb() {
-	const now = Date.now();
-	if (_db && (now - _dbTs) < INDEX_TTL_MS) {
-		return _db;
-	}
-
-	if (_buildPromise) {
-		return _buildPromise;
-	}
-
-	_buildPromise = buildIndex().then((db) => {
-		_db = db;
-		_dbTs = Date.now();
-		_buildPromise = null;
-		return db;
-	}).catch((err) => {
-		_buildPromise = null;
-		throw err;
-	});
-
-	return _buildPromise;
+    const now = Date.now();
+    if (_db && (now - _dbTs) < INDEX_TTL_MS) {
+        return _db;
+    }
+
+    if (_buildPromise) {
+        return _buildPromise;
+    }
+
+    _buildPromise = buildIndex()
+        .then((db) => {
+            _db = db;
+            _dbTs = Date.now();
+            _buildPromise = null;
+            return db;
+        })
+        .catch((err) => {
+            _buildPromise = null;
+            throw err;
+        });
+
+    return _buildPromise;
 }
 
 /** Invalidate the in-memory Orama index (e.g. after new embeddings are saved). */
 function invalidateIndex() {
-	_db = null;
-	_dbTs = 0;
-	winston().info('[search-agent] vectorSearchService: Orama index invalidated');
+    _db = null;
+    _dbTs = 0;
+    winston().info('[search-agent] vectorSearchService: Orama index invalidated');
 }
 
 /**
- * Performs semantic search against stored post embeddings using Orama vector search.
+ * Performs hybrid search against stored post embeddings using:
+ * 1. vector similarity on the original query embedding
+ * 2. lexical search on an expanded Hebrew query
+ * 3. field boosts to favor title/tags/category matches
  *
  * @param {string} query - The search query string.
- * @returns {Promise<Array<{ topic_id: number, post_id: number, content: string, score: number }>>}
- *   Top results sorted by cosine similarity descending.
+ * @param {number} limit - Max results to return.
+ * @returns {Promise<Array<{ topic_id: number, post_id: number, title: string, category: string, tags: string, content: string, score: number }>>}
  */
 async function search(query, limit = TOP_K) {
-	if (typeof query !== 'string' || query.trim() === '') {
-		throw new Error('search() requires a non-empty query string');
-	}
-
-	winston().verbose(`[search-agent] vectorSearchService: running Orama vector search for "${query.trim()}"`);
-
-	const [queryEmbedding, db] = await Promise.all([
-		embed(query),
-		getDb(),
-	]);
-
-	const results = await oramaSearch(db, {
-		mode: 'vector',
-		vector: { value: queryEmbedding, property: 'embedding' },
-		limit,
-		similarity: 0.1, 
-		includeVectors: false,
-	});
-
-	winston().verbose(`[search-agent] vectorSearchService: Orama returned ${results.hits.length} hit(s)`);
-
-	const filtered = results.hits.filter(hit => hit.score >= MIN_SCORE);
-	winston().verbose(
-		`[search-agent] vectorSearchService: ${filtered.length}/${results.hits.length} hit(s) passed noise floor (MIN_SCORE=${MIN_SCORE})`
-	);
-
-	return filtered.map(hit => ({
-		topic_id: hit.document.topic_id,
-		post_id: hit.document.post_id,
-		content: hit.document.content,
-		score: hit.score,
-	}));
+    if (typeof query !== 'string' || query.trim() === '') {
+        throw new Error('search() requires a non-empty query string');
+    }
+
+    const trimmed = query.trim();
+    const expanded = expandQuery(trimmed);
+
+    winston().verbose(
+        `[search-agent] vectorSearchService: running Orama hybrid search for "${trimmed}" (expanded="${expanded.term}")`
+    );
+
+    const [queryEmbedding, db] = await Promise.all([
+        // Keep the embedding on the original query only.
+        // Expansion is mainly for lexical recall, especially in Hebrew forum language.
+        embed(trimmed),
+        getDb(),
+    ]);
+
+    const results = await oramaSearch(db, {
+        mode: 'hybrid',
+        term: expanded.term,
+        properties: SEARCH_PROPERTIES,
+        boost: FIELD_BOOSTS,
+        vector: {
+            value: queryEmbedding,
+            property: 'embedding',
+        },
+        limit,
+        similarity: VECTOR_SIMILARITY,
+        includeVectors: false,
+    });
+
+    const hits = Array.isArray(results && results.hits) ? results.hits : [];
+
+    winston().verbose(`[search-agent] vectorSearchService: Orama returned ${hits.length} hit(s)`);
+
+    const filtered = hits.filter(hit => typeof hit.score === 'number' && hit.score >= MIN_SCORE);
+
+    winston().verbose(
+        `[search-agent] vectorSearchService: ${filtered.length}/${hits.length} hit(s) passed noise floor (MIN_SCORE=${MIN_SCORE})`
+    );
+
+    return filtered.map(hit => ({
+        topic_id: hit.document.topic_id,
+        post_id: hit.document.post_id,
+        title: hit.document.title || '',
+        category: hit.document.category || '',
+        tags: hit.document.tags || '',
+        content: hit.document.content,
+        score: hit.score,
+    }));
 }
 
-module.exports = { search, invalidateIndex };
+module.exports = {
+    search,
+    invalidateIndex,
+};

package/templates/admin/plugins/search-agent.tpl

@@ -73,6 +73,18 @@
 						</label>
 					</div>
 
+					<div class="form-check form-switch mb-3">
+						<input type="checkbox" class="form-check-input" id="hydeEnabled" name="hydeEnabled">
+						<label for="hydeEnabled" class="form-check-label">
+							Enable HyDE query expansion
+						</label>
+						<div class="form-text">
+							Generates a hypothetical forum post from the query before embedding — improves
+							recall for vague queries but adds <strong>one extra LLM call per search</strong>.
+							Disable to cut AI costs roughly in half.
+						</div>
+					</div>
+
 					<div class="mb-3">
 						<label class="form-label" for="openaiApiKey">OpenAI API Key</label>
 						<input

package/lib/cosineSimilarity.js

@@ -1,42 +0,0 @@
-'use strict';
-
-/**
- * Computes the cosine similarity between two numeric vectors.
- * Handles mismatched lengths by using the shorter vector's dimension.
- * Returns 0 if either vector has zero magnitude.
- *
- * @param {number[]} a
- * @param {number[]} b
- * @returns {number} similarity in [-1, 1]
- */
-function cosineSimilarity(a, b) {
-	const len = Math.min(a.length, b.length);
-	let dot = 0;
-	let magA = 0;
-	let magB = 0;
-
-	for (let i = 0; i < len; i++) {
-		dot += a[i] * b[i];
-		magA += a[i] * a[i];
-		magB += b[i] * b[i];
-	}
-
-	const denom = Math.sqrt(magA) * Math.sqrt(magB);
-	return denom === 0 ? 0 : dot / denom;
-}
-
-/**
- * Ranks items by cosine similarity to a query embedding.
- * Each item must have an `embedding` property (number[]).
- *
- * @param {number[]} queryEmbedding
- * @param {Array<{embedding: number[], [key: string]: any}>} items
- * @returns {Array<{item: object, score: number}>} sorted descending by score
- */
-function rankBySimilarity(queryEmbedding, items) {
-	return items
-		.map(item => ({ item, score: cosineSimilarity(queryEmbedding, item.embedding) }))
-		.sort((a, b) => b.score - a.score);
-}
-
-module.exports = { cosineSimilarity, rankBySimilarity };

package/test/testCosine.js

@@ -1,15 +0,0 @@
-const { cosineSimilarity } = require('../lib/cosineSimilarity');
-
-function testCosine() {
-  console.log("Testing cosine similarity...");
-
-  const a = cosineSimilarity([1, 0], [1, 0]);
-  const b = cosineSimilarity([1, 0], [0, 1]);
-
-  if (a < 0.9) throw new Error("Expected high similarity");
-  if (b > 0.1) throw new Error("Expected low similarity");
-
-  console.log("✅ Cosine OK");
-}
-
-module.exports = testCosine;