@bhushanpawar/sqldb 1.0.6 → 1.0.7

package/dist/search/search-ranker.js

@@ -0,0 +1,281 @@
+"use strict";
+/**
+ * Search Ranker
+ *
+ * Implements ranking algorithms for search results:
+ * - TF-IDF (Term Frequency - Inverse Document Frequency)
+ * - Field boosting (title matches rank higher than description)
+ * - Proximity scoring (words closer together rank higher)
+ * - Freshness scoring (newer documents rank higher - optional)
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SearchRanker = void 0;
+exports.createRanker = createRanker;
+const search_1 = require("../types/search");
+class SearchRanker {
+    constructor(weights) {
+        this.weights = {
+            ...search_1.DEFAULT_RANKING_WEIGHTS,
+            ...weights,
+        };
+    }
+    /**
+     * Calculate TF-IDF score for a term in a document
+     *
+     * TF (Term Frequency): How often does the term appear in this document?
+     * IDF (Inverse Document Frequency): How rare is this term across all documents?
+     *
+     * Formula: TF-IDF = TF * IDF
+     * where:
+     *   TF = (term frequency in document) / (total terms in document)
+     *   IDF = log((total documents) / (documents containing term))
+     */
+    calculateTfIdf(context) {
+        const { termFrequency, docFrequency, totalDocs } = context;
+        // Term Frequency (TF)
+        // Normalize by document length (not available here, so use raw frequency)
+        const tf = termFrequency;
+        // Inverse Document Frequency (IDF)
+        // Add 1 to avoid division by zero
+        const idf = Math.log((totalDocs + 1) / (docFrequency + 1));
+        return tf * idf;
+    }
+    /**
+     * Calculate field boost score
+     * Fields with higher boost values rank better
+     */
+    calculateFieldBoost(fieldBoost) {
+        return fieldBoost;
+    }
+    /**
+     * Calculate proximity score
+     * Documents where query terms appear close together rank higher
+     */
+    calculateProximityScore(proximityScore) {
+        return proximityScore || 0;
+    }
+    /**
+     * Calculate freshness score (optional)
+     * Newer documents rank higher
+     */
+    calculateFreshnessScore(createdAt, decayDays = 30) {
+        if (!createdAt)
+            return 0;
+        const now = Date.now();
+        const ageMs = now - createdAt;
+        const ageDays = ageMs / (1000 * 60 * 60 * 24);
+        // Exponential decay: score decreases as document ages
+        // After 30 days, score is ~0.5
+        // After 60 days, score is ~0.25
+        return Math.exp(-ageDays / decayDays);
+    }
+    /**
+     * Calculate combined score for a document
+     */
+    calculateScore(context) {
+        const tfIdf = this.calculateTfIdf(context);
+        const fieldBoost = this.calculateFieldBoost(context.fieldBoost);
+        const proximity = this.calculateProximityScore(context.proximityScore);
+        // Combine scores with weights
+        let score = tfIdf * this.weights.tfIdfWeight +
+            fieldBoost * this.weights.fieldBoostWeight +
+            proximity * this.weights.proximityWeight;
+        // Normalize to 0-1 range
+        score = Math.min(1, score / 10); // Divide by 10 as rough normalization
+        return score;
+    }
+    /**
+     * Rank search results by relevance
+     */
+    rankResults(results, queryTerms, termStats, totalDocs, fieldBoosts, proximityScores) {
+        const scored = [];
+        for (const result of results) {
+            let totalScore = 0;
+            const matchedTerms = [];
+            // Calculate score for each query term
+            for (const term of queryTerms) {
+                const stats = termStats.get(term);
+                if (!stats)
+                    continue;
+                const termFreq = stats.termFrequenciesByDoc.get(result.docId);
+                if (!termFreq)
+                    continue;
+                matchedTerms.push(term);
+                // Calculate score for this term
+                const context = {
+                    term,
+                    docId: result.docId,
+                    termFrequency: termFreq,
+                    docFrequency: stats.docFrequency,
+                    totalDocs,
+                    fieldBoost: this.getAverageFieldBoost(fieldBoosts || {}),
+                    proximityScore: proximityScores?.get(result.docId),
+                };
+                totalScore += this.calculateScore(context);
+            }
+            // Average score across all matched terms
+            const avgScore = matchedTerms.length > 0 ? totalScore / matchedTerms.length : 0;
+            scored.push({
+                score: avgScore,
+                data: result.data,
+                matchedTerms,
+            });
+        }
+        // Sort by score (highest first)
+        scored.sort((a, b) => b.score - a.score);
+        return scored;
+    }
+    /**
+     * Get average field boost value
+     */
+    getAverageFieldBoost(fieldBoosts) {
+        const values = Object.values(fieldBoosts);
+        if (values.length === 0)
+            return 1.0;
+        const sum = values.reduce((acc, val) => acc + val, 0);
+        return sum / values.length;
+    }
+    /**
+     * Calculate BM25 score (more advanced than TF-IDF)
+     *
+     * BM25 is a ranking function used by search engines.
+     * It's an improvement over TF-IDF with better handling of:
+     * - Document length normalization
+     * - Term saturation (diminishing returns for high TF)
+     *
+     * Formula: BM25 = IDF * (TF * (k1 + 1)) / (TF + k1 * (1 - b + b * (docLen / avgDocLen)))
+     * where:
+     *   k1 = term frequency saturation parameter (typical: 1.2-2.0)
+     *   b = length normalization parameter (typical: 0.75)
+     */
+    calculateBM25(termFrequency, docFrequency, totalDocs, docLength, avgDocLength, k1 = 1.5, b = 0.75) {
+        // IDF component (same as TF-IDF)
+        const idf = Math.log((totalDocs - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
+        // TF component with saturation and length normalization
+        const numerator = termFrequency * (k1 + 1);
+        const denominator = termFrequency + k1 * (1 - b + b * (docLength / avgDocLength));
+        const tf = numerator / denominator;
+        return idf * tf;
+    }
+    /**
+     * Highlight search terms in text
+     */
+    highlightText(text, terms, preTag = '<mark>', postTag = '</mark>', tokenizer) {
+        if (!text || terms.length === 0) {
+            return text;
+        }
+        // Create regex pattern for all terms (case-insensitive)
+        // Escape special regex characters
+        const escapedTerms = terms.map(term => term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'));
+        // Match whole words only
+        const pattern = new RegExp(`\\b(${escapedTerms.join('|')})\\b`, 'gi');
+        // Replace matches with highlighted version
+        return text.replace(pattern, `${preTag}$1${postTag}`);
+    }
+    /**
+     * Generate text snippet with highlighted terms
+     */
+    generateSnippet(text, terms, maxLength = 150, preTag = '<mark>', postTag = '</mark>') {
+        if (!text)
+            return '';
+        // Find first occurrence of any search term
+        const lowerText = text.toLowerCase();
+        const lowerTerms = terms.map(t => t.toLowerCase());
+        let firstIndex = -1;
+        for (const term of lowerTerms) {
+            const index = lowerText.indexOf(term);
+            if (index !== -1 && (firstIndex === -1 || index < firstIndex)) {
+                firstIndex = index;
+            }
+        }
+        if (firstIndex === -1) {
+            // No terms found, return start of text
+            return text.substring(0, maxLength) + (text.length > maxLength ? '...' : '');
+        }
+        // Calculate snippet boundaries
+        const contextLength = Math.floor((maxLength - 20) / 2);
+        let start = Math.max(0, firstIndex - contextLength);
+        let end = Math.min(text.length, firstIndex + contextLength);
+        // Adjust to word boundaries
+        if (start > 0) {
+            const spaceIndex = text.lastIndexOf(' ', start);
+            if (spaceIndex !== -1)
+                start = spaceIndex + 1;
+        }
+        if (end < text.length) {
+            const spaceIndex = text.indexOf(' ', end);
+            if (spaceIndex !== -1)
+                end = spaceIndex;
+        }
+        // Extract snippet
+        let snippet = text.substring(start, end);
+        // Add ellipsis
+        if (start > 0)
+            snippet = '...' + snippet;
+        if (end < text.length)
+            snippet = snippet + '...';
+        // Highlight terms
+        snippet = this.highlightText(snippet, terms, preTag, postTag);
+        return snippet;
+    }
+    /**
+     * Calculate proximity between multiple terms in a document
+     */
+    calculateMultiTermProximity(tokens, queryTerms) {
+        if (queryTerms.length <= 1)
+            return 0;
+        const positions = new Map();
+        // Collect positions for each query term
+        for (const token of tokens) {
+            if (queryTerms.includes(token.term)) {
+                if (!positions.has(token.term)) {
+                    positions.set(token.term, []);
+                }
+                positions.get(token.term).push(...token.positions);
+            }
+        }
+        // If not all terms found, proximity is 0
+        if (positions.size < queryTerms.length) {
+            return 0;
+        }
+        // Calculate minimum span containing all terms
+        const allPositions = Array.from(positions.values());
+        let minSpan = Infinity;
+        // For each position of the first term
+        for (const pos1 of allPositions[0]) {
+            // Find closest position of other terms
+            let maxDistance = 0;
+            for (let i = 1; i < allPositions.length; i++) {
+                const positions = allPositions[i];
+                // Find closest position to pos1
+                let minDistance = Infinity;
+                for (const pos of positions) {
+                    const distance = Math.abs(pos - pos1);
+                    minDistance = Math.min(minDistance, distance);
+                }
+                maxDistance = Math.max(maxDistance, minDistance);
+            }
+            minSpan = Math.min(minSpan, maxDistance);
+        }
+        // Convert span to score (closer = higher score)
+        if (minSpan === 0)
+            return 1.0;
+        if (minSpan <= 2)
+            return 0.9;
+        if (minSpan <= 5)
+            return 0.7;
+        if (minSpan <= 10)
+            return 0.5;
+        if (minSpan <= 20)
+            return 0.3;
+        return 0.1;
+    }
+}
+exports.SearchRanker = SearchRanker;
+/**
+ * Helper function to create ranker with custom weights
+ */
+function createRanker(weights) {
+    return new SearchRanker(weights);
+}
+//# sourceMappingURL=search-ranker.js.map

package/dist/search/search-ranker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-ranker.js","sourceRoot":"","sources":["../../src/search/search-ranker.ts"],"names":[],"mappings":";AAAA;;;;;;;;GAQG;;;AAgWH,oCAEC;AAhWD,4CAMyB;AAGzB,MAAa,YAAY;IAGvB,YAAY,OAAiC;QAC3C,IAAI,CAAC,OAAO,GAAG;YACb,GAAG,gCAAuB;YAC1B,GAAG,OAAO;SACX,CAAC;IACJ,CAAC;IAED;;;;;;;;;;OAUG;IACH,cAAc,CAAC,OAAuB;QACpC,MAAM,EAAE,aAAa,EAAE,YAAY,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAE3D,sBAAsB;QACtB,0EAA0E;QAC1E,MAAM,EAAE,GAAG,aAAa,CAAC;QAEzB,mCAAmC;QACnC,kCAAkC;QAClC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC;QAE3D,OAAO,EAAE,GAAG,GAAG,CAAC;IAClB,CAAC;IAED;;;OAGG;IACH,mBAAmB,CAAC,UAAkB;QACpC,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;;OAGG;IACH,uBAAuB,CAAC,cAAuB;QAC7C,OAAO,cAAc,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED;;;OAGG;IACH,uBAAuB,CAAC,SAAkB,EAAE,YAAoB,EAAE;QAChE,IAAI,CAAC,SAAS;YAAE,OAAO,CAAC,CAAC;QAEzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,GAAG,GAAG,SAAS,CAAC;QAC9B,MAAM,OAAO,GAAG,KAAK,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QAE9C,sDAAsD;QACtD,+BAA+B;QAC/B,gCAAgC;QAChC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,cAAc,CAAC,OAAuB;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAChE,MAAM,SAAS,GAAG,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAEvE,8BAA8B;QAC9B,IAAI,KAAK,GACP,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW;YAChC,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB;YAC1C,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC;QAE3C,yBAAyB;QACzB,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,sCAAsC;QAEvE,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,WAAW,CACT,OAA0C,EAC1C,UAAoB,EACpB,SAMC,EACD,SAAiB,EACjB,WAAoC,EACpC,eAAqC;QAErC,MAAM,MAAM,GAAsB,EAAE,CAAC;QAErC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,MAAM,YAAY,GAAa,EAAE,CAAC;YAElC,sCAAsC;YACtC,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;gBAC9B,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAClC,IAAI,CAAC,KAAK;oBAAE,SAAS;gBAErB,MAAM,QAAQ,GAAG,KAAK,CAAC,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC9D,IAAI,CAAC,QAAQ;oBAAE,SAAS;gBAExB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAExB,gCAAgC;gBAChC,MAAM,OAAO,GAAmB;oBAC9B,IAAI;oBACJ,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,aAAa,EAAE,QAAQ;oBACvB,YAAY,EAAE,KAAK,CAAC,YAAY;oBAChC,SAAS;oBACT,UAAU,EAAE,IAAI,CAAC,oBAAoB,CAAC,WAAW,IAAI,EAAE,CAAC;oBACxD,cAAc,EAAE,eAAe,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC;iBACnD,CAAC;gBAEF,UAAU,IAAI,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC7C,CAAC;YAED,yCAAyC;YACzC,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YAEhF,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,QAAQ;gBACf,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,YAAY;aACb,CAAC,CAAC;QACL,CAAC;QAED,gCAAgC;QAChC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAEzC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAC,WAAmC;QAC9D,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,GAAG,CAAC;QAEpC,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC;QACtD,OAAO,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,aAAa,CACX,aAAqB,EACrB,YAAoB,EACpB,SAAiB,EACjB,SAAiB,EACjB,YAAoB,EACpB,KAAa,GAAG,EAChB,IAAY,IAAI;QAEhB,iCAAiC;QACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,SAAS,GAAG,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QAElF,wDAAwD;QACxD,MAAM,SAAS,GAAG,aAAa,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,aAAa,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,GAAG,YAAY,CAAC,CAAC,CAAC;QAClF,MAAM,EAAE,GAAG,SAAS,GAAG,WAAW,CAAC;QAEnC,OAAO,GAAG,GAAG,EAAE,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,aAAa,CACX,IAAY,EACZ,KAAe,EACf,SAAiB,QAAQ,EACzB,UAAkB,SAAS,EAC3B,SAAqB;QAErB,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,wDAAwD;QACxD,kCAAkC;QAClC,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACpC,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAC5C,CAAC;QAEF,yBAAyB;QACzB,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,OAAO,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAEtE,2CAA2C;QAC3C,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,MAAM,KAAK,OAAO,EAAE,CAAC,CAAC;IACxD,CAAC;IAED;;OAEG;IACH,eAAe,CACb,IAAY,EACZ,KAAe,EACf,YAAoB,GAAG,EACvB,SAAiB,QAAQ,EACzB,UAAkB,SAAS;QAE3B,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QAErB,2CAA2C;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QAEnD,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;QACpB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,IAAI,KAAK,GAAG,UAAU,CAAC,EAAE,CAAC;gBAC9D,UAAU,GAAG,KAAK,CAAC;YACrB,CAAC;QACH,CAAC;QAED,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;YACtB,uCAAuC;YACvC,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC/E,CAAC;QAED,+BAA+B;QAC/B,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,IAAI,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,aAAa,CAAC,CAAC;QACpD,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,GAAG,aAAa,CAAC,CAAC;QAE5D,4BAA4B;QAC5B,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAChD,IAAI,UAAU,KAAK,CAAC,CAAC;gBAAE,KAAK,GAAG,UAAU,GAAG,CAAC,CAAC;QAChD,CAAC;QAED,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAC1C,IAAI,UAAU,KAAK,CAAC,CAAC;gBAAE,GAAG,GAAG,UAAU,CAAC;QAC1C,CAAC;QAED,kBAAkB;QAClB,IAAI,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAEzC,eAAe;QACf,IAAI,KAAK,GAAG,CAAC;YAAE,OAAO,GAAG,KAAK,GAAG,OAAO,CAAC;QACzC,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM;YAAE,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;QAEjD,kBAAkB;QAClB,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QAE9D,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,2BAA2B,CAAC,MAAuB,EAAE,UAAoB;QACvE,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC;QAErC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAC;QAE9C,wCAAwC;QACxC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC/B,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAChC,CAAC;gBACD,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QAED,yCAAyC;QACzC,IAAI,SAAS,CAAC,IAAI,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;YACvC,OAAO,CAAC,CAAC;QACX,CAAC;QAED,8CAA8C;QAC9C,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QACpD,IAAI,OAAO,GAAG,QAAQ,CAAC;QAEvB,sCAAsC;QACtC,KAAK,MAAM,IAAI,IAAI,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC;YACnC,uCAAuC;YACvC,IAAI,WAAW,GAAG,CAAC,CAAC;YAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7C,MAAM,SAAS,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;gBAElC,gCAAgC;gBAChC,IAAI,WAAW,GAAG,QAAQ,CAAC;gBAC3B,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;oBAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;oBACtC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;gBAChD,CAAC;gBAED,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;YACnD,CAAC;YAED,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAC3C,CAAC;QAED,gDAAgD;QAChD,IAAI,OAAO,KAAK,CAAC;YAAE,OAAO,GAAG,CAAC;QAC9B,IAAI,OAAO,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC;QAC7B,IAAI,OAAO,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC;QAC7B,IAAI,OAAO,IAAI,EAAE;YAAE,OAAO,GAAG,CAAC;QAC9B,IAAI,OAAO,IAAI,EAAE;YAAE,OAAO,GAAG,CAAC;QAC9B,OAAO,GAAG,CAAC;IACb,CAAC;CACF;AAhVD,oCAgVC;AAED;;GAEG;AACH,SAAgB,YAAY,CAAC,OAAiC;IAC5D,OAAO,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;AACnC,CAAC"}

package/dist/search/tokenizer.d.ts

@@ -0,0 +1,80 @@
+/**
+ * Text Tokenizer
+ *
+ * Converts text into searchable tokens using various strategies:
+ * - Simple: Split by whitespace, lowercase
+ * - Stemming: Reduce words to root form (running → run)
+ * - N-gram: Split into character sequences (plumb → pl, plu, lum, umb)
+ */
+import { TokenizerOptions, TokenMetadata } from '../types/search';
+export declare class Tokenizer {
+    private options;
+    constructor(options: Partial<TokenizerOptions>);
+    /**
+     * Tokenize text into searchable terms
+     */
+    tokenize(text: string, field?: string): TokenMetadata[];
+    /**
+     * Simple tokenization: split by whitespace, lowercase, remove punctuation
+     */
+    private tokenizeSimple;
+    /**
+     * Stemming tokenization: reduce words to root form
+     * Uses Porter Stemmer algorithm (simplified)
+     */
+    private tokenizeStemming;
+    /**
+     * N-gram tokenization: split into character sequences
+     * Example: "plumb" with n=3 → ["plu", "lum", "umb"]
+     */
+    private tokenizeNgram;
+    /**
+     * Porter Stemmer (simplified version)
+     * Reduces words to their root form
+     */
+    private stem;
+    private step1;
+    private step2;
+    private step3;
+    private step4;
+    private step5;
+    /**
+     * Measure the "size" of a word (number of consonant-vowel sequences)
+     */
+    private measureWord;
+    /**
+     * Check if word contains a vowel
+     */
+    private hasVowel;
+    /**
+     * Check if word ends with consonant-vowel-consonant pattern
+     */
+    private endsWithCVC;
+    /**
+     * Fix word ending after removing suffix
+     * Example: "hopping" → "hopp" → "hop"
+     */
+    private fixEnding;
+    /**
+     * Merge duplicate terms and combine their positions
+     */
+    private mergePositions;
+    /**
+     * Get all unique terms from tokens (useful for indexing)
+     */
+    getUniqueTerms(tokens: TokenMetadata[]): string[];
+    /**
+     * Calculate proximity score between terms
+     * Terms closer together get higher scores
+     */
+    calculateProximity(tokens: TokenMetadata[], term1: string, term2: string): number;
+}
+/**
+ * Helper function to create tokenizer from config
+ */
+export declare function createTokenizer(type: 'simple' | 'stemming' | 'ngram', options?: Partial<TokenizerOptions>): Tokenizer;
+/**
+ * Helper function to tokenize multiple fields
+ */
+export declare function tokenizeFields(tokenizer: Tokenizer, data: Record<string, any>, fields: string[]): TokenMetadata[];
+//# sourceMappingURL=tokenizer.d.ts.map

package/dist/search/tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../src/search/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAsB,MAAM,iBAAiB,CAAC;AAEtF,qBAAa,SAAS;IACpB,OAAO,CAAC,OAAO,CAA6B;gBAEhC,OAAO,EAAE,OAAO,CAAC,gBAAgB,CAAC;IAU9C;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,GAAE,MAAkB,GAAG,aAAa,EAAE;IAiBlE;;OAEG;IACH,OAAO,CAAC,cAAc;IAyCtB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAWxB;;;OAGG;IACH,OAAO,CAAC,aAAa;IAqCrB;;;OAGG;IACH,OAAO,CAAC,IAAI;IAmBZ,OAAO,CAAC,KAAK;IAqCb,OAAO,CAAC,KAAK;IAoCb,OAAO,CAAC,KAAK;IAuBb,OAAO,CAAC,KAAK;IAmBb,OAAO,CAAC,KAAK;IAkBb;;OAEG;IACH,OAAO,CAAC,WAAW;IAiBnB;;OAEG;IACH,OAAO,CAAC,QAAQ;IAIhB;;OAEG;IACH,OAAO,CAAC,WAAW;IAkBnB;;;OAGG;IACH,OAAO,CAAC,SAAS;IAuBjB;;OAEG;IACH,OAAO,CAAC,cAAc;IAiBtB;;OAEG;IACH,cAAc,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,EAAE;IAIjD;;;OAGG;IACH,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM;CA0BlF;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,IAAI,EAAE,QAAQ,GAAG,UAAU,GAAG,OAAO,EACrC,OAAO,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAClC,SAAS,CAKX;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC5B,SAAS,EAAE,SAAS,EACpB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EACzB,MAAM,EAAE,MAAM,EAAE,GACf,aAAa,EAAE,CAYjB"}