@bhushanpawar/sqldb 1.0.6 → 1.0.7

package/dist/query/query-tracker.js.map

@@ -1 +1 @@
-{"version":3,"file":"query-tracker.js","sourceRoot":"","sources":["../../src/query/query-tracker.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwCA,0CAUC;AAjDD,+CAAiC;AAEjC,MAAa,oBAAoB;IAAjC;QACU,YAAO,GAAiC,IAAI,GAAG,EAAE,CAAC;QAClD,eAAU,GAAoB,EAAE,CAAC;IA8B3C,CAAC;IA5BC,UAAU,CAAC,QAAuB;QAChC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE/B,IAAI,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YAChE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,UAAU,CAAC,aAAsB;QAC/B,IAAI,aAAa,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;QAC/C,CAAC;QACD,OAAO,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;IAC9B,CAAC;IAED,YAAY,CAAC,aAAsB;QACjC,IAAI,aAAa,EAAE,CAAC;YAClB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACnC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CACtC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,aAAa,CACzC,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACrB,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;CACF;AAhCD,oDAgCC;AAED;;GAEG;AACH,SAAgB,eAAe;IAC7B,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACtB,OAAO,MAAM,CAAC,UAAU,EAAE,CAAC;IAC7B,CAAC;IACD,mCAAmC;IACnC,OAAO,sCAAsC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE;QACnE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;QAC1C,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"query-tracker.js","sourceRoot":"","sources":["../../src/query/query-tracker.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoKA,0CAUC;AA7KD,+CAAiC;AAmBjC,MAAa,oBAAoB;IAK/B,YAAY,qBAA6B,IAAI;QAJrC,YAAO,GAAiC,IAAI,GAAG,EAAE,CAAC;QAClD,eAAU,GAAoB,EAAE,CAAC;QACjC,uBAAkB,GAAW,IAAI,CAAC,CAAC,KAAK;QAG9C,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;IAC/C,CAAC;IAED,UAAU,CAAC,QAAuB;QAChC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE/B,IAAI,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YAChE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,UAAU,CAAC,aAAsB;QAC/B,IAAI,aAAa,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;QAC/C,CAAC;QACD,OAAO,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;IAC9B,CAAC;IAED,YAAY,CAAC,aAAsB;QACjC,IAAI,aAAa,EAAE,CAAC;YAClB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACnC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CACtC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,aAAa,CACzC,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACrB,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,UAAU;QACR,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;QAE5C,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO;gBACL,YAAY,EAAE,CAAC;gBACf,cAAc,EAAE,EAAE;gBAClB,kBAAkB,EAAE,EAAE;gBACtB,WAAW,EAAE,CAAC;gBACd,WAAW,EAAE,CAAC;gBACd,WAAW,EAAE,CAAC;gBACd,WAAW,EAAE,EAAE;gBACf,SAAS,EAAE,CAAC;gBACZ,WAAW,EAAE,CAAC;aACf,CAAC;QACJ,CAAC;QAED,MAAM,cAAc,GAA2B,EAAE,CAAC;QAClD,MAAM,kBAAkB,GAA2B,EAAE,CAAC;QACtD,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,MAAM,WAAW,GAKZ,EAAE,CAAC;QACR,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpC,iBAAiB;YACjB,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;gBACpB,cAAc,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAC/E,CAAC;YAED,qBAAqB;YACrB,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;gBACpB,kBAAkB,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,kBAAkB,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YACvF,CAAC;YAED,kBAAkB;YAClB,IAAI,KAAK,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;gBACxC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;gBAEtC,qBAAqB;gBACrB,IAAI,KAAK,CAAC,eAAe,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;oBACrD,WAAW,CAAC,IAAI,CAAC;wBACf,KAAK,EAAE,KAAK,CAAC,GAAG;wBAChB,QAAQ,EAAE,KAAK,CAAC,eAAe;wBAC/B,SAAS,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;wBACpC,SAAS,EAAE,KAAK,CAAC,SAAS;qBAC3B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,eAAe;YACf,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBAChB,WAAW,EAAE,CAAC;YAChB,CAAC;QACH,CAAC;QAED,2BAA2B;QAC3B,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC;YACtC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM;YAC7D,CAAC,CAAC,CAAC,CAAC;QACN,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEtE,6CAA6C;QAC7C,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QAEpD,MAAM,SAAS,GAAG,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5E,OAAO;YACL,YAAY;YACZ,cAAc;YACd,kBAAkB;YAClB,WAAW;YACX,WAAW;YACX,WAAW;YACX,WAAW,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,sBAAsB;YAC7D,SAAS;YACT,WAAW;SACZ,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,YAAY,EAAE,CAAC;IACtB,CAAC;IAED;;OAEG;IACH,qBAAqB,CAAC,EAAU;QAC9B,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC;IAC/B,CAAC;CACF;AA3ID,oDA2IC;AAED;;GAEG;AACH,SAAgB,eAAe;IAC7B,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACtB,OAAO,MAAM,CAAC,UAAU,EAAE,CAAC;IAC7B,CAAC;IACD,mCAAmC;IACnC,OAAO,sCAAsC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE;QACnE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;QAC1C,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/search/inverted-index-manager.d.ts

@@ -0,0 +1,76 @@
+/**
+ * Inverted Index Manager
+ *
+ * Manages inverted indexes stored in Redis for fast text search.
+ * An inverted index maps words → document IDs for instant lookup.
+ *
+ * Storage structure in Redis:
+ * - Key: "sqldb:index:{table}:word:{word}"
+ * - Value: Sorted Set [(docId, score), ...]
+ *
+ * - Key: "sqldb:index:{table}:doc:{docId}"
+ * - Value: Set [word1, word2, ...]
+ *
+ * - Key: "sqldb:index:{table}:meta"
+ * - Value: Hash { lastBuild, docCount, termCount }
+ */
+import { RedisConnectionManager } from '../connection/redis';
+import { InvertedIndexConfig, IndexStats } from '../types/search';
+export declare class InvertedIndexManager {
+    private redis;
+    private config;
+    private tokenizers;
+    private keyPrefix;
+    constructor(redis: RedisConnectionManager, config: InvertedIndexConfig, keyPrefix?: string);
+    /**
+     * Build inverted index for a table from scratch
+     */
+    buildIndex(tableName: string, documents: Array<{
+        id: number;
+        [key: string]: any;
+    }>): Promise<IndexStats>;
+    /**
+     * Update index for a single document
+     */
+    updateDocument(tableName: string, docId: string | number, data: Record<string, any>): Promise<void>;
+    /**
+     * Delete document from index
+     */
+    deleteDocument(tableName: string, docId: string | number): Promise<void>;
+    /**
+     * Search for documents matching query terms
+     * Returns document IDs as strings (supports both numeric IDs and UUIDs)
+     */
+    search(tableName: string, query: string, limit?: number): Promise<string[]>;
+    /**
+     * Get document IDs for a specific term
+     */
+    getDocumentsForTerm(tableName: string, term: string, limit?: number): Promise<string[]>;
+    /**
+     * Get statistics for an index
+     */
+    getStats(tableName: string): Promise<IndexStats | null>;
+    /**
+     * Clear entire index for a table
+     */
+    clearIndex(tableName: string): Promise<void>;
+    /**
+     * Get all indexed terms for a table
+     */
+    getAllTerms(tableName: string, limit?: number): Promise<string[]>;
+    /**
+     * Check if index exists for a table
+     */
+    indexExists(tableName: string): Promise<boolean>;
+    private buildWordKey;
+    private buildDocKey;
+    private buildMetaKey;
+    private storeMetadata;
+    /**
+     * Extract document ID from a record
+     * Supports: id, {table}_id, or first field ending in '_id'
+     * Returns a string representation for Redis storage (works with both UUIDs and numbers)
+     */
+    private extractDocumentId;
+}
+//# sourceMappingURL=inverted-index-manager.d.ts.map

package/dist/search/inverted-index-manager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"inverted-index-manager.d.ts","sourceRoot":"","sources":["../../src/search/inverted-index-manager.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAGH,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,mBAAmB,EAAE,UAAU,EAAiB,MAAM,iBAAiB,CAAC;AAGjF,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,KAAK,CAAyB;IACtC,OAAO,CAAC,MAAM,CAAsB;IACpC,OAAO,CAAC,UAAU,CAAqC;IACvD,OAAO,CAAC,SAAS,CAAS;gBAEd,KAAK,EAAE,sBAAsB,EAAE,MAAM,EAAE,mBAAmB,EAAE,SAAS,CAAC,EAAE,MAAM;IAmB1F;;OAEG;IACG,UAAU,CACd,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAE,CAAC,GACnD,OAAO,CAAC,UAAU,CAAC;IA2GtB;;OAEG;IACG,cAAc,CAClB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GAAG,MAAM,EACtB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GACxB,OAAO,CAAC,IAAI,CAAC;IAuDhB;;OAEG;IACG,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA4B9E;;;OAGG;IACG,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAW,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAiDrF;;OAEG;IACG,mBAAmB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,GAAE,MAAY,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAYlG;;OAEG;IACG,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAwB7D;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAelD;;OAEG;IACG,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,GAAE,MAAa,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAgB7E;;OAEG;IACG,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAOtD,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,WAAW;IAInB,OAAO,CAAC,YAAY;YAIN,aAAa;IAiB3B;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;CA2B1B"}

package/dist/search/inverted-index-manager.js

@@ -0,0 +1,371 @@
+"use strict";
+/**
+ * Inverted Index Manager
+ *
+ * Manages inverted indexes stored in Redis for fast text search.
+ * An inverted index maps words → document IDs for instant lookup.
+ *
+ * Storage structure in Redis:
+ * - Key: "sqldb:index:{table}:word:{word}"
+ * - Value: Sorted Set [(docId, score), ...]
+ *
+ * - Key: "sqldb:index:{table}:doc:{docId}"
+ * - Value: Set [word1, word2, ...]
+ *
+ * - Key: "sqldb:index:{table}:meta"
+ * - Value: Hash { lastBuild, docCount, termCount }
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InvertedIndexManager = void 0;
+const tokenizer_1 = require("./tokenizer");
+class InvertedIndexManager {
+    constructor(redis, config, keyPrefix) {
+        this.tokenizers = new Map();
+        this.redis = redis;
+        this.config = config;
+        this.keyPrefix = keyPrefix || 'sqldb';
+        // Initialize tokenizers for each table
+        for (const [tableName, tableConfig] of Object.entries(config.tables)) {
+            this.tokenizers.set(tableName, new tokenizer_1.Tokenizer({
+                type: tableConfig.tokenizer,
+                minWordLength: tableConfig.minWordLength,
+                stopWords: tableConfig.stopWords,
+                caseSensitive: tableConfig.caseSensitive,
+            }));
+        }
+    }
+    /**
+     * Build inverted index for a table from scratch
+     */
+    async buildIndex(tableName, documents) {
+        const startTime = Date.now();
+        const tableConfig = this.config.tables[tableName];
+        if (!tableConfig) {
+            throw new Error(`No search configuration found for table: ${tableName}`);
+        }
+        const tokenizer = this.tokenizers.get(tableName);
+        if (!tokenizer) {
+            throw new Error(`No tokenizer found for table: ${tableName}`);
+        }
+        // Clear existing index
+        await this.clearIndex(tableName);
+        const client = this.redis.getClient();
+        if (!client) {
+            throw new Error('Redis client not connected');
+        }
+        let totalTerms = 0;
+        let totalTokens = 0;
+        const termFrequency = new Map(); // Track document frequency per term
+        // Build index for each document
+        for (const doc of documents) {
+            // Try to find the document ID (supports id, {table}_id, or any numeric primary key)
+            const docId = this.extractDocumentId(doc, tableName);
+            // Skip documents without an ID
+            if (docId === undefined || docId === null) {
+                console.warn(`Skipping document without ID in table ${tableName}:`, Object.keys(doc).join(', '));
+                continue;
+            }
+            // Tokenize all searchable fields
+            const tokens = (0, tokenizer_1.tokenizeFields)(tokenizer, doc, tableConfig.searchableFields);
+            totalTokens += tokens.length;
+            // Group tokens by term
+            const termMap = new Map();
+            for (const token of tokens) {
+                if (!termMap.has(token.term)) {
+                    termMap.set(token.term, token);
+                }
+                else {
+                    // Merge positions
+                    const existing = termMap.get(token.term);
+                    existing.positions.push(...token.positions);
+                }
+            }
+            // Store each term in Redis
+            const pipeline = client.pipeline();
+            for (const [term, metadata] of termMap.entries()) {
+                // Calculate term frequency for this document
+                const tf = metadata.positions.length;
+                // Field boost (if configured)
+                const fieldBoost = tableConfig.fieldBoosts?.[metadata.field] || 1.0;
+                // Combine TF and field boost for initial score
+                const score = tf * fieldBoost;
+                // Add to inverted index: term → [docId, score]
+                const wordKey = this.buildWordKey(tableName, term);
+                pipeline.zadd(wordKey, score, String(docId));
+                // Track document frequency
+                termFrequency.set(term, (termFrequency.get(term) || 0) + 1);
+            }
+            // Store document → terms mapping (for updates/deletes)
+            const docKey = this.buildDocKey(tableName, docId);
+            const terms = Array.from(termMap.keys());
+            if (terms.length > 0) {
+                pipeline.sadd(docKey, ...terms);
+            }
+            // Execute pipeline
+            await pipeline.exec();
+        }
+        // Store metadata
+        totalTerms = termFrequency.size;
+        await this.storeMetadata(tableName, {
+            tableName,
+            totalDocuments: documents.length,
+            totalTerms,
+            totalTokens,
+            lastBuildTime: Date.now(),
+            buildDurationMs: Date.now() - startTime,
+            fields: tableConfig.searchableFields,
+        });
+        return {
+            tableName,
+            totalDocuments: documents.length,
+            totalTerms,
+            totalTokens,
+            lastBuildTime: Date.now(),
+            buildDurationMs: Date.now() - startTime,
+            fields: tableConfig.searchableFields,
+        };
+    }
+    /**
+     * Update index for a single document
+     */
+    async updateDocument(tableName, docId, data) {
+        const tableConfig = this.config.tables[tableName];
+        if (!tableConfig) {
+            throw new Error(`No search configuration found for table: ${tableName}`);
+        }
+        const tokenizer = this.tokenizers.get(tableName);
+        if (!tokenizer) {
+            throw new Error(`No tokenizer found for table: ${tableName}`);
+        }
+        const client = this.redis.getClient();
+        if (!client) {
+            throw new Error('Redis client not connected');
+        }
+        // First, remove old document from index
+        await this.deleteDocument(tableName, docId);
+        // Tokenize new data
+        const tokens = (0, tokenizer_1.tokenizeFields)(tokenizer, data, tableConfig.searchableFields);
+        // Group tokens by term
+        const termMap = new Map();
+        for (const token of tokens) {
+            if (!termMap.has(token.term)) {
+                termMap.set(token.term, token);
+            }
+            else {
+                const existing = termMap.get(token.term);
+                existing.positions.push(...token.positions);
+            }
+        }
+        // Store each term in Redis
+        const pipeline = client.pipeline();
+        for (const [term, metadata] of termMap.entries()) {
+            const tf = metadata.positions.length;
+            const fieldBoost = tableConfig.fieldBoosts?.[metadata.field] || 1.0;
+            const score = tf * fieldBoost;
+            const wordKey = this.buildWordKey(tableName, term);
+            pipeline.zadd(wordKey, score, String(docId));
+        }
+        // Store document → terms mapping
+        const docKey = this.buildDocKey(tableName, docId);
+        const terms = Array.from(termMap.keys());
+        if (terms.length > 0) {
+            pipeline.sadd(docKey, ...terms);
+        }
+        await pipeline.exec();
+    }
+    /**
+     * Delete document from index
+     */
+    async deleteDocument(tableName, docId) {
+        const client = this.redis.getClient();
+        if (!client) {
+            throw new Error('Redis client not connected');
+        }
+        // Get all terms for this document
+        const docKey = this.buildDocKey(tableName, docId);
+        const terms = await client.smembers(docKey);
+        if (terms.length === 0) {
+            return;
+        }
+        // Remove document from each term's sorted set
+        const pipeline = client.pipeline();
+        for (const term of terms) {
+            const wordKey = this.buildWordKey(tableName, term);
+            pipeline.zrem(wordKey, String(docId));
+        }
+        // Delete document key
+        pipeline.del(docKey);
+        await pipeline.exec();
+    }
+    /**
+     * Search for documents matching query terms
+     * Returns document IDs as strings (supports both numeric IDs and UUIDs)
+     */
+    async search(tableName, query, limit = 10) {
+        const tableConfig = this.config.tables[tableName];
+        if (!tableConfig) {
+            throw new Error(`No search configuration found for table: ${tableName}`);
+        }
+        const tokenizer = this.tokenizers.get(tableName);
+        if (!tokenizer) {
+            throw new Error(`No tokenizer found for table: ${tableName}`);
+        }
+        const client = this.redis.getClient();
+        if (!client) {
+            throw new Error('Redis client not connected');
+        }
+        // Tokenize query
+        const tokens = tokenizer.tokenize(query);
+        if (tokens.length === 0) {
+            return [];
+        }
+        const terms = tokenizer.getUniqueTerms(tokens);
+        // If single term, just get documents for that term
+        if (terms.length === 1) {
+            const wordKey = this.buildWordKey(tableName, terms[0]);
+            const results = await client.zrevrange(wordKey, 0, limit - 1);
+            return results; // Already strings
+        }
+        // Multiple terms: use Redis ZINTERSTORE for intersection
+        const tempKey = `${this.keyPrefix}:temp:search:${Date.now()}`;
+        const wordKeys = terms.map(term => this.buildWordKey(tableName, term));
+        try {
+            // Intersect all term keys (documents must contain ALL terms)
+            await client.zinterstore(tempKey, terms.length, ...wordKeys, 'AGGREGATE', 'SUM');
+            // Get top results
+            const results = await client.zrevrange(tempKey, 0, limit - 1);
+            return results; // Already strings
+        }
+        finally {
+            // Clean up temp key
+            await client.del(tempKey);
+        }
+    }
+    /**
+     * Get document IDs for a specific term
+     */
+    async getDocumentsForTerm(tableName, term, limit = 100) {
+        const client = this.redis.getClient();
+        if (!client) {
+            return [];
+        }
+        const wordKey = this.buildWordKey(tableName, term);
+        const results = await client.zrevrange(wordKey, 0, limit - 1);
+        return results; // Already strings
+    }
+    /**
+     * Get statistics for an index
+     */
+    async getStats(tableName) {
+        const client = this.redis.getClient();
+        if (!client) {
+            return null;
+        }
+        const metaKey = this.buildMetaKey(tableName);
+        const meta = await client.hgetall(metaKey);
+        if (!meta || Object.keys(meta).length === 0) {
+            return null;
+        }
+        return {
+            tableName,
+            totalDocuments: parseInt(meta.totalDocuments || '0', 10),
+            totalTerms: parseInt(meta.totalTerms || '0', 10),
+            totalTokens: parseInt(meta.totalTokens || '0', 10),
+            lastBuildTime: parseInt(meta.lastBuildTime || '0', 10),
+            buildDurationMs: parseInt(meta.buildDurationMs || '0', 10),
+            fields: meta.fields ? JSON.parse(meta.fields) : [],
+        };
+    }
+    /**
+     * Clear entire index for a table
+     */
+    async clearIndex(tableName) {
+        const client = this.redis.getClient();
+        if (!client) {
+            return;
+        }
+        // Delete all keys matching the table pattern
+        const pattern = `${this.keyPrefix}:index:${tableName}:*`;
+        const keys = await this.redis.scan(pattern);
+        if (keys.length > 0) {
+            await client.del(...keys);
+        }
+    }
+    /**
+     * Get all indexed terms for a table
+     */
+    async getAllTerms(tableName, limit = 1000) {
+        const client = this.redis.getClient();
+        if (!client) {
+            return [];
+        }
+        const pattern = `${this.keyPrefix}:index:${tableName}:word:*`;
+        const keys = await this.redis.scan(pattern);
+        // Extract term from key
+        const prefix = `${this.keyPrefix}:index:${tableName}:word:`;
+        const terms = keys.map(key => key.substring(prefix.length)).slice(0, limit);
+        return terms;
+    }
+    /**
+     * Check if index exists for a table
+     */
+    async indexExists(tableName) {
+        const stats = await this.getStats(tableName);
+        return stats !== null && stats.totalDocuments > 0;
+    }
+    // ========== Private Helper Methods ==========
+    buildWordKey(tableName, term) {
+        return `${this.keyPrefix}:index:${tableName}:word:${term}`;
+    }
+    buildDocKey(tableName, docId) {
+        return `${this.keyPrefix}:index:${tableName}:doc:${docId}`;
+    }
+    buildMetaKey(tableName) {
+        return `${this.keyPrefix}:index:${tableName}:meta`;
+    }
+    async storeMetadata(tableName, stats) {
+        const client = this.redis.getClient();
+        if (!client) {
+            return;
+        }
+        const metaKey = this.buildMetaKey(tableName);
+        await client.hset(metaKey, {
+            totalDocuments: stats.totalDocuments.toString(),
+            totalTerms: stats.totalTerms.toString(),
+            totalTokens: stats.totalTokens.toString(),
+            lastBuildTime: stats.lastBuildTime.toString(),
+            buildDurationMs: stats.buildDurationMs.toString(),
+            fields: JSON.stringify(stats.fields),
+        });
+    }
+    /**
+     * Extract document ID from a record
+     * Supports: id, {table}_id, or first field ending in '_id'
+     * Returns a string representation for Redis storage (works with both UUIDs and numbers)
+     */
+    extractDocumentId(doc, tableName) {
+        // Try 'id' first
+        if (doc.id !== undefined && doc.id !== null) {
+            return String(doc.id);
+        }
+        // Try '{table}_id' (e.g., service_id, user_id)
+        const tableIdField = `${tableName.replace(/s$/, '')}_id`; // services → service_id
+        if (doc[tableIdField] !== undefined && doc[tableIdField] !== null) {
+            return String(doc[tableIdField]);
+        }
+        // Try exact table name with _id
+        const exactTableIdField = `${tableName}_id`;
+        if (doc[exactTableIdField] !== undefined && doc[exactTableIdField] !== null) {
+            return String(doc[exactTableIdField]);
+        }
+        // As last resort, find first field that ends with '_id'
+        for (const key of Object.keys(doc)) {
+            if (key.endsWith('_id') && doc[key] !== undefined && doc[key] !== null) {
+                return String(doc[key]);
+            }
+        }
+        return undefined;
+    }
+}
+exports.InvertedIndexManager = InvertedIndexManager;
+//# sourceMappingURL=inverted-index-manager.js.map

package/dist/search/inverted-index-manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"inverted-index-manager.js","sourceRoot":"","sources":["../../src/search/inverted-index-manager.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;GAeG;;;AAKH,2CAAwD;AAExD,MAAa,oBAAoB;IAM/B,YAAY,KAA6B,EAAE,MAA2B,EAAE,SAAkB;QAHlF,eAAU,GAA2B,IAAI,GAAG,EAAE,CAAC;QAIrD,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,OAAO,CAAC;QAEtC,uCAAuC;QACvC,KAAK,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YACrE,IAAI,CAAC,UAAU,CAAC,GAAG,CACjB,SAAS,EACT,IAAI,qBAAS,CAAC;gBACZ,IAAI,EAAE,WAAW,CAAC,SAAS;gBAC3B,aAAa,EAAE,WAAW,CAAC,aAAa;gBACxC,SAAS,EAAE,WAAW,CAAC,SAAS;gBAChC,aAAa,EAAE,WAAW,CAAC,aAAa;aACzC,CAAC,CACH,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CACd,SAAiB,EACjB,SAAoD;QAEpD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAElD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,4CAA4C,SAAS,EAAE,CAAC,CAAC;QAC3E,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,iCAAiC,SAAS,EAAE,CAAC,CAAC;QAChE,CAAC;QAED,uBAAuB;QACvB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAEjC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC;QAED,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC,CAAC,oCAAoC;QAErF,gCAAgC;QAChC,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,oFAAoF;YACpF,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;YAErD,+BAA+B;YAC/B,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBAC1C,OAAO,CAAC,IAAI,CAAC,yCAAyC,SAAS,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;gBACjG,SAAS;YACX,CAAC;YAED,iCAAiC;YACjC,MAAM,MAAM,GAAG,IAAA,0BAAc,EAAC,SAAS,EAAE,GAAG,EAAE,WAAW,CAAC,gBAAgB,CAAC,CAAC;YAC5E,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC;YAE7B,uBAAuB;YACvB,MAAM,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;YACjD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBACjC,CAAC;qBAAM,CAAC;oBACN,kBAAkB;oBAClB,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;oBAC1C,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;gBAC9C,CAAC;YACH,CAAC;YAED,2BAA2B;YAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;YAEnC,KAAK,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;gBACjD,6CAA6C;gBAC7C,MAAM,EAAE,GAAG,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC;gBAErC,8BAA8B;gBAC9B,MAAM,UAAU,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC;gBAEpE,+CAA+C;gBAC/C,MAAM,KAAK,GAAG,EAAE,GAAG,UAAU,CAAC;gBAE9B,+CAA+C;gBAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;gBACnD,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAE7C,2BAA2B;gBAC3B,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9D,CAAC;YAED,uDAAuD;YACvD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YAClD,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YACzC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC;YAClC,CAAC;YAED,mBAAmB;YACnB,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACxB,CAAC;QAED,iBAAiB;QACjB,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC;QAChC,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE;YAClC,SAAS;YACT,cAAc,EAAE,SAAS,CAAC,MAAM;YAChC,UAAU;YACV,WAAW;YACX,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE;YACzB,eAAe,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YACvC,MAAM,EAAE,WAAW,CAAC,gBAAgB;SACrC,CAAC,CAAC;QAEH,OAAO;YACL,SAAS;YACT,cAAc,EAAE,SAAS,CAAC,MAAM;YAChC,UAAU;YACV,WAAW;YACX,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE;YACzB,eAAe,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YACvC,MAAM,EAAE,WAAW,CAAC,gBAAgB;SACrC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAClB,SAAiB,EACjB,KAAsB,EACtB,IAAyB;QAEzB,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,4CAA4C,SAAS,EAAE,CAAC,CAAC;QAC3E,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,iCAAiC,SAAS,EAAE,CAAC,CAAC;QAChE,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC;QAED,wCAAwC;QACxC,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAE5C,oBAAoB;QACpB,MAAM,MAAM,GAAG,IAAA,0BAAc,EAAC,SAAS,EAAE,IAAI,EAAE,WAAW,CAAC,gBAAgB,CAAC,CAAC;QAE7E,uBAAuB;QACvB,MAAM,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;QACjD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACjC,CAAC;iBAAM,CAAC;gBACN,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;gBAC1C,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;QAED,2BAA2B;QAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;QAEnC,KAAK,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YACjD,MAAM,EAAE,GAAG,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC;YACrC,MAAM,UAAU,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC;YACpE,MAAM,KAAK,GAAG,EAAE,GAAG,UAAU,CAAC;YAE9B,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YACnD,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAC/C,CAAC;QAED,iCAAiC;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAClD,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QACzC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,SAAiB,EAAE,KAAsB;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC;QAED,kCAAkC;QAClC,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAClD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAE5C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QAED,8CAA8C;QAC9C,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;QAEnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YACnD,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACxC,CAAC;QAED,sBAAsB;QACtB,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAErB,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,SAAiB,EAAE,KAAa,EAAE,QAAgB,EAAE;QAC/D,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,4CAA4C,SAAS,EAAE,CAAC,CAAC;QAC3E,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,iCAAiC,SAAS,EAAE,CAAC,CAAC;QAChE,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC;QAED,iBAAiB;QACjB,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,KAAK,GAAG,SAAS,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAE/C,mDAAmD;QACnD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACvD,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YAC9D,OAAO,OAAO,CAAC,CAAC,kBAAkB;QACpC,CAAC;QAED,yDAAyD;QACzD,MAAM,OAAO,GAAG,GAAG,IAAI,CAAC,SAAS,gBAAgB,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QAC9D,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC;QAEvE,IAAI,CAAC;YACH,6DAA6D;YAC7D,MAAM,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;YAEjF,kBAAkB;YAClB,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YAE9D,OAAO,OAAO,CAAC,CAAC,kBAAkB;QACpC,CAAC;gBAAS,CAAC;YACT,oBAAoB;YACpB,MAAM,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,SAAiB,EAAE,IAAY,EAAE,QAAgB,GAAG;QAC5E,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACnD,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QAE9D,OAAO,OAAO,CAAC,CAAC,kBAAkB;IACpC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,SAAiB;QAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAC7C,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAE3C,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5C,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO;YACL,SAAS;YACT,cAAc,EAAE,QAAQ,CAAC,IAAI,CAAC,cAAc,IAAI,GAAG,EAAE,EAAE,CAAC;YACxD,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,UAAU,IAAI,GAAG,EAAE,EAAE,CAAC;YAChD,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,IAAI,GAAG,EAAE,EAAE,CAAC;YAClD,aAAa,EAAE,QAAQ,CAAC,IAAI,CAAC,aAAa,IAAI,GAAG,EAAE,EAAE,CAAC;YACtD,eAAe,EAAE,QAAQ,CAAC,IAAI,CAAC,eAAe,IAAI,GAAG,EAAE,EAAE,CAAC;YAC1D,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;SACnD,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,SAAiB;QAChC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO;QACT,CAAC;QAED,6CAA6C;QAC7C,MAAM,OAAO,GAAG,GAAG,IAAI,CAAC,SAAS,UAAU,SAAS,IAAI,CAAC;QACzD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAE5C,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,MAAM,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,SAAiB,EAAE,QAAgB,IAAI;QACvD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,OAAO,GAAG,GAAG,IAAI,CAAC,SAAS,UAAU,SAAS,SAAS,CAAC;QAC9D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAE5C,wBAAwB;QACxB,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,UAAU,SAAS,QAAQ,CAAC;QAC5D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAE5E,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,SAAiB;QACjC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC7C,OAAO,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,CAAC;IACpD,CAAC;IAED,+CAA+C;IAEvC,YAAY,CAAC,SAAiB,EAAE,IAAY;QAClD,OAAO,GAAG,IAAI,CAAC,SAAS,UAAU,SAAS,SAAS,IAAI,EAAE,CAAC;IAC7D,CAAC;IAEO,WAAW,CAAC,SAAiB,EAAE,KAAsB;QAC3D,OAAO,GAAG,IAAI,CAAC,SAAS,UAAU,SAAS,QAAQ,KAAK,EAAE,CAAC;IAC7D,CAAC;IAEO,YAAY,CAAC,SAAiB;QACpC,OAAO,GAAG,IAAI,CAAC,SAAS,UAAU,SAAS,OAAO,CAAC;IACrD,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,SAAiB,EAAE,KAAiB;QAC9D,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAC7C,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE;YACzB,cAAc,EAAE,KAAK,CAAC,cAAc,CAAC,QAAQ,EAAE;YAC/C,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ,EAAE;YACvC,WAAW,EAAE,KAAK,CAAC,WAAW,CAAC,QAAQ,EAAE;YACzC,aAAa,EAAE,KAAK,CAAC,aAAa,CAAC,QAAQ,EAAE;YAC7C,eAAe,EAAE,KAAK,CAAC,eAAe,CAAC,QAAQ,EAAE;YACjD,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC;SACrC,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,iBAAiB,CAAC,GAAwB,EAAE,SAAiB;QACnE,iBAAiB;QACjB,IAAI,GAAG,CAAC,EAAE,KAAK,SAAS,IAAI,GAAG,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;YAC5C,OAAO,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACxB,CAAC;QAED,+CAA+C;QAC/C,MAAM,YAAY,GAAG,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,wBAAwB;QAClF,IAAI,GAAG,CAAC,YAAY,CAAC,KAAK,SAAS,IAAI,GAAG,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE,CAAC;YAClE,OAAO,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC;QACnC,CAAC;QAED,gCAAgC;QAChC,MAAM,iBAAiB,GAAG,GAAG,SAAS,KAAK,CAAC;QAC5C,IAAI,GAAG,CAAC,iBAAiB,CAAC,KAAK,SAAS,IAAI,GAAG,CAAC,iBAAiB,CAAC,KAAK,IAAI,EAAE,CAAC;YAC5E,OAAO,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC;QACxC,CAAC;QAED,wDAAwD;QACxD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;gBACvE,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;CACF;AAlbD,oDAkbC"}

package/dist/search/search-ranker.d.ts

@@ -0,0 +1,91 @@
+/**
+ * Search Ranker
+ *
+ * Implements ranking algorithms for search results:
+ * - TF-IDF (Term Frequency - Inverse Document Frequency)
+ * - Field boosting (title matches rank higher than description)
+ * - Proximity scoring (words closer together rank higher)
+ * - Freshness scoring (newer documents rank higher - optional)
+ */
+import { ScoringContext, RankingWeights, SearchResult, TokenMetadata } from '../types/search';
+import { Tokenizer } from './tokenizer';
+export declare class SearchRanker {
+    private weights;
+    constructor(weights?: Partial<RankingWeights>);
+    /**
+     * Calculate TF-IDF score for a term in a document
+     *
+     * TF (Term Frequency): How often does the term appear in this document?
+     * IDF (Inverse Document Frequency): How rare is this term across all documents?
+     *
+     * Formula: TF-IDF = TF * IDF
+     * where:
+     *   TF = (term frequency in document) / (total terms in document)
+     *   IDF = log((total documents) / (documents containing term))
+     */
+    calculateTfIdf(context: ScoringContext): number;
+    /**
+     * Calculate field boost score
+     * Fields with higher boost values rank better
+     */
+    calculateFieldBoost(fieldBoost: number): number;
+    /**
+     * Calculate proximity score
+     * Documents where query terms appear close together rank higher
+     */
+    calculateProximityScore(proximityScore?: number): number;
+    /**
+     * Calculate freshness score (optional)
+     * Newer documents rank higher
+     */
+    calculateFreshnessScore(createdAt?: number, decayDays?: number): number;
+    /**
+     * Calculate combined score for a document
+     */
+    calculateScore(context: ScoringContext): number;
+    /**
+     * Rank search results by relevance
+     */
+    rankResults<T>(results: Array<{
+        docId: number;
+        data: T;
+    }>, queryTerms: string[], termStats: Map<string, {
+        docFrequency: number;
+        termFrequenciesByDoc: Map<number, number>;
+    }>, totalDocs: number, fieldBoosts?: Record<string, number>, proximityScores?: Map<number, number>): SearchResult<T>[];
+    /**
+     * Get average field boost value
+     */
+    private getAverageFieldBoost;
+    /**
+     * Calculate BM25 score (more advanced than TF-IDF)
+     *
+     * BM25 is a ranking function used by search engines.
+     * It's an improvement over TF-IDF with better handling of:
+     * - Document length normalization
+     * - Term saturation (diminishing returns for high TF)
+     *
+     * Formula: BM25 = IDF * (TF * (k1 + 1)) / (TF + k1 * (1 - b + b * (docLen / avgDocLen)))
+     * where:
+     *   k1 = term frequency saturation parameter (typical: 1.2-2.0)
+     *   b = length normalization parameter (typical: 0.75)
+     */
+    calculateBM25(termFrequency: number, docFrequency: number, totalDocs: number, docLength: number, avgDocLength: number, k1?: number, b?: number): number;
+    /**
+     * Highlight search terms in text
+     */
+    highlightText(text: string, terms: string[], preTag?: string, postTag?: string, tokenizer?: Tokenizer): string;
+    /**
+     * Generate text snippet with highlighted terms
+     */
+    generateSnippet(text: string, terms: string[], maxLength?: number, preTag?: string, postTag?: string): string;
+    /**
+     * Calculate proximity between multiple terms in a document
+     */
+    calculateMultiTermProximity(tokens: TokenMetadata[], queryTerms: string[]): number;
+}
+/**
+ * Helper function to create ranker with custom weights
+ */
+export declare function createRanker(weights?: Partial<RankingWeights>): SearchRanker;
+//# sourceMappingURL=search-ranker.d.ts.map

package/dist/search/search-ranker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-ranker.d.ts","sourceRoot":"","sources":["../../src/search/search-ranker.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACL,cAAc,EACd,cAAc,EAEd,YAAY,EACZ,aAAa,EACd,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,qBAAa,YAAY;IACvB,OAAO,CAAC,OAAO,CAAiB;gBAEpB,OAAO,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC;IAO7C;;;;;;;;;;OAUG;IACH,cAAc,CAAC,OAAO,EAAE,cAAc,GAAG,MAAM;IAc/C;;;OAGG;IACH,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM;IAI/C;;;OAGG;IACH,uBAAuB,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,MAAM;IAIxD;;;OAGG;IACH,uBAAuB,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,SAAS,GAAE,MAAW,GAAG,MAAM;IAa3E;;OAEG;IACH,cAAc,CAAC,OAAO,EAAE,cAAc,GAAG,MAAM;IAiB/C;;OAEG;IACH,WAAW,CAAC,CAAC,EACX,OAAO,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,CAAC,CAAA;KAAE,CAAC,EAC1C,UAAU,EAAE,MAAM,EAAE,EACpB,SAAS,EAAE,GAAG,CACZ,MAAM,EACN;QACE,YAAY,EAAE,MAAM,CAAC;QACrB,oBAAoB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC3C,CACF,EACD,SAAS,EAAE,MAAM,EACjB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACpC,eAAe,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GACpC,YAAY,CAAC,CAAC,CAAC,EAAE;IA+CpB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAQ5B;;;;;;;;;;;;OAYG;IACH,aAAa,CACX,aAAa,EAAE,MAAM,EACrB,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,EACpB,EAAE,GAAE,MAAY,EAChB,CAAC,GAAE,MAAa,GACf,MAAM;IAYT;;OAEG;IACH,aAAa,CACX,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EAAE,EACf,MAAM,GAAE,MAAiB,EACzB,OAAO,GAAE,MAAkB,EAC3B,SAAS,CAAC,EAAE,SAAS,GACpB,MAAM;IAkBT;;OAEG;IACH,eAAe,CACb,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EAAE,EACf,SAAS,GAAE,MAAY,EACvB,MAAM,GAAE,MAAiB,EACzB,OAAO,GAAE,MAAkB,GAC1B,MAAM;IAiDT;;OAEG;IACH,2BAA2B,CAAC,MAAM,EAAE,aAAa,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,MAAM;CAqDnF;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG,YAAY,CAE5E"}