@pleaseai/context-please-core 0.1.0

package/dist/vectordb/sparse/simple-bm25.d.ts

@@ -0,0 +1,104 @@
+import { SparseVectorGenerator } from './sparse-vector-generator';
+import { SparseVector, SparseVectorConfig } from './types';
+/**
+ * Configuration for BM25 algorithm
+ */
+export interface BM25Config {
+    /**
+     * Term frequency saturation parameter
+     * Controls how quickly term frequency impact saturates
+     * @default 1.2
+     */
+    k1?: number;
+    /**
+     * Length normalization parameter
+     * 0 = no normalization, 1 = full normalization
+     * @default 0.75
+     */
+    b?: number;
+    /**
+     * Minimum term length to include
+     * @default 2
+     */
+    minTermLength?: number;
+    /**
+     * Stop words to exclude
+     * @default []
+     */
+    stopWords?: Set<string>;
+}
+/**
+ * Simple BM25 implementation for sparse vector generation
+ *
+ * BM25 (Best Matching 25) is a probabilistic ranking function
+ * used for information retrieval and text search.
+ *
+ * Formula:
+ * score(D,Q) = Σ IDF(qi) × (f(qi,D) × (k1 + 1)) / (f(qi,D) + k1 × (1 - b + b × |D| / avgdl))
+ *
+ * Where:
+ * - D = document
+ * - Q = query
+ * - qi = query term i
+ * - f(qi,D) = term frequency in document
+ * - |D| = document length
+ * - avgdl = average document length
+ * - IDF(qi) = inverse document frequency
+ */
+export declare class SimpleBM25 implements SparseVectorGenerator {
+    private k1;
+    private b;
+    private minTermLength;
+    private stopWords;
+    private vocabulary;
+    private idf;
+    private avgDocLength;
+    private trained;
+    constructor(config?: BM25Config);
+    /**
+     * Tokenize text into terms
+     * Simple but effective tokenization:
+     * 1. Lowercase
+     * 2. Remove special characters
+     * 3. Split on whitespace
+     * 4. Filter by length and stop words
+     */
+    private tokenize;
+    /**
+     * Calculate term frequency for a document
+     */
+    private calculateTermFrequency;
+    /**
+     * Learn vocabulary and IDF from corpus
+     */
+    learn(documents: string[]): void;
+    /**
+     * Generate sparse vector for a single text
+     */
+    generate(text: string, config?: SparseVectorConfig): SparseVector;
+    /**
+     * Generate sparse vectors for multiple texts (batch operation)
+     */
+    generateBatch(texts: string[], config?: SparseVectorConfig): SparseVector[];
+    /**
+     * Get vocabulary size
+     */
+    getVocabularySize(): number;
+    /**
+     * Get average document length
+     */
+    getAverageDocumentLength(): number;
+    /**
+     * Check if trained
+     */
+    isTrained(): boolean;
+    /**
+     * Get the vocabulary (for debugging/inspection)
+     */
+    getVocabulary(): Map<string, number>;
+    /**
+     * Get IDF scores (for debugging/inspection)
+     */
+    getIDFScores(): Map<string, number>;
+}
+//# sourceMappingURL=simple-bm25.d.ts.map

package/dist/vectordb/sparse/simple-bm25.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"simple-bm25.d.ts","sourceRoot":"","sources":["../../../src/vectordb/sparse/simple-bm25.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAE3D;;GAEG;AACH,MAAM,WAAW,UAAU;IACvB;;;;OAIG;IACH,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ;;;;OAIG;IACH,CAAC,CAAC,EAAE,MAAM,CAAC;IAEX;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,UAAW,YAAW,qBAAqB;IACpD,OAAO,CAAC,EAAE,CAAS;IACnB,OAAO,CAAC,CAAC,CAAS;IAClB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,SAAS,CAAc;IAG/B,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,GAAG,CAAkC;IAC7C,OAAO,CAAC,YAAY,CAAa;IACjC,OAAO,CAAC,OAAO,CAAkB;gBAErB,MAAM,GAAE,UAAe;IAOnC;;;;;;;OAOG;IACH,OAAO,CAAC,QAAQ;IAWhB;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAQ9B;;OAEG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,IAAI;IA8ChC;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,kBAAkB,GAAG,YAAY;IA+DjE;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,MAAM,CAAC,EAAE,kBAAkB,GAAG,YAAY,EAAE;IAI3E;;OAEG;IACH,iBAAiB,IAAI,MAAM;IAI3B;;OAEG;IACH,wBAAwB,IAAI,MAAM;IAIlC;;OAEG;IACH,SAAS,IAAI,OAAO;IAIpB;;OAEG;IACH,aAAa,IAAI,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAIpC;;OAEG;IACH,YAAY,IAAI,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;CAGtC"}

package/dist/vectordb/sparse/simple-bm25.js

@@ -0,0 +1,189 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SimpleBM25 = void 0;
+/**
+ * Simple BM25 implementation for sparse vector generation
+ *
+ * BM25 (Best Matching 25) is a probabilistic ranking function
+ * used for information retrieval and text search.
+ *
+ * Formula:
+ * score(D,Q) = Σ IDF(qi) × (f(qi,D) × (k1 + 1)) / (f(qi,D) + k1 × (1 - b + b × |D| / avgdl))
+ *
+ * Where:
+ * - D = document
+ * - Q = query
+ * - qi = query term i
+ * - f(qi,D) = term frequency in document
+ * - |D| = document length
+ * - avgdl = average document length
+ * - IDF(qi) = inverse document frequency
+ */
+class SimpleBM25 {
+    constructor(config = {}) {
+        // Learned from corpus
+        this.vocabulary = new Map();
+        this.idf = new Map();
+        this.avgDocLength = 0;
+        this.trained = false;
+        this.k1 = config.k1 ?? 1.2;
+        this.b = config.b ?? 0.75;
+        this.minTermLength = config.minTermLength ?? 2;
+        this.stopWords = config.stopWords ?? new Set();
+    }
+    /**
+     * Tokenize text into terms
+     * Simple but effective tokenization:
+     * 1. Lowercase
+     * 2. Remove special characters
+     * 3. Split on whitespace
+     * 4. Filter by length and stop words
+     */
+    tokenize(text) {
+        return text
+            .toLowerCase()
+            .replace(/[^\w\s]/g, ' ')
+            .split(/\s+/)
+            .filter(term => term.length >= this.minTermLength &&
+            !this.stopWords.has(term));
+    }
+    /**
+     * Calculate term frequency for a document
+     */
+    calculateTermFrequency(tokens) {
+        const termFreq = new Map();
+        tokens.forEach(token => {
+            termFreq.set(token, (termFreq.get(token) || 0) + 1);
+        });
+        return termFreq;
+    }
+    /**
+     * Learn vocabulary and IDF from corpus
+     */
+    learn(documents) {
+        if (documents.length === 0) {
+            throw new Error('Cannot learn from empty corpus');
+        }
+        // Reset state
+        this.vocabulary.clear();
+        this.idf.clear();
+        // Tokenize all documents
+        const tokenizedDocs = documents.map(doc => this.tokenize(doc));
+        // Calculate average document length
+        const totalLength = tokenizedDocs.reduce((sum, tokens) => sum + tokens.length, 0);
+        this.avgDocLength = totalLength / tokenizedDocs.length;
+        // Calculate document frequency for each term
+        const docFreq = new Map();
+        tokenizedDocs.forEach(tokens => {
+            const uniqueTerms = new Set(tokens);
+            uniqueTerms.forEach(term => {
+                docFreq.set(term, (docFreq.get(term) || 0) + 1);
+            });
+        });
+        // Build vocabulary and calculate IDF
+        // IDF formula: log((N - df + 0.5) / (df + 0.5))
+        const numDocs = documents.length;
+        let vocabIndex = 0;
+        docFreq.forEach((df, term) => {
+            // Add to vocabulary
+            this.vocabulary.set(term, vocabIndex++);
+            // Calculate IDF
+            const idf = Math.log((numDocs - df + 0.5) / (df + 0.5));
+            this.idf.set(term, idf);
+        });
+        this.trained = true;
+        console.log(`[SimpleBM25] Learned from ${numDocs} documents`);
+        console.log(`[SimpleBM25] Vocabulary size: ${this.vocabulary.size}`);
+        console.log(`[SimpleBM25] Average document length: ${this.avgDocLength.toFixed(2)}`);
+    }
+    /**
+     * Generate sparse vector for a single text
+     */
+    generate(text, config) {
+        if (!this.trained) {
+            throw new Error('BM25 generator must be trained before generating vectors. Call learn() first.');
+        }
+        const tokens = this.tokenize(text);
+        const termFreq = this.calculateTermFrequency(tokens);
+        const docLength = tokens.length;
+        const indices = [];
+        const values = [];
+        // Calculate BM25 score for each term
+        termFreq.forEach((tf, term) => {
+            const vocabIndex = this.vocabulary.get(term);
+            const idfScore = this.idf.get(term);
+            // Skip terms not in vocabulary (unknown terms)
+            if (vocabIndex === undefined || idfScore === undefined) {
+                return;
+            }
+            // BM25 formula
+            const normalizedTF = (tf * (this.k1 + 1)) /
+                (tf + this.k1 * (1 - this.b + (this.b * docLength) / this.avgDocLength));
+            const score = idfScore * normalizedTF;
+            // Apply minimum score threshold if configured
+            if (config?.minScore !== undefined && score < config.minScore) {
+                return;
+            }
+            indices.push(vocabIndex);
+            values.push(score);
+        });
+        // Sort by score descending and apply maxTerms limit
+        if (config?.maxTerms !== undefined && indices.length > config.maxTerms) {
+            const combined = indices.map((idx, i) => ({ idx, val: values[i] }));
+            combined.sort((a, b) => b.val - a.val);
+            combined.splice(config.maxTerms);
+            indices.length = 0;
+            values.length = 0;
+            combined.forEach(({ idx, val }) => {
+                indices.push(idx);
+                values.push(val);
+            });
+        }
+        // Normalize if requested
+        if (config?.normalize && values.length > 0) {
+            const norm = Math.sqrt(values.reduce((sum, val) => sum + val * val, 0));
+            for (let i = 0; i < values.length; i++) {
+                values[i] /= norm;
+            }
+        }
+        return { indices, values };
+    }
+    /**
+     * Generate sparse vectors for multiple texts (batch operation)
+     */
+    generateBatch(texts, config) {
+        return texts.map(text => this.generate(text, config));
+    }
+    /**
+     * Get vocabulary size
+     */
+    getVocabularySize() {
+        return this.vocabulary.size;
+    }
+    /**
+     * Get average document length
+     */
+    getAverageDocumentLength() {
+        return this.avgDocLength;
+    }
+    /**
+     * Check if trained
+     */
+    isTrained() {
+        return this.trained;
+    }
+    /**
+     * Get the vocabulary (for debugging/inspection)
+     */
+    getVocabulary() {
+        return new Map(this.vocabulary);
+    }
+    /**
+     * Get IDF scores (for debugging/inspection)
+     */
+    getIDFScores() {
+        return new Map(this.idf);
+    }
+}
+exports.SimpleBM25 = SimpleBM25;
+//# sourceMappingURL=simple-bm25.js.map

package/dist/vectordb/sparse/simple-bm25.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"simple-bm25.js","sourceRoot":"","sources":["../../../src/vectordb/sparse/simple-bm25.ts"],"names":[],"mappings":";;;AAkCA;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAa,UAAU;IAYnB,YAAY,SAAqB,EAAE;QANnC,sBAAsB;QACd,eAAU,GAAwB,IAAI,GAAG,EAAE,CAAC;QAC5C,QAAG,GAAwB,IAAI,GAAG,EAAE,CAAC;QACrC,iBAAY,GAAW,CAAC,CAAC;QACzB,YAAO,GAAY,KAAK,CAAC;QAG7B,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,EAAE,IAAI,GAAG,CAAC;QAC3B,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,IAAI,CAAC;QAC1B,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC;QAC/C,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,IAAI,GAAG,EAAE,CAAC;IACnD,CAAC;IAED;;;;;;;OAOG;IACK,QAAQ,CAAC,IAAY;QACzB,OAAO,IAAI;aACN,WAAW,EAAE;aACb,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;aACxB,KAAK,CAAC,KAAK,CAAC;aACZ,MAAM,CAAC,IAAI,CAAC,EAAE,CACX,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,aAAa;YACjC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAC5B,CAAC;IACV,CAAC;IAED;;OAEG;IACK,sBAAsB,CAAC,MAAgB;QAC3C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC3C,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;YACnB,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAmB;QACrB,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACtD,CAAC;QAED,cAAc;QACd,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACxB,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QAEjB,yBAAyB;QACzB,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;QAE/D,oCAAoC;QACpC,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAClF,IAAI,CAAC,YAAY,GAAG,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC;QAEvD,6CAA6C;QAC7C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC1C,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;YAC3B,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;YACpC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;gBACvB,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,qCAAqC;QACrC,gDAAgD;QAChD,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC;QACjC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE;YACzB,oBAAoB;YACpB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;YAExC,gBAAgB;YAChB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;YACxD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QAEpB,OAAO,CAAC,GAAG,CAAC,6BAA6B,OAAO,YAAY,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,iCAAiC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;QACrE,OAAO,CAAC,GAAG,CAAC,yCAAyC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACzF,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,IAAY,EAAE,MAA2B;QAC9C,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,+EAA+E,CAAC,CAAC;QACrG,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC;QAEhC,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,qCAAqC;QACrC,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE;YAC1B,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAEpC,+CAA+C;YAC/C,IAAI,UAAU,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBACrD,OAAO;YACX,CAAC;YAED,eAAe;YACf,MAAM,YAAY,GACd,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;gBACpB,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;YAE7E,MAAM,KAAK,GAAG,QAAQ,GAAG,YAAY,CAAC;YAEtC,8CAA8C;YAC9C,IAAI,MAAM,EAAE,QAAQ,KAAK,SAAS,IAAI,KAAK,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;gBAC5D,OAAO;YACX,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,oDAAoD;QACpD,IAAI,MAAM,EAAE,QAAQ,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;YACrE,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACpE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YACvC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAEjC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;YACnB,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;YAClB,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE;gBAC9B,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrB,CAAC,CAAC,CAAC;QACP,CAAC;QAED,yBAAyB;QACzB,IAAI,MAAM,EAAE,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;YACxE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;YACtB,CAAC;QACL,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,KAAe,EAAE,MAA2B;QACtD,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACH,iBAAiB;QACb,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,wBAAwB;QACpB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,aAAa;QACT,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,YAAY;QACR,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;CACJ;AA7MD,gCA6MC"}

package/dist/vectordb/sparse/sparse-vector-generator.d.ts

@@ -0,0 +1,54 @@
+import { SparseVector, SparseVectorConfig } from './types';
+/**
+ * Abstract interface for sparse vector generation
+ *
+ * Sparse vectors are used for:
+ * - BM25 keyword-based search
+ * - Hybrid search (combining dense + sparse)
+ * - Text-based retrieval with term importance
+ *
+ * Implementations should:
+ * 1. Learn vocabulary from a corpus
+ * 2. Generate sparse vectors with indices (vocabulary positions) and values (scores)
+ * 3. Support both single document and batch processing
+ */
+export interface SparseVectorGenerator {
+    /**
+     * Learn vocabulary and statistics from a corpus of documents
+     * This should be called during indexing to build IDF statistics
+     *
+     * @param documents - Array of document texts to learn from
+     */
+    learn(documents: string[]): void;
+    /**
+     * Generate sparse vector for a single text
+     *
+     * @param text - Input text to vectorize
+     * @param config - Optional configuration for vector generation
+     * @returns Sparse vector with indices and values
+     */
+    generate(text: string, config?: SparseVectorConfig): SparseVector;
+    /**
+     * Generate sparse vectors for multiple texts (batch operation)
+     *
+     * @param texts - Array of input texts to vectorize
+     * @param config - Optional configuration for vector generation
+     * @returns Array of sparse vectors
+     */
+    generateBatch(texts: string[], config?: SparseVectorConfig): SparseVector[];
+    /**
+     * Get the vocabulary size (number of unique terms)
+     * Useful for debugging and understanding the sparse vector space
+     */
+    getVocabularySize(): number;
+    /**
+     * Get average document length from the learned corpus
+     * Used for BM25 length normalization
+     */
+    getAverageDocumentLength(): number;
+    /**
+     * Check if the generator has been trained (learned from corpus)
+     */
+    isTrained(): boolean;
+}
+//# sourceMappingURL=sparse-vector-generator.d.ts.map

package/dist/vectordb/sparse/sparse-vector-generator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sparse-vector-generator.d.ts","sourceRoot":"","sources":["../../../src/vectordb/sparse/sparse-vector-generator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAE3D;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,qBAAqB;IAClC;;;;;OAKG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAEjC;;;;;;OAMG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,kBAAkB,GAAG,YAAY,CAAC;IAElE;;;;;;OAMG;IACH,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,MAAM,CAAC,EAAE,kBAAkB,GAAG,YAAY,EAAE,CAAC;IAE5E;;;OAGG;IACH,iBAAiB,IAAI,MAAM,CAAC;IAE5B;;;OAGG;IACH,wBAAwB,IAAI,MAAM,CAAC;IAEnC;;OAEG;IACH,SAAS,IAAI,OAAO,CAAC;CACxB"}

package/dist/vectordb/sparse/sparse-vector-generator.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=sparse-vector-generator.js.map

package/dist/vectordb/sparse/sparse-vector-generator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sparse-vector-generator.js","sourceRoot":"","sources":["../../../src/vectordb/sparse/sparse-vector-generator.ts"],"names":[],"mappings":""}

package/dist/vectordb/sparse/types.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Sparse vector representation
+ * Used for BM25 and other sparse encoding methods
+ */
+export interface SparseVector {
+    /**
+     * Indices of non-zero elements
+     * Each index maps to a position in the vocabulary
+     */
+    indices: number[];
+    /**
+     * Values corresponding to the indices
+     * Typically BM25 scores or TF-IDF weights
+     */
+    values: number[];
+}
+/**
+ * Configuration for sparse vector generation
+ */
+export interface SparseVectorConfig {
+    /**
+     * Minimum score threshold for including a term
+     * Terms with scores below this will be filtered out
+     * @default 0
+     */
+    minScore?: number;
+    /**
+     * Maximum number of terms to include in sparse vector
+     * @default undefined (no limit)
+     */
+    maxTerms?: number;
+    /**
+     * Whether to normalize vector values
+     * @default false
+     */
+    normalize?: boolean;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/vectordb/sparse/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/vectordb/sparse/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,WAAW,YAAY;IACzB;;;OAGG;IACH,OAAO,EAAE,MAAM,EAAE,CAAC;IAElB;;;OAGG;IACH,MAAM,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB"}

package/dist/vectordb/sparse/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/vectordb/sparse/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/vectordb/sparse/types.ts"],"names":[],"mappings":""}

package/dist/vectordb/types.d.ts

@@ -0,0 +1,120 @@
+export interface VectorDocument {
+    id: string;
+    vector: number[];
+    content: string;
+    relativePath: string;
+    startLine: number;
+    endLine: number;
+    fileExtension: string;
+    metadata: Record<string, any>;
+}
+export interface SearchOptions {
+    topK?: number;
+    filter?: Record<string, any>;
+    threshold?: number;
+    filterExpr?: string;
+}
+export interface HybridSearchRequest {
+    data: number[] | string;
+    anns_field: string;
+    param: Record<string, any>;
+    limit: number;
+}
+export interface HybridSearchOptions {
+    rerank?: RerankStrategy;
+    limit?: number;
+    filterExpr?: string;
+}
+export interface RerankStrategy {
+    strategy: 'rrf' | 'weighted';
+    params?: Record<string, any>;
+}
+export interface VectorSearchResult {
+    document: VectorDocument;
+    score: number;
+}
+export interface HybridSearchResult {
+    document: VectorDocument;
+    score: number;
+}
+export interface VectorDatabase {
+    /**
+     * Create collection
+     * @param collectionName Collection name
+     * @param dimension Vector dimension
+     * @param description Collection description
+     */
+    createCollection(collectionName: string, dimension: number, description?: string): Promise<void>;
+    /**
+     * Create collection with hybrid search support
+     * @param collectionName Collection name
+     * @param dimension Dense vector dimension
+     * @param description Collection description
+     */
+    createHybridCollection(collectionName: string, dimension: number, description?: string): Promise<void>;
+    /**
+     * Drop collection
+     * @param collectionName Collection name
+     */
+    dropCollection(collectionName: string): Promise<void>;
+    /**
+     * Check if collection exists
+     * @param collectionName Collection name
+     */
+    hasCollection(collectionName: string): Promise<boolean>;
+    /**
+     * List all collections
+     */
+    listCollections(): Promise<string[]>;
+    /**
+     * Insert vector documents
+     * @param collectionName Collection name
+     * @param documents Document array
+     */
+    insert(collectionName: string, documents: VectorDocument[]): Promise<void>;
+    /**
+     * Insert hybrid vector documents
+     * @param collectionName Collection name
+     * @param documents Document array
+     */
+    insertHybrid(collectionName: string, documents: VectorDocument[]): Promise<void>;
+    /**
+     * Search similar vectors
+     * @param collectionName Collection name
+     * @param queryVector Query vector
+     * @param options Search options
+     */
+    search(collectionName: string, queryVector: number[], options?: SearchOptions): Promise<VectorSearchResult[]>;
+    /**
+     * Hybrid search with multiple vector fields
+     * @param collectionName Collection name
+     * @param searchRequests Array of search requests for different fields
+     * @param options Hybrid search options including reranking
+     */
+    hybridSearch(collectionName: string, searchRequests: HybridSearchRequest[], options?: HybridSearchOptions): Promise<HybridSearchResult[]>;
+    /**
+     * Delete documents
+     * @param collectionName Collection name
+     * @param ids Document ID array
+     */
+    delete(collectionName: string, ids: string[]): Promise<void>;
+    /**
+     * Query documents with filter conditions
+     * @param collectionName Collection name
+     * @param filter Filter expression
+     * @param outputFields Fields to return
+     * @param limit Maximum number of results
+     */
+    query(collectionName: string, filter: string, outputFields: string[], limit?: number): Promise<Record<string, any>[]>;
+    /**
+     * Check collection limit
+     * Returns true if collection can be created, false if limit exceeded
+     */
+    checkCollectionLimit(): Promise<boolean>;
+}
+/**
+ * Special error message for collection limit exceeded
+ * This allows us to distinguish it from other errors across all Milvus implementations
+ */
+export declare const COLLECTION_LIMIT_MESSAGE = "[Error]: Your Zilliz Cloud account has hit its collection limit. To continue creating collections, you'll need to expand your capacity. We recommend visiting https://zilliz.com/pricing to explore options for dedicated or serverless clusters.";
+//# sourceMappingURL=types.d.ts.map

package/dist/vectordb/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/vectordb/types.ts"],"names":[],"mappings":"AACA,MAAM,WAAW,cAAc;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,aAAa;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB;AAGD,MAAM,WAAW,mBAAmB;IAChC,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,mBAAmB;IAChC,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,cAAc;IAC3B,QAAQ,EAAE,KAAK,GAAG,UAAU,CAAC;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,kBAAkB;IAC/B,QAAQ,EAAE,cAAc,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,kBAAkB;IAC/B,QAAQ,EAAE,cAAc,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC3B;;;;;OAKG;IACH,gBAAgB,CAAC,cAAc,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjG;;;;;OAKG;IACH,sBAAsB,CAAC,cAAc,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvG;;;OAGG;IACH,cAAc,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtD;;;OAGG;IACH,aAAa,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAExD;;OAEG;IACH,eAAe,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAErC;;;;OAIG;IACH,MAAM,CAAC,cAAc,EAAE,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE3E;;;;OAIG;IACH,YAAY,CAAC,cAAc,EAAE,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjF;;;;;OAKG;IACH,MAAM,CAAC,cAAc,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAE9G;;;;;OAKG;IACH,YAAY,CAAC,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,mBAAmB,EAAE,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAE1I;;;;OAIG;IACH,MAAM,CAAC,cAAc,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE7D;;;;;;OAMG;IACH,KAAK,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAEtH;;;OAGG;IACH,oBAAoB,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;CAC5C;AAED;;;GAGG;AACH,eAAO,MAAM,wBAAwB,sPAAsP,CAAC"}

package/dist/vectordb/types.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.COLLECTION_LIMIT_MESSAGE = void 0;
+/**
+ * Special error message for collection limit exceeded
+ * This allows us to distinguish it from other errors across all Milvus implementations
+ */
+exports.COLLECTION_LIMIT_MESSAGE = "[Error]: Your Zilliz Cloud account has hit its collection limit. To continue creating collections, you'll need to expand your capacity. We recommend visiting https://zilliz.com/pricing to explore options for dedicated or serverless clusters.";
+//# sourceMappingURL=types.js.map

package/dist/vectordb/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/vectordb/types.ts"],"names":[],"mappings":";;;AAuIA;;;GAGG;AACU,QAAA,wBAAwB,GAAG,mPAAmP,CAAC"}