agentdb 1.0.12 → 1.1.1

package/dist/quantization/product-quantization.js

@@ -1,287 +0,0 @@
-"use strict";
-/**
- * Product Quantization (PQ) for 4-32x vector compression
- *
- * Reduces storage requirements while maintaining 95%+ search accuracy.
- * Splits vectors into subvectors and quantizes each independently.
- *
- * @example
- * ```typescript
- * const pq = new ProductQuantizer({ dimensions: 768, subvectors: 8, bits: 8 });
- * await pq.train(trainingVectors); // Train codebooks
- *
- * // Compress vectors (768 floats → 8 bytes = 96x reduction)
- * const codes = pq.encode(vector);
- *
- * // Decompress for similarity search
- * const approximation = pq.decode(codes);
- * ```
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ProductQuantizer = void 0;
-class ProductQuantizer {
-    constructor(config) {
-        this.codebooks = [];
-        this.trained = false;
-        // Statistics
-        this.stats = {
-            encodeCount: 0,
-            decodeCount: 0,
-            totalEncodeTime: 0,
-            totalDecodeTime: 0
-        };
-        this.config = {
-            dimensions: config.dimensions,
-            subvectors: config.subvectors,
-            bits: config.bits,
-            kmeansIterations: config.kmeansIterations ?? 20,
-            enableStats: config.enableStats ?? true
-        };
-        if (config.dimensions % config.subvectors !== 0) {
-            throw new Error(`Dimensions (${config.dimensions}) must be divisible by subvectors (${config.subvectors})`);
-        }
-        this.subvectorDim = config.dimensions / config.subvectors;
-    }
-    /**
-     * Train codebooks using k-means clustering on training vectors
-     */
-    async train(trainingVectors) {
-        if (trainingVectors.length === 0) {
-            throw new Error('Training vectors cannot be empty');
-        }
-        const numCentroids = Math.pow(2, this.config.bits);
-        this.codebooks = [];
-        // Train one codebook per subvector
-        for (let m = 0; m < this.config.subvectors; m++) {
-            const subvectors = this.extractSubvectors(trainingVectors, m);
-            const centroids = this.trainKMeans(subvectors, numCentroids);
-            this.codebooks.push({ centroids });
-        }
-        this.trained = true;
-    }
-    /**
-     * Extract subvectors for a specific segment
-     */
-    extractSubvectors(vectors, segmentIndex) {
-        const start = segmentIndex * this.subvectorDim;
-        const end = start + this.subvectorDim;
-        return vectors.map(v => v.slice(start, end));
-    }
-    /**
-     * Train k-means clustering to create centroids
-     */
-    trainKMeans(vectors, k) {
-        const n = vectors.length;
-        const d = vectors[0].length;
-        // Initialize centroids randomly
-        const centroids = [];
-        const indices = new Set();
-        while (centroids.length < Math.min(k, n)) {
-            const idx = Math.floor(Math.random() * n);
-            if (!indices.has(idx)) {
-                centroids.push([...vectors[idx]]);
-                indices.add(idx);
-            }
-        }
-        // K-means iterations
-        for (let iter = 0; iter < this.config.kmeansIterations; iter++) {
-            // Assignment step
-            const assignments = new Array(n).fill(0);
-            for (let i = 0; i < n; i++) {
-                let minDist = Infinity;
-                let bestCluster = 0;
-                for (let c = 0; c < centroids.length; c++) {
-                    const dist = this.euclideanDistance(vectors[i], centroids[c]);
-                    if (dist < minDist) {
-                        minDist = dist;
-                        bestCluster = c;
-                    }
-                }
-                assignments[i] = bestCluster;
-            }
-            // Update step
-            const clusterSums = Array(centroids.length)
-                .fill(0)
-                .map(() => Array(d).fill(0));
-            const clusterCounts = Array(centroids.length).fill(0);
-            for (let i = 0; i < n; i++) {
-                const cluster = assignments[i];
-                clusterCounts[cluster]++;
-                for (let j = 0; j < d; j++) {
-                    clusterSums[cluster][j] += vectors[i][j];
-                }
-            }
-            // Update centroids
-            for (let c = 0; c < centroids.length; c++) {
-                if (clusterCounts[c] > 0) {
-                    for (let j = 0; j < d; j++) {
-                        centroids[c][j] = clusterSums[c][j] / clusterCounts[c];
-                    }
-                }
-            }
-        }
-        // Fill remaining centroids if k > n
-        while (centroids.length < k) {
-            centroids.push(Array(d).fill(0));
-        }
-        return centroids;
-    }
-    /**
-     * Encode vector into quantized codes
-     */
-    encode(vector) {
-        if (!this.trained) {
-            throw new Error('Quantizer must be trained before encoding');
-        }
-        if (vector.length !== this.config.dimensions) {
-            throw new Error(`Vector dimension (${vector.length}) does not match config (${this.config.dimensions})`);
-        }
-        const startTime = performance.now();
-        const codes = new Uint8Array(this.config.subvectors);
-        for (let m = 0; m < this.config.subvectors; m++) {
-            const start = m * this.subvectorDim;
-            const end = start + this.subvectorDim;
-            const subvector = vector.slice(start, end);
-            // Find nearest centroid
-            let minDist = Infinity;
-            let bestCode = 0;
-            for (let c = 0; c < this.codebooks[m].centroids.length; c++) {
-                const dist = this.euclideanDistance(subvector, this.codebooks[m].centroids[c]);
-                if (dist < minDist) {
-                    minDist = dist;
-                    bestCode = c;
-                }
-            }
-            codes[m] = bestCode;
-        }
-        if (this.config.enableStats) {
-            this.stats.encodeCount++;
-            this.stats.totalEncodeTime += performance.now() - startTime;
-        }
-        return codes;
-    }
-    /**
-     * Decode quantized codes back to approximate vector
-     */
-    decode(codes) {
-        if (!this.trained) {
-            throw new Error('Quantizer must be trained before decoding');
-        }
-        if (codes.length !== this.config.subvectors) {
-            throw new Error(`Code length (${codes.length}) does not match subvectors (${this.config.subvectors})`);
-        }
-        const startTime = performance.now();
-        const vector = [];
-        for (let m = 0; m < this.config.subvectors; m++) {
-            const code = codes[m];
-            const centroid = this.codebooks[m].centroids[code];
-            vector.push(...centroid);
-        }
-        if (this.config.enableStats) {
-            this.stats.decodeCount++;
-            this.stats.totalDecodeTime += performance.now() - startTime;
-        }
-        return vector;
-    }
-    /**
-     * Compute asymmetric distance for faster search
-     * Compare query vector directly with quantized database vectors
-     */
-    asymmetricDistance(query, codes) {
-        if (!this.trained) {
-            throw new Error('Quantizer must be trained before computing distances');
-        }
-        let distance = 0;
-        for (let m = 0; m < this.config.subvectors; m++) {
-            const start = m * this.subvectorDim;
-            const end = start + this.subvectorDim;
-            const querySubvector = query.slice(start, end);
-            const code = codes[m];
-            const centroid = this.codebooks[m].centroids[code];
-            distance += this.euclideanDistanceSquared(querySubvector, centroid);
-        }
-        return Math.sqrt(distance);
-    }
-    /**
-     * Euclidean distance between two vectors
-     */
-    euclideanDistance(a, b) {
-        return Math.sqrt(this.euclideanDistanceSquared(a, b));
-    }
-    /**
-     * Squared Euclidean distance (faster, no sqrt)
-     */
-    euclideanDistanceSquared(a, b) {
-        let sum = 0;
-        for (let i = 0; i < a.length; i++) {
-            const diff = a[i] - b[i];
-            sum += diff * diff;
-        }
-        return sum;
-    }
-    /**
-     * Get compression statistics
-     */
-    getStats() {
-        const originalSize = this.config.dimensions * 4; // 4 bytes per float32
-        const compressedSize = this.config.subvectors; // 1 byte per subvector
-        return {
-            originalSize,
-            compressedSize,
-            compressionRatio: originalSize / compressedSize,
-            encodeCount: this.stats.encodeCount,
-            decodeCount: this.stats.decodeCount,
-            avgEncodeTime: this.stats.encodeCount > 0
-                ? this.stats.totalEncodeTime / this.stats.encodeCount
-                : 0,
-            avgDecodeTime: this.stats.decodeCount > 0
-                ? this.stats.totalDecodeTime / this.stats.decodeCount
-                : 0
-        };
-    }
-    /**
-     * Reset statistics
-     */
-    resetStats() {
-        this.stats = {
-            encodeCount: 0,
-            decodeCount: 0,
-            totalEncodeTime: 0,
-            totalDecodeTime: 0
-        };
-    }
-    /**
-     * Check if quantizer is trained
-     */
-    isTrained() {
-        return this.trained;
-    }
-    /**
-     * Export codebooks for persistence
-     */
-    exportCodebooks() {
-        return this.codebooks.map(cb => ({
-            centroids: cb.centroids.map(c => [...c])
-        }));
-    }
-    /**
-     * Import codebooks from persistence
-     */
-    importCodebooks(codebooks) {
-        if (codebooks.length !== this.config.subvectors) {
-            throw new Error(`Codebook count (${codebooks.length}) does not match subvectors (${this.config.subvectors})`);
-        }
-        this.codebooks = codebooks.map(cb => ({
-            centroids: cb.centroids.map(c => [...c])
-        }));
-        this.trained = true;
-    }
-    /**
-     * Get configuration
-     */
-    getConfig() {
-        return { ...this.config };
-    }
-}
-exports.ProductQuantizer = ProductQuantizer;
-//# sourceMappingURL=product-quantization.js.map

package/dist/quantization/product-quantization.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"product-quantization.js","sourceRoot":"","sources":["../../src/quantization/product-quantization.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;GAiBG;;;AA6BH,MAAa,gBAAgB;IAc3B,YAAY,MAA8B;QAZlC,cAAS,GAAe,EAAE,CAAC;QAE3B,YAAO,GAAY,KAAK,CAAC;QAEjC,aAAa;QACL,UAAK,GAAG;YACd,WAAW,EAAE,CAAC;YACd,WAAW,EAAE,CAAC;YACd,eAAe,EAAE,CAAC;YAClB,eAAe,EAAE,CAAC;SACnB,CAAC;QAGA,IAAI,CAAC,MAAM,GAAG;YACZ,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,EAAE;YAC/C,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI;SACxC,CAAC;QAEF,IAAI,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CACb,eAAe,MAAM,CAAC,UAAU,sCAAsC,MAAM,CAAC,UAAU,GAAG,CAC3F,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;IAC5D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CAAC,eAA2B;QACrC,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACtD,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACnD,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QAEpB,mCAAmC;QACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;YAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;YAC7D,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,OAAmB,EAAE,YAAoB;QACjE,MAAM,KAAK,GAAG,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QAC/C,MAAM,GAAG,GAAG,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC;QACtC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,OAAmB,EAAE,CAAS;QAChD,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;QACzB,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAE5B,gCAAgC;QAChC,MAAM,SAAS,GAAe,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,OAAO,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YACzC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;YAC1C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBACtB,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAClC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,IAAI,EAAE,EAAE,CAAC;YAC/D,kBAAkB;YAClB,MAAM,WAAW,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3B,IAAI,OAAO,GAAG,QAAQ,CAAC;gBACvB,IAAI,WAAW,GAAG,CAAC,CAAC;gBAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC9D,IAAI,IAAI,GAAG,OAAO,EAAE,CAAC;wBACnB,OAAO,GAAG,IAAI,CAAC;wBACf,WAAW,GAAG,CAAC,CAAC;oBAClB,CAAC;gBACH,CAAC;gBAED,WAAW,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC;YAC/B,CAAC;YAED,cAAc;YACd,MAAM,WAAW,GAAe,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC;iBACpD,IAAI,CAAC,CAAC,CAAC;iBACP,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,aAAa,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAEtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3B,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;gBAC/B,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;gBACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC3B,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,CAAC;YACH,CAAC;YAED,mBAAmB;YACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,IAAI,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;wBAC3B,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;oBACzD,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,oCAAoC;QACpC,OAAO,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,MAAgB;QACrB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;YAC7C,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,CAAC,MAAM,4BAA4B,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,CACxF,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAErD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC;YACpC,MAAM,GAAG,GAAG,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC;YACtC,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAE3C,wBAAwB;YACxB,IAAI,OAAO,GAAG,QAAQ,CAAC;YACvB,IAAI,QAAQ,GAAG,CAAC,CAAC;YAEjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/E,IAAI,IAAI,GAAG,OAAO,EAAE,CAAC;oBACnB,OAAO,GAAG,IAAI,CAAC;oBACf,QAAQ,GAAG,CAAC,CAAC;gBACf,CAAC;YACH,CAAC;YAED,KAAK,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC;QACtB,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAC5B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;YACzB,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAC9D,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAiB;QACtB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CACb,gBAAgB,KAAK,CAAC,MAAM,gCAAgC,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,CACtF,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACnD,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;QAC3B,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAC5B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;YACzB,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAC9D,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,KAAe,EAAE,KAAiB;QACnD,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAC1E,CAAC;QAED,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC;YACpC,MAAM,GAAG,GAAG,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC;YACtC,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAE/C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAEnD,QAAQ,IAAI,IAAI,CAAC,wBAAwB,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;QACtE,CAAC;QAED,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC7B,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,CAAW,EAAE,CAAW;QAChD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACxD,CAAC;IAED;;OAEG;IACK,wBAAwB,CAAC,CAAW,EAAE,CAAW;QACvD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACzB,GAAG,IAAI,IAAI,GAAG,IAAI,CAAC;QACrB,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,sBAAsB;QACvE,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,uBAAuB;QAEtE,OAAO;YACL,YAAY;YACZ,cAAc;YACd,gBAAgB,EAAE,YAAY,GAAG,cAAc;YAC/C,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW;YACnC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW;YACnC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,CAAC;gBACvC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW;gBACrD,CAAC,CAAC,CAAC;YACL,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,CAAC;gBACvC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW;gBACrD,CAAC,CAAC,CAAC;SACN,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,UAAU;QACR,IAAI,CAAC,KAAK,GAAG;YACX,WAAW,EAAE,CAAC;YACd,WAAW,EAAE,CAAC;YACd,eAAe,EAAE,CAAC;YAClB,eAAe,EAAE,CAAC;SACnB,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED;;OAEG;IACH,eAAe;QACb,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAC/B,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;SACzC,CAAC,CAAC,CAAC;IACN,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,SAAqB;QACnC,IAAI,SAAS,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CACb,mBAAmB,SAAS,CAAC,MAAM,gCAAgC,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,CAC7F,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YACpC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;SACzC,CAAC,CAAC,CAAC;QACJ,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC5B,CAAC;CACF;AAjUD,4CAiUC"}

package/dist/quantization/product-quantization.mjs

@@ -1,282 +0,0 @@
-/**
- * Product Quantization (PQ) for 4-32x vector compression
- *
- * Reduces storage requirements while maintaining 95%+ search accuracy.
- * Splits vectors into subvectors and quantizes each independently.
- *
- * @example
- * ```typescript
- * const pq = new ProductQuantizer({ dimensions: 768, subvectors: 8, bits: 8 });
- * await pq.train(trainingVectors); // Train codebooks
- *
- * // Compress vectors (768 floats → 8 bytes = 96x reduction)
- * const codes = pq.encode(vector);
- *
- * // Decompress for similarity search
- * const approximation = pq.decode(codes);
- * ```
- */
-export class ProductQuantizer {
-    constructor(config) {
-        this.codebooks = [];
-        this.trained = false;
-        // Statistics
-        this.stats = {
-            encodeCount: 0,
-            decodeCount: 0,
-            totalEncodeTime: 0,
-            totalDecodeTime: 0
-        };
-        this.config = {
-            dimensions: config.dimensions,
-            subvectors: config.subvectors,
-            bits: config.bits,
-            kmeansIterations: config.kmeansIterations ?? 20,
-            enableStats: config.enableStats ?? true
-        };
-        if (config.dimensions % config.subvectors !== 0) {
-            throw new Error(`Dimensions (${config.dimensions}) must be divisible by subvectors (${config.subvectors})`);
-        }
-        this.subvectorDim = config.dimensions / config.subvectors;
-    }
-    /**
-     * Train codebooks using k-means clustering on training vectors
-     */
-    async train(trainingVectors) {
-        if (trainingVectors.length === 0) {
-            throw new Error('Training vectors cannot be empty');
-        }
-        const numCentroids = Math.pow(2, this.config.bits);
-        this.codebooks = [];
-        // Train one codebook per subvector
-        for (let m = 0; m < this.config.subvectors; m++) {
-            const subvectors = this.extractSubvectors(trainingVectors, m);
-            const centroids = this.trainKMeans(subvectors, numCentroids);
-            this.codebooks.push({ centroids });
-        }
-        this.trained = true;
-    }
-    /**
-     * Extract subvectors for a specific segment
-     */
-    extractSubvectors(vectors, segmentIndex) {
-        const start = segmentIndex * this.subvectorDim;
-        const end = start + this.subvectorDim;
-        return vectors.map(v => v.slice(start, end));
-    }
-    /**
-     * Train k-means clustering to create centroids
-     */
-    trainKMeans(vectors, k) {
-        const n = vectors.length;
-        const d = vectors[0].length;
-        // Initialize centroids randomly
-        const centroids = [];
-        const indices = new Set();
-        while (centroids.length < Math.min(k, n)) {
-            const idx = Math.floor(Math.random() * n);
-            if (!indices.has(idx)) {
-                centroids.push([...vectors[idx]]);
-                indices.add(idx);
-            }
-        }
-        // K-means iterations
-        for (let iter = 0; iter < this.config.kmeansIterations; iter++) {
-            // Assignment step
-            const assignments = new Array(n).fill(0);
-            for (let i = 0; i < n; i++) {
-                let minDist = Infinity;
-                let bestCluster = 0;
-                for (let c = 0; c < centroids.length; c++) {
-                    const dist = this.euclideanDistance(vectors[i], centroids[c]);
-                    if (dist < minDist) {
-                        minDist = dist;
-                        bestCluster = c;
-                    }
-                }
-                assignments[i] = bestCluster;
-            }
-            // Update step
-            const clusterSums = Array(centroids.length)
-                .fill(0)
-                .map(() => Array(d).fill(0));
-            const clusterCounts = Array(centroids.length).fill(0);
-            for (let i = 0; i < n; i++) {
-                const cluster = assignments[i];
-                clusterCounts[cluster]++;
-                for (let j = 0; j < d; j++) {
-                    clusterSums[cluster][j] += vectors[i][j];
-                }
-            }
-            // Update centroids
-            for (let c = 0; c < centroids.length; c++) {
-                if (clusterCounts[c] > 0) {
-                    for (let j = 0; j < d; j++) {
-                        centroids[c][j] = clusterSums[c][j] / clusterCounts[c];
-                    }
-                }
-            }
-        }
-        // Fill remaining centroids if k > n
-        while (centroids.length < k) {
-            centroids.push(Array(d).fill(0));
-        }
-        return centroids;
-    }
-    /**
-     * Encode vector into quantized codes
-     */
-    encode(vector) {
-        if (!this.trained) {
-            throw new Error('Quantizer must be trained before encoding');
-        }
-        if (vector.length !== this.config.dimensions) {
-            throw new Error(`Vector dimension (${vector.length}) does not match config (${this.config.dimensions})`);
-        }
-        const startTime = performance.now();
-        const codes = new Uint8Array(this.config.subvectors);
-        for (let m = 0; m < this.config.subvectors; m++) {
-            const start = m * this.subvectorDim;
-            const end = start + this.subvectorDim;
-            const subvector = vector.slice(start, end);
-            // Find nearest centroid
-            let minDist = Infinity;
-            let bestCode = 0;
-            for (let c = 0; c < this.codebooks[m].centroids.length; c++) {
-                const dist = this.euclideanDistance(subvector, this.codebooks[m].centroids[c]);
-                if (dist < minDist) {
-                    minDist = dist;
-                    bestCode = c;
-                }
-            }
-            codes[m] = bestCode;
-        }
-        if (this.config.enableStats) {
-            this.stats.encodeCount++;
-            this.stats.totalEncodeTime += performance.now() - startTime;
-        }
-        return codes;
-    }
-    /**
-     * Decode quantized codes back to approximate vector
-     */
-    decode(codes) {
-        if (!this.trained) {
-            throw new Error('Quantizer must be trained before decoding');
-        }
-        if (codes.length !== this.config.subvectors) {
-            throw new Error(`Code length (${codes.length}) does not match subvectors (${this.config.subvectors})`);
-        }
-        const startTime = performance.now();
-        const vector = [];
-        for (let m = 0; m < this.config.subvectors; m++) {
-            const code = codes[m];
-            const centroid = this.codebooks[m].centroids[code];
-            vector.push(...centroid);
-        }
-        if (this.config.enableStats) {
-            this.stats.decodeCount++;
-            this.stats.totalDecodeTime += performance.now() - startTime;
-        }
-        return vector;
-    }
-    /**
-     * Compute asymmetric distance for faster search
-     * Compare query vector directly with quantized database vectors
-     */
-    asymmetricDistance(query, codes) {
-        if (!this.trained) {
-            throw new Error('Quantizer must be trained before computing distances');
-        }
-        let distance = 0;
-        for (let m = 0; m < this.config.subvectors; m++) {
-            const start = m * this.subvectorDim;
-            const end = start + this.subvectorDim;
-            const querySubvector = query.slice(start, end);
-            const code = codes[m];
-            const centroid = this.codebooks[m].centroids[code];
-            distance += this.euclideanDistanceSquared(querySubvector, centroid);
-        }
-        return Math.sqrt(distance);
-    }
-    /**
-     * Euclidean distance between two vectors
-     */
-    euclideanDistance(a, b) {
-        return Math.sqrt(this.euclideanDistanceSquared(a, b));
-    }
-    /**
-     * Squared Euclidean distance (faster, no sqrt)
-     */
-    euclideanDistanceSquared(a, b) {
-        let sum = 0;
-        for (let i = 0; i < a.length; i++) {
-            const diff = a[i] - b[i];
-            sum += diff * diff;
-        }
-        return sum;
-    }
-    /**
-     * Get compression statistics
-     */
-    getStats() {
-        const originalSize = this.config.dimensions * 4; // 4 bytes per float32
-        const compressedSize = this.config.subvectors; // 1 byte per subvector
-        return {
-            originalSize,
-            compressedSize,
-            compressionRatio: originalSize / compressedSize,
-            encodeCount: this.stats.encodeCount,
-            decodeCount: this.stats.decodeCount,
-            avgEncodeTime: this.stats.encodeCount > 0
-                ? this.stats.totalEncodeTime / this.stats.encodeCount
-                : 0,
-            avgDecodeTime: this.stats.decodeCount > 0
-                ? this.stats.totalDecodeTime / this.stats.decodeCount
-                : 0
-        };
-    }
-    /**
-     * Reset statistics
-     */
-    resetStats() {
-        this.stats = {
-            encodeCount: 0,
-            decodeCount: 0,
-            totalEncodeTime: 0,
-            totalDecodeTime: 0
-        };
-    }
-    /**
-     * Check if quantizer is trained
-     */
-    isTrained() {
-        return this.trained;
-    }
-    /**
-     * Export codebooks for persistence
-     */
-    exportCodebooks() {
-        return this.codebooks.map(cb => ({
-            centroids: cb.centroids.map(c => [...c])
-        }));
-    }
-    /**
-     * Import codebooks from persistence
-     */
-    importCodebooks(codebooks) {
-        if (codebooks.length !== this.config.subvectors) {
-            throw new Error(`Codebook count (${codebooks.length}) does not match subvectors (${this.config.subvectors})`);
-        }
-        this.codebooks = codebooks.map(cb => ({
-            centroids: cb.centroids.map(c => [...c])
-        }));
-        this.trained = true;
-    }
-    /**
-     * Get configuration
-     */
-    getConfig() {
-        return { ...this.config };
-    }
-}

package/dist/quantization/scalar-quantization.d.ts

@@ -1,100 +0,0 @@
-/**
- * Scalar Quantization - Simple yet powerful vector compression
- *
- * Achieves 85-95% accuracy with 4-16x compression by quantizing each dimension independently.
- * Unlike Product Quantization, works excellently with ANY data distribution including random data.
- *
- * Key advantages:
- * 1. Simple per-dimension min/max scaling
- * 2. No clustering required (faster training)
- * 3. Better accuracy on random/diverse data
- * 4. Fast encode/decode (< 1ms per vector)
- * 5. Guaranteed compression ratio
- *
- * Supported bit depths:
- * - 4-bit: 16 levels per dimension, 8x compression
- * - 8-bit: 256 levels per dimension, 4x compression
- * - 16-bit: 65536 levels per dimension, 2x compression
- */
-export interface ScalarQuantizerConfig {
-    dimensions: number;
-    bits: 4 | 8 | 16;
-    normalize?: boolean;
-}
-export interface AccuracyMetrics {
-    avgError: number;
-    maxError: number;
-    minError: number;
-    rmse: number;
-    accuracy: number;
-    recall10: number;
-}
-export interface ScalarQuantizationStats {
-    dimensions: number;
-    bits: 4 | 8 | 16;
-    levels: number;
-    compressionRatio: number;
-    bytesPerVector: number;
-    trained: boolean;
-    minValues: number[];
-    maxValues: number[];
-    ranges: number[];
-}
-/**
- * Scalar Quantizer - Per-dimension quantization for guaranteed accuracy
- *
- * Works by tracking min/max for each dimension independently, then
- * mapping values to discrete levels (16, 256, or 65536 levels).
- */
-export declare class ScalarQuantizer {
-    private dimensions;
-    private bits;
-    private normalize;
-    private levels;
-    private minValues;
-    private maxValues;
-    private ranges;
-    private trained;
-    constructor(config: ScalarQuantizerConfig);
-    /**
-     * Train the quantizer by finding min/max for each dimension
-     */
-    train(vectors: number[][]): Promise<void>;
-    /**
-     * Encode a vector to quantized codes
-     */
-    encode(vector: number[]): Uint8Array | Uint16Array;
-    /**
-     * Decode quantized codes back to a vector
-     */
-    decode(codes: Uint8Array | Uint16Array): number[];
-    /**
-     * Calculate asymmetric distance between query vector and quantized codes
-     * This is more accurate than symmetric distance for search
-     */
-    asymmetricDistance(query: number[], codes: Uint8Array | Uint16Array): number;
-    /**
-     * Evaluate accuracy on test vectors
-     */
-    evaluateAccuracy(testVectors: number[][]): AccuracyMetrics;
-    /**
-     * Get quantizer statistics
-     */
-    getStats(): ScalarQuantizationStats;
-    /**
-     * Check if quantizer is trained
-     */
-    isTrained(): boolean;
-    private encode4Bit;
-    private decode4Bit;
-    private encode8Bit;
-    private decode8Bit;
-    private encode16Bit;
-    private decode16Bit;
-    private normalizeVector;
-    private euclideanDistance;
-    private calculateRecall10;
-    private getCompressionRatio;
-    private getBytesPerVector;
-}
-//# sourceMappingURL=scalar-quantization.d.ts.map

package/dist/quantization/scalar-quantization.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"scalar-quantization.d.ts","sourceRoot":"","sources":["../../src/quantization/scalar-quantization.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,MAAM,WAAW,qBAAqB;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,uBAAuB;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED;;;;;GAKG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,SAAS,CAAU;IAC3B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAW;IAC5B,OAAO,CAAC,SAAS,CAAW;IAC5B,OAAO,CAAC,MAAM,CAAW;IACzB,OAAO,CAAC,OAAO,CAAkB;gBAErB,MAAM,EAAE,qBAAqB;IAUzC;;OAEG;IACG,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAuC/C;;OAEG;IACH,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,GAAG,WAAW;IAsBlD;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,UAAU,GAAG,WAAW,GAAG,MAAM,EAAE;IAejD;;;OAGG;IACH,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,UAAU,GAAG,WAAW,GAAG,MAAM;IAK5E;;OAEG;IACH,gBAAgB,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,GAAG,eAAe;IAiD1D;;OAEG;IACH,QAAQ,IAAI,uBAAuB;IAcnC;;OAEG;IACH,SAAS,IAAI,OAAO;IAMpB,OAAO,CAAC,UAAU;IAsBlB,OAAO,CAAC,UAAU;IAkBlB,OAAO,CAAC,UAAU;IAYlB,OAAO,CAAC,UAAU;IAWlB,OAAO,CAAC,WAAW;IAYnB,OAAO,CAAC,WAAW;IAWnB,OAAO,CAAC,eAAe;IAYvB,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,iBAAiB;IAuCzB,OAAO,CAAC,mBAAmB;IAM3B,OAAO,CAAC,iBAAiB;CAS1B"}