agentdb 1.0.0

package/dist/quantization/product-quantization.mjs

@@ -0,0 +1,282 @@
+/**
+ * Product Quantization (PQ) for 4-32x vector compression
+ *
+ * Reduces storage requirements while maintaining 95%+ search accuracy.
+ * Splits vectors into subvectors and quantizes each independently.
+ *
+ * @example
+ * ```typescript
+ * const pq = new ProductQuantizer({ dimensions: 768, subvectors: 8, bits: 8 });
+ * await pq.train(trainingVectors); // Train codebooks
+ *
+ * // Compress vectors (768 floats → 8 bytes = 96x reduction)
+ * const codes = pq.encode(vector);
+ *
+ * // Decompress for similarity search
+ * const approximation = pq.decode(codes);
+ * ```
+ */
+export class ProductQuantizer {
+    constructor(config) {
+        this.codebooks = [];
+        this.trained = false;
+        // Statistics
+        this.stats = {
+            encodeCount: 0,
+            decodeCount: 0,
+            totalEncodeTime: 0,
+            totalDecodeTime: 0
+        };
+        this.config = {
+            dimensions: config.dimensions,
+            subvectors: config.subvectors,
+            bits: config.bits,
+            kmeansIterations: config.kmeansIterations ?? 20,
+            enableStats: config.enableStats ?? true
+        };
+        if (config.dimensions % config.subvectors !== 0) {
+            throw new Error(`Dimensions (${config.dimensions}) must be divisible by subvectors (${config.subvectors})`);
+        }
+        this.subvectorDim = config.dimensions / config.subvectors;
+    }
+    /**
+     * Train codebooks using k-means clustering on training vectors
+     */
+    async train(trainingVectors) {
+        if (trainingVectors.length === 0) {
+            throw new Error('Training vectors cannot be empty');
+        }
+        const numCentroids = Math.pow(2, this.config.bits);
+        this.codebooks = [];
+        // Train one codebook per subvector
+        for (let m = 0; m < this.config.subvectors; m++) {
+            const subvectors = this.extractSubvectors(trainingVectors, m);
+            const centroids = this.trainKMeans(subvectors, numCentroids);
+            this.codebooks.push({ centroids });
+        }
+        this.trained = true;
+    }
+    /**
+     * Extract subvectors for a specific segment
+     */
+    extractSubvectors(vectors, segmentIndex) {
+        const start = segmentIndex * this.subvectorDim;
+        const end = start + this.subvectorDim;
+        return vectors.map(v => v.slice(start, end));
+    }
+    /**
+     * Train k-means clustering to create centroids
+     */
+    trainKMeans(vectors, k) {
+        const n = vectors.length;
+        const d = vectors[0].length;
+        // Initialize centroids randomly
+        const centroids = [];
+        const indices = new Set();
+        while (centroids.length < Math.min(k, n)) {
+            const idx = Math.floor(Math.random() * n);
+            if (!indices.has(idx)) {
+                centroids.push([...vectors[idx]]);
+                indices.add(idx);
+            }
+        }
+        // K-means iterations
+        for (let iter = 0; iter < this.config.kmeansIterations; iter++) {
+            // Assignment step
+            const assignments = new Array(n).fill(0);
+            for (let i = 0; i < n; i++) {
+                let minDist = Infinity;
+                let bestCluster = 0;
+                for (let c = 0; c < centroids.length; c++) {
+                    const dist = this.euclideanDistance(vectors[i], centroids[c]);
+                    if (dist < minDist) {
+                        minDist = dist;
+                        bestCluster = c;
+                    }
+                }
+                assignments[i] = bestCluster;
+            }
+            // Update step
+            const clusterSums = Array(centroids.length)
+                .fill(0)
+                .map(() => Array(d).fill(0));
+            const clusterCounts = Array(centroids.length).fill(0);
+            for (let i = 0; i < n; i++) {
+                const cluster = assignments[i];
+                clusterCounts[cluster]++;
+                for (let j = 0; j < d; j++) {
+                    clusterSums[cluster][j] += vectors[i][j];
+                }
+            }
+            // Update centroids
+            for (let c = 0; c < centroids.length; c++) {
+                if (clusterCounts[c] > 0) {
+                    for (let j = 0; j < d; j++) {
+                        centroids[c][j] = clusterSums[c][j] / clusterCounts[c];
+                    }
+                }
+            }
+        }
+        // Fill remaining centroids if k > n
+        while (centroids.length < k) {
+            centroids.push(Array(d).fill(0));
+        }
+        return centroids;
+    }
+    /**
+     * Encode vector into quantized codes
+     */
+    encode(vector) {
+        if (!this.trained) {
+            throw new Error('Quantizer must be trained before encoding');
+        }
+        if (vector.length !== this.config.dimensions) {
+            throw new Error(`Vector dimension (${vector.length}) does not match config (${this.config.dimensions})`);
+        }
+        const startTime = performance.now();
+        const codes = new Uint8Array(this.config.subvectors);
+        for (let m = 0; m < this.config.subvectors; m++) {
+            const start = m * this.subvectorDim;
+            const end = start + this.subvectorDim;
+            const subvector = vector.slice(start, end);
+            // Find nearest centroid
+            let minDist = Infinity;
+            let bestCode = 0;
+            for (let c = 0; c < this.codebooks[m].centroids.length; c++) {
+                const dist = this.euclideanDistance(subvector, this.codebooks[m].centroids[c]);
+                if (dist < minDist) {
+                    minDist = dist;
+                    bestCode = c;
+                }
+            }
+            codes[m] = bestCode;
+        }
+        if (this.config.enableStats) {
+            this.stats.encodeCount++;
+            this.stats.totalEncodeTime += performance.now() - startTime;
+        }
+        return codes;
+    }
+    /**
+     * Decode quantized codes back to approximate vector
+     */
+    decode(codes) {
+        if (!this.trained) {
+            throw new Error('Quantizer must be trained before decoding');
+        }
+        if (codes.length !== this.config.subvectors) {
+            throw new Error(`Code length (${codes.length}) does not match subvectors (${this.config.subvectors})`);
+        }
+        const startTime = performance.now();
+        const vector = [];
+        for (let m = 0; m < this.config.subvectors; m++) {
+            const code = codes[m];
+            const centroid = this.codebooks[m].centroids[code];
+            vector.push(...centroid);
+        }
+        if (this.config.enableStats) {
+            this.stats.decodeCount++;
+            this.stats.totalDecodeTime += performance.now() - startTime;
+        }
+        return vector;
+    }
+    /**
+     * Compute asymmetric distance for faster search
+     * Compare query vector directly with quantized database vectors
+     */
+    asymmetricDistance(query, codes) {
+        if (!this.trained) {
+            throw new Error('Quantizer must be trained before computing distances');
+        }
+        let distance = 0;
+        for (let m = 0; m < this.config.subvectors; m++) {
+            const start = m * this.subvectorDim;
+            const end = start + this.subvectorDim;
+            const querySubvector = query.slice(start, end);
+            const code = codes[m];
+            const centroid = this.codebooks[m].centroids[code];
+            distance += this.euclideanDistanceSquared(querySubvector, centroid);
+        }
+        return Math.sqrt(distance);
+    }
+    /**
+     * Euclidean distance between two vectors
+     */
+    euclideanDistance(a, b) {
+        return Math.sqrt(this.euclideanDistanceSquared(a, b));
+    }
+    /**
+     * Squared Euclidean distance (faster, no sqrt)
+     */
+    euclideanDistanceSquared(a, b) {
+        let sum = 0;
+        for (let i = 0; i < a.length; i++) {
+            const diff = a[i] - b[i];
+            sum += diff * diff;
+        }
+        return sum;
+    }
+    /**
+     * Get compression statistics
+     */
+    getStats() {
+        const originalSize = this.config.dimensions * 4; // 4 bytes per float32
+        const compressedSize = this.config.subvectors; // 1 byte per subvector
+        return {
+            originalSize,
+            compressedSize,
+            compressionRatio: originalSize / compressedSize,
+            encodeCount: this.stats.encodeCount,
+            decodeCount: this.stats.decodeCount,
+            avgEncodeTime: this.stats.encodeCount > 0
+                ? this.stats.totalEncodeTime / this.stats.encodeCount
+                : 0,
+            avgDecodeTime: this.stats.decodeCount > 0
+                ? this.stats.totalDecodeTime / this.stats.decodeCount
+                : 0
+        };
+    }
+    /**
+     * Reset statistics
+     */
+    resetStats() {
+        this.stats = {
+            encodeCount: 0,
+            decodeCount: 0,
+            totalEncodeTime: 0,
+            totalDecodeTime: 0
+        };
+    }
+    /**
+     * Check if quantizer is trained
+     */
+    isTrained() {
+        return this.trained;
+    }
+    /**
+     * Export codebooks for persistence
+     */
+    exportCodebooks() {
+        return this.codebooks.map(cb => ({
+            centroids: cb.centroids.map(c => [...c])
+        }));
+    }
+    /**
+     * Import codebooks from persistence
+     */
+    importCodebooks(codebooks) {
+        if (codebooks.length !== this.config.subvectors) {
+            throw new Error(`Codebook count (${codebooks.length}) does not match subvectors (${this.config.subvectors})`);
+        }
+        this.codebooks = codebooks.map(cb => ({
+            centroids: cb.centroids.map(c => [...c])
+        }));
+        this.trained = true;
+    }
+    /**
+     * Get configuration
+     */
+    getConfig() {
+        return { ...this.config };
+    }
+}

package/dist/quantization/scalar-quantization.d.ts

@@ -0,0 +1,100 @@
+/**
+ * Scalar Quantization - Simple yet powerful vector compression
+ *
+ * Achieves 85-95% accuracy with 4-16x compression by quantizing each dimension independently.
+ * Unlike Product Quantization, works excellently with ANY data distribution including random data.
+ *
+ * Key advantages:
+ * 1. Simple per-dimension min/max scaling
+ * 2. No clustering required (faster training)
+ * 3. Better accuracy on random/diverse data
+ * 4. Fast encode/decode (< 1ms per vector)
+ * 5. Guaranteed compression ratio
+ *
+ * Supported bit depths:
+ * - 4-bit: 16 levels per dimension, 8x compression
+ * - 8-bit: 256 levels per dimension, 4x compression
+ * - 16-bit: 65536 levels per dimension, 2x compression
+ */
+export interface ScalarQuantizerConfig {
+    dimensions: number;
+    bits: 4 | 8 | 16;
+    normalize?: boolean;
+}
+export interface AccuracyMetrics {
+    avgError: number;
+    maxError: number;
+    minError: number;
+    rmse: number;
+    accuracy: number;
+    recall10: number;
+}
+export interface ScalarQuantizationStats {
+    dimensions: number;
+    bits: 4 | 8 | 16;
+    levels: number;
+    compressionRatio: number;
+    bytesPerVector: number;
+    trained: boolean;
+    minValues: number[];
+    maxValues: number[];
+    ranges: number[];
+}
+/**
+ * Scalar Quantizer - Per-dimension quantization for guaranteed accuracy
+ *
+ * Works by tracking min/max for each dimension independently, then
+ * mapping values to discrete levels (16, 256, or 65536 levels).
+ */
+export declare class ScalarQuantizer {
+    private dimensions;
+    private bits;
+    private normalize;
+    private levels;
+    private minValues;
+    private maxValues;
+    private ranges;
+    private trained;
+    constructor(config: ScalarQuantizerConfig);
+    /**
+     * Train the quantizer by finding min/max for each dimension
+     */
+    train(vectors: number[][]): Promise<void>;
+    /**
+     * Encode a vector to quantized codes
+     */
+    encode(vector: number[]): Uint8Array | Uint16Array;
+    /**
+     * Decode quantized codes back to a vector
+     */
+    decode(codes: Uint8Array | Uint16Array): number[];
+    /**
+     * Calculate asymmetric distance between query vector and quantized codes
+     * This is more accurate than symmetric distance for search
+     */
+    asymmetricDistance(query: number[], codes: Uint8Array | Uint16Array): number;
+    /**
+     * Evaluate accuracy on test vectors
+     */
+    evaluateAccuracy(testVectors: number[][]): AccuracyMetrics;
+    /**
+     * Get quantizer statistics
+     */
+    getStats(): ScalarQuantizationStats;
+    /**
+     * Check if quantizer is trained
+     */
+    isTrained(): boolean;
+    private encode4Bit;
+    private decode4Bit;
+    private encode8Bit;
+    private decode8Bit;
+    private encode16Bit;
+    private decode16Bit;
+    private normalizeVector;
+    private euclideanDistance;
+    private calculateRecall10;
+    private getCompressionRatio;
+    private getBytesPerVector;
+}
+//# sourceMappingURL=scalar-quantization.d.ts.map

package/dist/quantization/scalar-quantization.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scalar-quantization.d.ts","sourceRoot":"","sources":["../../src/quantization/scalar-quantization.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,MAAM,WAAW,qBAAqB;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,uBAAuB;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED;;;;;GAKG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,SAAS,CAAU;IAC3B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAW;IAC5B,OAAO,CAAC,SAAS,CAAW;IAC5B,OAAO,CAAC,MAAM,CAAW;IACzB,OAAO,CAAC,OAAO,CAAkB;gBAErB,MAAM,EAAE,qBAAqB;IAUzC;;OAEG;IACG,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAuC/C;;OAEG;IACH,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,GAAG,WAAW;IAsBlD;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,UAAU,GAAG,WAAW,GAAG,MAAM,EAAE;IAejD;;;OAGG;IACH,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,UAAU,GAAG,WAAW,GAAG,MAAM;IAK5E;;OAEG;IACH,gBAAgB,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,GAAG,eAAe;IAiD1D;;OAEG;IACH,QAAQ,IAAI,uBAAuB;IAcnC;;OAEG;IACH,SAAS,IAAI,OAAO;IAMpB,OAAO,CAAC,UAAU;IAsBlB,OAAO,CAAC,UAAU;IAkBlB,OAAO,CAAC,UAAU;IAYlB,OAAO,CAAC,UAAU;IAWlB,OAAO,CAAC,WAAW;IAYnB,OAAO,CAAC,WAAW;IAWnB,OAAO,CAAC,eAAe;IAYvB,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,iBAAiB;IAuCzB,OAAO,CAAC,mBAAmB;IAM3B,OAAO,CAAC,iBAAiB;CAS1B"}