agentdb 1.0.0

package/dist/cli/quantization/product-quantization.d.ts

@@ -0,0 +1,108 @@
+/**
+ * Product Quantization (PQ) for 4-32x vector compression
+ *
+ * Reduces storage requirements while maintaining 95%+ search accuracy.
+ * Splits vectors into subvectors and quantizes each independently.
+ *
+ * @example
+ * ```typescript
+ * const pq = new ProductQuantizer({ dimensions: 768, subvectors: 8, bits: 8 });
+ * await pq.train(trainingVectors); // Train codebooks
+ *
+ * // Compress vectors (768 floats → 8 bytes = 96x reduction)
+ * const codes = pq.encode(vector);
+ *
+ * // Decompress for similarity search
+ * const approximation = pq.decode(codes);
+ * ```
+ */
+export interface ProductQuantizerConfig {
+    /** Vector dimensions (e.g., 768 for many embeddings) */
+    dimensions: number;
+    /** Number of subvectors (typically 4-16) */
+    subvectors: number;
+    /** Bits per subvector (typically 8 = 256 centroids) */
+    bits: number;
+    /** K-means iterations for training (default: 20) */
+    kmeansIterations?: number;
+    /** Enable compression statistics (default: true) */
+    enableStats?: boolean;
+}
+export interface CompressionStats {
+    originalSize: number;
+    compressedSize: number;
+    compressionRatio: number;
+    encodeCount: number;
+    decodeCount: number;
+    avgEncodeTime: number;
+    avgDecodeTime: number;
+}
+interface Codebook {
+    centroids: number[][];
+}
+export declare class ProductQuantizer {
+    private config;
+    private codebooks;
+    private subvectorDim;
+    private trained;
+    private stats;
+    constructor(config: ProductQuantizerConfig);
+    /**
+     * Train codebooks using k-means clustering on training vectors
+     */
+    train(trainingVectors: number[][]): Promise<void>;
+    /**
+     * Extract subvectors for a specific segment
+     */
+    private extractSubvectors;
+    /**
+     * Train k-means clustering to create centroids
+     */
+    private trainKMeans;
+    /**
+     * Encode vector into quantized codes
+     */
+    encode(vector: number[]): Uint8Array;
+    /**
+     * Decode quantized codes back to approximate vector
+     */
+    decode(codes: Uint8Array): number[];
+    /**
+     * Compute asymmetric distance for faster search
+     * Compare query vector directly with quantized database vectors
+     */
+    asymmetricDistance(query: number[], codes: Uint8Array): number;
+    /**
+     * Euclidean distance between two vectors
+     */
+    private euclideanDistance;
+    /**
+     * Squared Euclidean distance (faster, no sqrt)
+     */
+    private euclideanDistanceSquared;
+    /**
+     * Get compression statistics
+     */
+    getStats(): CompressionStats;
+    /**
+     * Reset statistics
+     */
+    resetStats(): void;
+    /**
+     * Check if quantizer is trained
+     */
+    isTrained(): boolean;
+    /**
+     * Export codebooks for persistence
+     */
+    exportCodebooks(): Codebook[];
+    /**
+     * Import codebooks from persistence
+     */
+    importCodebooks(codebooks: Codebook[]): void;
+    /**
+     * Get configuration
+     */
+    getConfig(): Required<ProductQuantizerConfig>;
+}
+export {};

package/dist/cli/quantization/product-quantization.js

@@ -0,0 +1,350 @@
+"use strict";
+/**
+ * Product Quantization (PQ) for 4-32x vector compression
+ *
+ * Reduces storage requirements while maintaining 95%+ search accuracy.
+ * Splits vectors into subvectors and quantizes each independently.
+ *
+ * @example
+ * ```typescript
+ * const pq = new ProductQuantizer({ dimensions: 768, subvectors: 8, bits: 8 });
+ * await pq.train(trainingVectors); // Train codebooks
+ *
+ * // Compress vectors (768 floats → 8 bytes = 96x reduction)
+ * const codes = pq.encode(vector);
+ *
+ * // Decompress for similarity search
+ * const approximation = pq.decode(codes);
+ * ```
+ */
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
+    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
+    if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
+        if (ar || !(i in from)) {
+            if (!ar) ar = Array.prototype.slice.call(from, 0, i);
+            ar[i] = from[i];
+        }
+    }
+    return to.concat(ar || Array.prototype.slice.call(from));
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ProductQuantizer = void 0;
+var ProductQuantizer = /** @class */ (function () {
+    function ProductQuantizer(config) {
+        var _a, _b;
+        this.codebooks = [];
+        this.trained = false;
+        // Statistics
+        this.stats = {
+            encodeCount: 0,
+            decodeCount: 0,
+            totalEncodeTime: 0,
+            totalDecodeTime: 0
+        };
+        this.config = {
+            dimensions: config.dimensions,
+            subvectors: config.subvectors,
+            bits: config.bits,
+            kmeansIterations: (_a = config.kmeansIterations) !== null && _a !== void 0 ? _a : 20,
+            enableStats: (_b = config.enableStats) !== null && _b !== void 0 ? _b : true
+        };
+        if (config.dimensions % config.subvectors !== 0) {
+            throw new Error("Dimensions (".concat(config.dimensions, ") must be divisible by subvectors (").concat(config.subvectors, ")"));
+        }
+        this.subvectorDim = config.dimensions / config.subvectors;
+    }
+    /**
+     * Train codebooks using k-means clustering on training vectors
+     */
+    ProductQuantizer.prototype.train = function (trainingVectors) {
+        return __awaiter(this, void 0, void 0, function () {
+            var numCentroids, m, subvectors, centroids;
+            return __generator(this, function (_a) {
+                if (trainingVectors.length === 0) {
+                    throw new Error('Training vectors cannot be empty');
+                }
+                numCentroids = Math.pow(2, this.config.bits);
+                this.codebooks = [];
+                // Train one codebook per subvector
+                for (m = 0; m < this.config.subvectors; m++) {
+                    subvectors = this.extractSubvectors(trainingVectors, m);
+                    centroids = this.trainKMeans(subvectors, numCentroids);
+                    this.codebooks.push({ centroids: centroids });
+                }
+                this.trained = true;
+                return [2 /*return*/];
+            });
+        });
+    };
+    /**
+     * Extract subvectors for a specific segment
+     */
+    ProductQuantizer.prototype.extractSubvectors = function (vectors, segmentIndex) {
+        var start = segmentIndex * this.subvectorDim;
+        var end = start + this.subvectorDim;
+        return vectors.map(function (v) { return v.slice(start, end); });
+    };
+    /**
+     * Train k-means clustering to create centroids
+     */
+    ProductQuantizer.prototype.trainKMeans = function (vectors, k) {
+        var n = vectors.length;
+        var d = vectors[0].length;
+        // Initialize centroids randomly
+        var centroids = [];
+        var indices = new Set();
+        while (centroids.length < Math.min(k, n)) {
+            var idx = Math.floor(Math.random() * n);
+            if (!indices.has(idx)) {
+                centroids.push(__spreadArray([], vectors[idx], true));
+                indices.add(idx);
+            }
+        }
+        // K-means iterations
+        for (var iter = 0; iter < this.config.kmeansIterations; iter++) {
+            // Assignment step
+            var assignments = new Array(n).fill(0);
+            for (var i = 0; i < n; i++) {
+                var minDist = Infinity;
+                var bestCluster = 0;
+                for (var c = 0; c < centroids.length; c++) {
+                    var dist = this.euclideanDistance(vectors[i], centroids[c]);
+                    if (dist < minDist) {
+                        minDist = dist;
+                        bestCluster = c;
+                    }
+                }
+                assignments[i] = bestCluster;
+            }
+            // Update step
+            var clusterSums = Array(centroids.length)
+                .fill(0)
+                .map(function () { return Array(d).fill(0); });
+            var clusterCounts = Array(centroids.length).fill(0);
+            for (var i = 0; i < n; i++) {
+                var cluster = assignments[i];
+                clusterCounts[cluster]++;
+                for (var j = 0; j < d; j++) {
+                    clusterSums[cluster][j] += vectors[i][j];
+                }
+            }
+            // Update centroids
+            for (var c = 0; c < centroids.length; c++) {
+                if (clusterCounts[c] > 0) {
+                    for (var j = 0; j < d; j++) {
+                        centroids[c][j] = clusterSums[c][j] / clusterCounts[c];
+                    }
+                }
+            }
+        }
+        // Fill remaining centroids if k > n
+        while (centroids.length < k) {
+            centroids.push(Array(d).fill(0));
+        }
+        return centroids;
+    };
+    /**
+     * Encode vector into quantized codes
+     */
+    ProductQuantizer.prototype.encode = function (vector) {
+        if (!this.trained) {
+            throw new Error('Quantizer must be trained before encoding');
+        }
+        if (vector.length !== this.config.dimensions) {
+            throw new Error("Vector dimension (".concat(vector.length, ") does not match config (").concat(this.config.dimensions, ")"));
+        }
+        var startTime = performance.now();
+        var codes = new Uint8Array(this.config.subvectors);
+        for (var m = 0; m < this.config.subvectors; m++) {
+            var start = m * this.subvectorDim;
+            var end = start + this.subvectorDim;
+            var subvector = vector.slice(start, end);
+            // Find nearest centroid
+            var minDist = Infinity;
+            var bestCode = 0;
+            for (var c = 0; c < this.codebooks[m].centroids.length; c++) {
+                var dist = this.euclideanDistance(subvector, this.codebooks[m].centroids[c]);
+                if (dist < minDist) {
+                    minDist = dist;
+                    bestCode = c;
+                }
+            }
+            codes[m] = bestCode;
+        }
+        if (this.config.enableStats) {
+            this.stats.encodeCount++;
+            this.stats.totalEncodeTime += performance.now() - startTime;
+        }
+        return codes;
+    };
+    /**
+     * Decode quantized codes back to approximate vector
+     */
+    ProductQuantizer.prototype.decode = function (codes) {
+        if (!this.trained) {
+            throw new Error('Quantizer must be trained before decoding');
+        }
+        if (codes.length !== this.config.subvectors) {
+            throw new Error("Code length (".concat(codes.length, ") does not match subvectors (").concat(this.config.subvectors, ")"));
+        }
+        var startTime = performance.now();
+        var vector = [];
+        for (var m = 0; m < this.config.subvectors; m++) {
+            var code = codes[m];
+            var centroid = this.codebooks[m].centroids[code];
+            vector.push.apply(vector, centroid);
+        }
+        if (this.config.enableStats) {
+            this.stats.decodeCount++;
+            this.stats.totalDecodeTime += performance.now() - startTime;
+        }
+        return vector;
+    };
+    /**
+     * Compute asymmetric distance for faster search
+     * Compare query vector directly with quantized database vectors
+     */
+    ProductQuantizer.prototype.asymmetricDistance = function (query, codes) {
+        if (!this.trained) {
+            throw new Error('Quantizer must be trained before computing distances');
+        }
+        var distance = 0;
+        for (var m = 0; m < this.config.subvectors; m++) {
+            var start = m * this.subvectorDim;
+            var end = start + this.subvectorDim;
+            var querySubvector = query.slice(start, end);
+            var code = codes[m];
+            var centroid = this.codebooks[m].centroids[code];
+            distance += this.euclideanDistanceSquared(querySubvector, centroid);
+        }
+        return Math.sqrt(distance);
+    };
+    /**
+     * Euclidean distance between two vectors
+     */
+    ProductQuantizer.prototype.euclideanDistance = function (a, b) {
+        return Math.sqrt(this.euclideanDistanceSquared(a, b));
+    };
+    /**
+     * Squared Euclidean distance (faster, no sqrt)
+     */
+    ProductQuantizer.prototype.euclideanDistanceSquared = function (a, b) {
+        var sum = 0;
+        for (var i = 0; i < a.length; i++) {
+            var diff = a[i] - b[i];
+            sum += diff * diff;
+        }
+        return sum;
+    };
+    /**
+     * Get compression statistics
+     */
+    ProductQuantizer.prototype.getStats = function () {
+        var originalSize = this.config.dimensions * 4; // 4 bytes per float32
+        var compressedSize = this.config.subvectors; // 1 byte per subvector
+        return {
+            originalSize: originalSize,
+            compressedSize: compressedSize,
+            compressionRatio: originalSize / compressedSize,
+            encodeCount: this.stats.encodeCount,
+            decodeCount: this.stats.decodeCount,
+            avgEncodeTime: this.stats.encodeCount > 0
+                ? this.stats.totalEncodeTime / this.stats.encodeCount
+                : 0,
+            avgDecodeTime: this.stats.decodeCount > 0
+                ? this.stats.totalDecodeTime / this.stats.decodeCount
+                : 0
+        };
+    };
+    /**
+     * Reset statistics
+     */
+    ProductQuantizer.prototype.resetStats = function () {
+        this.stats = {
+            encodeCount: 0,
+            decodeCount: 0,
+            totalEncodeTime: 0,
+            totalDecodeTime: 0
+        };
+    };
+    /**
+     * Check if quantizer is trained
+     */
+    ProductQuantizer.prototype.isTrained = function () {
+        return this.trained;
+    };
+    /**
+     * Export codebooks for persistence
+     */
+    ProductQuantizer.prototype.exportCodebooks = function () {
+        return this.codebooks.map(function (cb) { return ({
+            centroids: cb.centroids.map(function (c) { return __spreadArray([], c, true); })
+        }); });
+    };
+    /**
+     * Import codebooks from persistence
+     */
+    ProductQuantizer.prototype.importCodebooks = function (codebooks) {
+        if (codebooks.length !== this.config.subvectors) {
+            throw new Error("Codebook count (".concat(codebooks.length, ") does not match subvectors (").concat(this.config.subvectors, ")"));
+        }
+        this.codebooks = codebooks.map(function (cb) { return ({
+            centroids: cb.centroids.map(function (c) { return __spreadArray([], c, true); })
+        }); });
+        this.trained = true;
+    };
+    /**
+     * Get configuration
+     */
+    ProductQuantizer.prototype.getConfig = function () {
+        return __assign({}, this.config);
+    };
+    return ProductQuantizer;
+}());
+exports.ProductQuantizer = ProductQuantizer;