agentic-qe 3.7.21 → 3.8.0

package/dist/integrations/ruvector/compressed-hnsw-integration.d.ts

@@ -0,0 +1,176 @@
+/**
+ * Compressed HNSW Integration
+ * ADR-085: Temporal Tensor Pattern Compression + HNSW Backend
+ *
+ * Wraps an IHnswIndexProvider to transparently handle compressed vectors.
+ * On add(), the original vector is passed to the underlying HNSW index for
+ * accurate search, while a compressed copy is stored for memory savings.
+ * On search(), results come from the HNSW index normally.
+ *
+ * The compressIndex() method bulk-compresses cold/warm vectors, and
+ * getMemoryStats() reports the savings achieved.
+ *
+ * Activation is controlled by the `useTemporalCompression` feature flag.
+ *
+ * @module integrations/ruvector/compressed-hnsw-integration
+ */
+import type { IHnswIndexProvider, SearchResult } from '../../kernel/hnsw-index-provider.js';
+import type { CompressionTier } from './temporal-compression.js';
+import { TemporalCompressionService } from './temporal-compression.js';
+/**
+ * Memory statistics for the compressed HNSW index.
+ */
+export interface CompressedHnswMemoryStats {
+    /** Total vectors in the index */
+    totalVectors: number;
+    /** Vectors that are still uncompressed */
+    uncompressedCount: number;
+    /** Vectors that have been compressed */
+    compressedCount: number;
+    /** Breakdown of compressed vectors by tier */
+    compressedByTier: Record<CompressionTier, number>;
+    /** Total bytes used by original (uncompressed) vectors */
+    originalBytes: number;
+    /** Total bytes used by compressed vectors (effective) */
+    compressedBytes: number;
+    /** Total bytes saved through compression */
+    bytesSaved: number;
+    /** Savings as a percentage (0-100) */
+    savingsPercent: number;
+    /** Whether temporal compression is enabled via feature flag */
+    compressionEnabled: boolean;
+}
+/**
+ * Wrapper around IHnswIndexProvider that transparently manages compressed
+ * vector storage alongside the HNSW index.
+ *
+ * The HNSW index always receives full-precision vectors for accurate search.
+ * Compression is a separate memory-optimization layer that stores compact
+ * representations of vectors that can be decompressed on demand.
+ *
+ * Usage:
+ * ```typescript
+ * const baseIndex = new ProgressiveHnswBackend({ dimensions: 384 });
+ * const compressed = new CompressedHnswIntegration(baseIndex);
+ *
+ * compressed.add(1, embedding, { domain: 'auth' });
+ * const results = compressed.search(queryEmbedding, 10);
+ *
+ * // Compress cold vectors to save memory
+ * const stats = compressed.compressIndex();
+ * console.log(`Compressed ${stats.compressedCount} vectors`);
+ * ```
+ */
+export declare class CompressedHnswIntegration implements IHnswIndexProvider {
+    private readonly backend;
+    private readonly compressionService;
+    private readonly vectorEntries;
+    /**
+     * Create a CompressedHnswIntegration.
+     *
+     * @param backend - The underlying HNSW index provider to delegate to
+     * @param compressionService - Optional compression service (defaults to new instance)
+     */
+    constructor(backend: IHnswIndexProvider, compressionService?: TemporalCompressionService);
+    /**
+     * Add a vector to the index.
+     *
+     * The full-precision vector is passed to the HNSW backend for search.
+     * A copy is stored locally for potential future compression.
+     * If temporal compression is enabled, the vector is also compressed
+     * immediately and the compressed form is stored.
+     */
+    add(id: number, vector: Float32Array, metadata?: Record<string, unknown>): void;
+    /**
+     * Search for the k nearest neighbors.
+     *
+     * Delegates entirely to the HNSW backend which holds full-precision
+     * vectors. Updates access timestamps for returned results so they
+     * stay in the hot tier.
+     */
+    search(query: Float32Array, k: number): SearchResult[];
+    /**
+     * Remove a vector from the index.
+     */
+    remove(id: number): boolean;
+    /**
+     * Get the number of vectors in the index.
+     */
+    size(): number;
+    /**
+     * Get the configured vector dimensions.
+     */
+    dimensions(): number;
+    /**
+     * Get estimated recall. Delegates to the underlying backend.
+     */
+    recall(): number;
+    /**
+     * Compress vectors in the index based on their access tier.
+     *
+     * Iterates over all stored vectors and compresses those classified as
+     * warm or cold. Hot vectors are left uncompressed for maximum fidelity.
+     * After compression, the original float32 data for warm/cold vectors
+     * is released to free memory (the HNSW backend still holds its own copy
+     * for search operations).
+     *
+     * @param options - Optional configuration
+     * @param options.compressHot - If true, also compress hot-tier vectors (default: false)
+     * @returns Statistics about the compression operation
+     */
+    compressIndex(options?: {
+        compressHot?: boolean;
+    }): CompressedHnswBulkResult;
+    /**
+     * Decompress and retrieve the vector for a given ID.
+     *
+     * Returns the original vector if still available, otherwise
+     * decompresses from the stored compressed form.
+     *
+     * @param id - The vector ID
+     * @returns The decompressed Float32Array, or null if not found
+     */
+    getVector(id: number): Float32Array | null;
+    /**
+     * Get the compression tier for a specific vector.
+     *
+     * @param id - The vector ID
+     * @returns The tier, or null if not found or not compressed
+     */
+    getVectorTier(id: number): CompressionTier | null;
+    /**
+     * Update the last-accessed timestamp for a vector.
+     * This affects which tier it falls into during the next compressIndex() call.
+     *
+     * @param id - The vector ID
+     * @param accessDate - The access date (defaults to now)
+     */
+    touchVector(id: number, accessDate?: Date): void;
+    /**
+     * Get memory usage statistics for the compressed index.
+     */
+    getMemoryStats(): CompressedHnswMemoryStats;
+    /**
+     * Get the underlying HNSW backend.
+     * Useful for accessing backend-specific methods (e.g., NativeHnswBackend.getMetrics()).
+     */
+    getBackend(): IHnswIndexProvider;
+    /**
+     * Get the compression service instance.
+     */
+    getCompressionService(): TemporalCompressionService;
+}
+/**
+ * Result of a bulk compression operation via compressIndex().
+ */
+export interface CompressedHnswBulkResult {
+    /** Number of vectors compressed in this operation */
+    compressedCount: number;
+    /** Number of vectors skipped (already compressed or hot) */
+    skippedCount: number;
+    /** Bytes freed by releasing original vectors */
+    bytesFreed: number;
+    /** Breakdown of newly compressed vectors by tier */
+    byTier: Record<CompressionTier, number>;
+}
+//# sourceMappingURL=compressed-hnsw-integration.d.ts.map

package/dist/integrations/ruvector/compressed-hnsw-integration.js

@@ -0,0 +1,301 @@
+/**
+ * Compressed HNSW Integration
+ * ADR-085: Temporal Tensor Pattern Compression + HNSW Backend
+ *
+ * Wraps an IHnswIndexProvider to transparently handle compressed vectors.
+ * On add(), the original vector is passed to the underlying HNSW index for
+ * accurate search, while a compressed copy is stored for memory savings.
+ * On search(), results come from the HNSW index normally.
+ *
+ * The compressIndex() method bulk-compresses cold/warm vectors, and
+ * getMemoryStats() reports the savings achieved.
+ *
+ * Activation is controlled by the `useTemporalCompression` feature flag.
+ *
+ * @module integrations/ruvector/compressed-hnsw-integration
+ */
+import { createTemporalCompressionService, } from './temporal-compression.js';
+import { isTemporalCompressionEnabled } from './feature-flags.js';
+import { LoggerFactory } from '../../logging/index.js';
+const logger = LoggerFactory.create('compressed-hnsw');
+// ============================================================================
+// CompressedHnswIntegration
+// ============================================================================
+/**
+ * Wrapper around IHnswIndexProvider that transparently manages compressed
+ * vector storage alongside the HNSW index.
+ *
+ * The HNSW index always receives full-precision vectors for accurate search.
+ * Compression is a separate memory-optimization layer that stores compact
+ * representations of vectors that can be decompressed on demand.
+ *
+ * Usage:
+ * ```typescript
+ * const baseIndex = new ProgressiveHnswBackend({ dimensions: 384 });
+ * const compressed = new CompressedHnswIntegration(baseIndex);
+ *
+ * compressed.add(1, embedding, { domain: 'auth' });
+ * const results = compressed.search(queryEmbedding, 10);
+ *
+ * // Compress cold vectors to save memory
+ * const stats = compressed.compressIndex();
+ * console.log(`Compressed ${stats.compressedCount} vectors`);
+ * ```
+ */
+export class CompressedHnswIntegration {
+    backend;
+    compressionService;
+    vectorEntries = new Map();
+    /**
+     * Create a CompressedHnswIntegration.
+     *
+     * @param backend - The underlying HNSW index provider to delegate to
+     * @param compressionService - Optional compression service (defaults to new instance)
+     */
+    constructor(backend, compressionService) {
+        this.backend = backend;
+        this.compressionService = compressionService ?? createTemporalCompressionService();
+    }
+    // ==========================================================================
+    // IHnswIndexProvider Implementation
+    // ==========================================================================
+    /**
+     * Add a vector to the index.
+     *
+     * The full-precision vector is passed to the HNSW backend for search.
+     * A copy is stored locally for potential future compression.
+     * If temporal compression is enabled, the vector is also compressed
+     * immediately and the compressed form is stored.
+     */
+    add(id, vector, metadata) {
+        // Always delegate the full-precision vector to the HNSW backend
+        this.backend.add(id, vector, metadata);
+        const now = new Date();
+        const entry = {
+            original: new Float32Array(vector),
+            compressed: null,
+            tier: null,
+            lastAccessedAt: now,
+        };
+        // If compression is enabled, compress immediately but keep the original
+        // in the HNSW index for accurate search
+        if (isTemporalCompressionEnabled()) {
+            const tier = this.compressionService.classifyTier(now);
+            entry.compressed = this.compressionService.compress(vector, tier);
+            entry.tier = tier;
+        }
+        this.vectorEntries.set(id, entry);
+    }
+    /**
+     * Search for the k nearest neighbors.
+     *
+     * Delegates entirely to the HNSW backend which holds full-precision
+     * vectors. Updates access timestamps for returned results so they
+     * stay in the hot tier.
+     */
+    search(query, k) {
+        const results = this.backend.search(query, k);
+        // Update last-accessed timestamps for returned results
+        const now = new Date();
+        for (const result of results) {
+            const entry = this.vectorEntries.get(result.id);
+            if (entry) {
+                entry.lastAccessedAt = now;
+            }
+        }
+        return results;
+    }
+    /**
+     * Remove a vector from the index.
+     */
+    remove(id) {
+        this.vectorEntries.delete(id);
+        return this.backend.remove(id);
+    }
+    /**
+     * Get the number of vectors in the index.
+     */
+    size() {
+        return this.backend.size();
+    }
+    /**
+     * Get the configured vector dimensions.
+     */
+    dimensions() {
+        return this.backend.dimensions();
+    }
+    /**
+     * Get estimated recall. Delegates to the underlying backend.
+     */
+    recall() {
+        return this.backend.recall();
+    }
+    // ==========================================================================
+    // Compression Operations
+    // ==========================================================================
+    /**
+     * Compress vectors in the index based on their access tier.
+     *
+     * Iterates over all stored vectors and compresses those classified as
+     * warm or cold. Hot vectors are left uncompressed for maximum fidelity.
+     * After compression, the original float32 data for warm/cold vectors
+     * is released to free memory (the HNSW backend still holds its own copy
+     * for search operations).
+     *
+     * @param options - Optional configuration
+     * @param options.compressHot - If true, also compress hot-tier vectors (default: false)
+     * @returns Statistics about the compression operation
+     */
+    compressIndex(options) {
+        const compressHot = options?.compressHot ?? false;
+        let compressedCount = 0;
+        let skippedCount = 0;
+        let bytesFreed = 0;
+        const byTier = { hot: 0, warm: 0, cold: 0 };
+        for (const [id, entry] of this.vectorEntries) {
+            const tier = this.compressionService.classifyTier(entry.lastAccessedAt);
+            // Skip hot vectors unless explicitly requested
+            if (tier === 'hot' && !compressHot) {
+                skippedCount++;
+                continue;
+            }
+            // Skip if already compressed at the same or more aggressive tier
+            if (entry.compressed && entry.tier === tier) {
+                skippedCount++;
+                continue;
+            }
+            // Need the original vector to compress. If we already freed it,
+            // we cannot re-compress (the HNSW backend doesn't expose vectors).
+            if (!entry.original) {
+                // If we have an existing compressed form at a less aggressive tier,
+                // we could decompress and recompress, but that adds error.
+                // For now, skip.
+                skippedCount++;
+                continue;
+            }
+            const compressed = this.compressionService.compress(entry.original, tier);
+            const originalSize = entry.original.byteLength;
+            entry.compressed = compressed;
+            entry.tier = tier;
+            // For warm and cold tiers, release the original to save memory.
+            // The HNSW backend still holds its own copy for search.
+            if (tier !== 'hot') {
+                entry.original = null;
+                bytesFreed += originalSize;
+            }
+            compressedCount++;
+            byTier[tier]++;
+        }
+        logger.debug(`compressIndex: compressed=${compressedCount}, skipped=${skippedCount}, freed=${bytesFreed} bytes`);
+        return {
+            compressedCount,
+            skippedCount,
+            bytesFreed,
+            byTier,
+        };
+    }
+    /**
+     * Decompress and retrieve the vector for a given ID.
+     *
+     * Returns the original vector if still available, otherwise
+     * decompresses from the stored compressed form.
+     *
+     * @param id - The vector ID
+     * @returns The decompressed Float32Array, or null if not found
+     */
+    getVector(id) {
+        const entry = this.vectorEntries.get(id);
+        if (!entry)
+            return null;
+        // Return original if still available
+        if (entry.original)
+            return entry.original;
+        // Decompress from compressed form
+        if (entry.compressed) {
+            return this.compressionService.decompress(entry.compressed);
+        }
+        return null;
+    }
+    /**
+     * Get the compression tier for a specific vector.
+     *
+     * @param id - The vector ID
+     * @returns The tier, or null if not found or not compressed
+     */
+    getVectorTier(id) {
+        const entry = this.vectorEntries.get(id);
+        return entry?.tier ?? null;
+    }
+    /**
+     * Update the last-accessed timestamp for a vector.
+     * This affects which tier it falls into during the next compressIndex() call.
+     *
+     * @param id - The vector ID
+     * @param accessDate - The access date (defaults to now)
+     */
+    touchVector(id, accessDate) {
+        const entry = this.vectorEntries.get(id);
+        if (entry) {
+            entry.lastAccessedAt = accessDate ?? new Date();
+        }
+    }
+    // ==========================================================================
+    // Memory Statistics
+    // ==========================================================================
+    /**
+     * Get memory usage statistics for the compressed index.
+     */
+    getMemoryStats() {
+        let uncompressedCount = 0;
+        let compressedCount = 0;
+        let originalBytes = 0;
+        let compressedBytes = 0;
+        const compressedByTier = { hot: 0, warm: 0, cold: 0 };
+        for (const entry of this.vectorEntries.values()) {
+            if (entry.compressed && entry.tier) {
+                compressedCount++;
+                compressedByTier[entry.tier]++;
+                compressedBytes += entry.compressed.compressedByteSize;
+                originalBytes += entry.compressed.originalByteSize;
+            }
+            else if (entry.original) {
+                uncompressedCount++;
+                const byteSize = entry.original.byteLength;
+                originalBytes += byteSize;
+                compressedBytes += byteSize; // Not compressed, so same size
+            }
+        }
+        const bytesSaved = originalBytes - compressedBytes;
+        const savingsPercent = originalBytes > 0
+            ? (bytesSaved / originalBytes) * 100
+            : 0;
+        return {
+            totalVectors: this.vectorEntries.size,
+            uncompressedCount,
+            compressedCount,
+            compressedByTier,
+            originalBytes,
+            compressedBytes,
+            bytesSaved,
+            savingsPercent,
+            compressionEnabled: isTemporalCompressionEnabled(),
+        };
+    }
+    // ==========================================================================
+    // Accessors
+    // ==========================================================================
+    /**
+     * Get the underlying HNSW backend.
+     * Useful for accessing backend-specific methods (e.g., NativeHnswBackend.getMetrics()).
+     */
+    getBackend() {
+        return this.backend;
+    }
+    /**
+     * Get the compression service instance.
+     */
+    getCompressionService() {
+        return this.compressionService;
+    }
+}
+//# sourceMappingURL=compressed-hnsw-integration.js.map

package/dist/integrations/ruvector/dither-adapter.d.ts

@@ -0,0 +1,122 @@
+/**
+ * Deterministic Dithering Adapter for Cross-Platform Reproducibility
+ *
+ * Provides golden-ratio quasi-random dithering for embedding quantization.
+ * Ensures identical quantization results across x86, ARM, and WASM platforms
+ * by using only deterministic integer and float arithmetic.
+ *
+ * The golden ratio sequence produces a low-discrepancy quasi-random sequence
+ * that distributes quantization error more evenly than truncation alone,
+ * improving reconstruction quality at low bit depths.
+ *
+ * @module integrations/ruvector/dither-adapter
+ */
+/**
+ * Result of applying dithered quantization to a vector
+ */
+export interface DitheredResult {
+    /** Quantized values as integers in [0, 2^bitDepth - 1] */
+    quantized: Int32Array;
+    /** Dequantized (reconstructed) float values */
+    dequantized: Float32Array;
+    /** Bit depth used for quantization */
+    bitDepth: number;
+    /** Seed used for dither generation */
+    seed: number;
+    /** Step size (quantization interval width) */
+    stepSize: number;
+    /** Min value of the original vector (used for dequantization) */
+    minValue: number;
+    /** Max value of the original vector (used for dequantization) */
+    maxValue: number;
+}
+/**
+ * Options for the dither adapter
+ */
+export interface DitherOptions {
+    /** Bit depth for quantization (1-32, typical: 3, 5, 7, 8) */
+    bitDepth: number;
+    /** Seed for the dither sequence (default: 0) */
+    seed?: number;
+}
+/**
+ * Check if a native dither module is available.
+ * Always returns false — the TypeScript implementation is production-ready.
+ */
+export declare function isNativeDitherAvailable(): boolean;
+/**
+ * Create a deterministic quasi-random dither sequence using the golden ratio.
+ *
+ * The sequence d_n = frac((seed + (n+1) * phi)) produces values uniformly
+ * distributed in [0, 1) with low discrepancy (no clumping). This is a
+ * Weyl sequence / additive recurrence, which is deterministic and
+ * platform-independent since it uses only standard IEEE 754 arithmetic.
+ *
+ * @param length - Number of dither values to generate
+ * @param seed - Starting offset for the sequence (default: 0)
+ * @returns Float32Array of dither values in [0, 1)
+ */
+export declare function createDitherSequence(length: number, seed?: number): Float32Array;
+/**
+ * Apply deterministic dithered quantization to a float vector.
+ *
+ * The algorithm:
+ * 1. Compute the range [min, max] of the input vector.
+ * 2. Compute step_size = range / (2^bitDepth - 1).
+ * 3. Generate a golden-ratio dither sequence of the same length.
+ * 4. For each element: quantized[i] = clamp(round((value[i] - min) / stepSize + (dither[i] - 0.5)), 0, maxLevel)
+ * 5. Dequantize: dequantized[i] = quantized[i] * stepSize + min
+ *
+ * The dither offset (dither[i] - 0.5) centers the dither noise around zero,
+ * which reduces systematic bias compared to truncation-only quantization.
+ *
+ * @param vector - Input float vector to quantize
+ * @param bitDepth - Number of bits for quantization (1-32)
+ * @param seed - Seed for dither sequence reproducibility (default: 0)
+ * @returns DitheredResult with quantized and reconstructed values
+ */
+export declare function applyDither(vector: Float32Array, bitDepth: number, seed?: number): DitheredResult;
+/**
+ * Apply naive (non-dithered) quantization for comparison purposes.
+ *
+ * Uses simple rounding without dither noise. This tends to produce
+ * systematic quantization artifacts, especially at low bit depths.
+ *
+ * @param vector - Input float vector to quantize
+ * @param bitDepth - Number of bits for quantization (1-32)
+ * @returns DitheredResult with quantized and reconstructed values (seed = -1 to indicate no dither)
+ */
+export declare function applyNaiveQuantization(vector: Float32Array, bitDepth: number): DitheredResult;
+/**
+ * Verify that dithered quantization is deterministic for the given input.
+ *
+ * Runs the quantization twice with the same parameters and compares results.
+ * This guards against any non-determinism from floating point ordering,
+ * platform differences, or hidden state.
+ *
+ * @param vector - Input float vector to test
+ * @param bitDepth - Number of bits for quantization
+ * @returns true if both runs produce identical quantized values
+ */
+export declare function verifyDeterminism(vector: Float32Array, bitDepth: number): boolean;
+/**
+ * Compute mean squared error between original and reconstructed vectors.
+ *
+ * @param original - Original float vector
+ * @param reconstructed - Reconstructed (dequantized) float vector
+ * @returns Mean squared error
+ */
+export declare function computeMSE(original: Float32Array, reconstructed: Float32Array): number;
+/**
+ * Compute signal-to-noise ratio (SNR) in decibels.
+ *
+ * SNR = 10 * log10(signal_power / noise_power)
+ *
+ * Higher SNR means better reconstruction quality.
+ *
+ * @param original - Original float vector
+ * @param reconstructed - Reconstructed (dequantized) float vector
+ * @returns SNR in dB, or Infinity if reconstruction is perfect
+ */
+export declare function computeSNR(original: Float32Array, reconstructed: Float32Array): number;
+//# sourceMappingURL=dither-adapter.d.ts.map