agentic-qe 3.9.5 → 3.9.6

package/dist/integrations/ruvector/feature-flags.js

@@ -14,14 +14,27 @@
 // Default Configuration
 // ============================================================================
 /**
- * Default feature flags - all enabled by default
+ * Default feature flags — all enabled by default.
+ *
+ * useNativeHNSW history:
+ *   - v3.9.5: flipped to false after the @ruvector/router native binding
+ *     was found to deadlock on certain inputs (futex contention inside the
+ *     Rust-side HNSW). The JS ProgressiveHnswBackend was the safe default.
+ *   - issue #399 (April 2026): empirical verification revealed four bugs in
+ *     @ruvector/router 0.1.28 — broken HNSW search (recall ~0% on textbook
+ *     fixtures), CWD pollution via vectors.db, process-wide singleton lock,
+ *     and unreleased file lock on dispose. NativeHnswBackend was rewritten
+ *     to wrap hnswlib-node (the canonical Hnswlib reference implementation
+ *     by Yury Malkov) which fixes all four bugs and gives users with large
+ *     codebases real sublinear HNSW search. Default flipped back to true.
+ *     See ADR-090.
  */
 const DEFAULT_FEATURE_FLAGS = {
     useQESONA: true,
     useQEFlashAttention: true,
     useQEGNNIndex: true,
     logMigrationMetrics: true,
-    useNativeHNSW: false,
+    useNativeHNSW: true,
     useTemporalCompression: true,
     useMetadataFiltering: true,
     useDeterministicDither: true,

package/dist/kernel/hnsw-adapter.js

@@ -199,7 +199,7 @@ export class HnswAdapter {
         if (this.backend instanceof ProgressiveHnswBackend) {
             return this.backend.isRuvectorAvailable();
         }
-        // NativeHnswBackend uses @ruvector/router VectorDb
+        // NativeHnswBackend wraps hnswlib-node HierarchicalNSW (post #399 / ADR-090)
         if (this.backend instanceof NativeHnswBackend) {
             return this.backend.isNativeAvailable();
         }

package/dist/kernel/native-hnsw-backend.d.ts

@@ -1,22 +1,49 @@
 /**
- * Native HNSW Backend via @ruvector/router VectorDb
+ * Native HNSW Backend via hnswlib-node
  *
- * Provides a high-performance HNSW index backed by the Rust-based
- * @ruvector/router library. This backend implements the same
- * IHnswIndexProvider interface as ProgressiveHnswBackend but delegates
- * vector storage and search to native code for lower latency and
- * higher throughput.
+ * Provides a high-performance HNSW index backed by hnswlib-node, the
+ * canonical Node.js binding for the C++ Hnswlib reference implementation
+ * (Yury Malkov's HNSW library, the same code Pinecone, Weaviate, Qdrant
+ * and chromadb-default-embed are built on top of).
  *
- * When @ruvector/router is unavailable (e.g., unsupported platform,
- * missing binary), construction throws NativeHnswUnavailableError so the
- * caller (HnswAdapter factory) can fall back to the JS backend.
+ * This backend implements the same IHnswIndexProvider interface as
+ * ProgressiveHnswBackend but delegates vector storage and search to
+ * native code for sublinear search latency on large indexes.
  *
- * @see ADR-081: Native HNSW NAPI Integration
+ * History — issue #399 (April 2026):
+ *   This backend previously wrapped @ruvector/router's VectorDb. Empirical
+ *   verification (scripts/diagnose-issue-399*.mjs) found four serious bugs
+ *   in @ruvector/router 0.1.28:
+ *
+ *   1. HNSW search returned essentially random results — recall@10 ≈ 0%
+ *      on textbook unit-Gaussian random vectors at default M/efC/efS,
+ *      could not find self-vectors. Pumping efSearch to N≈index-size
+ *      restored correctness but defeated HNSW's purpose.
+ *   2. The VectorDb constructor unconditionally created a `vectors.db`
+ *      redb file in the current working directory (NOT in .agentic-qe/),
+ *      violating the unified memory architecture and polluting users'
+ *      project roots with multi-MB persistence files they never asked for.
+ *   3. The redb file held a process-wide exclusive lock — only ONE
+ *      VectorDb instance could exist per process. Subsequent constructors
+ *      threw "Database already open. Cannot acquire lock."
+ *   4. NAPI dispose did not synchronously release the redb lock, so the
+ *      lock outlived our dispose() call and caused the v3.9.5 futex
+ *      deadlock when the indexer tried to recreate after reset.
+ *
+ *   Comparison test on 1000 vector self-query (linux-arm64, M=16, efC=200,
+ *   efS=100, cosine):
+ *     - @ruvector/router 0.1.28: recall@10 = 10%, top-1 = wrong vector
+ *     - hnswlib-node 3.0.0:      recall@10 = 100%, top-1 = id 42 ✓
+ *
+ *   The migration to hnswlib-node fixes all four bugs in one swap.
+ *
+ * @see ADR-090: hnswlib-node migration
+ * @see https://github.com/proffesor-for-testing/agentic-qe/issues/399
  * @module kernel/native-hnsw-backend
  */
 import type { IHnswIndexProvider, SearchResult, HnswConfig } from './hnsw-index-provider.js';
 /**
- * Thrown when @ruvector/router native binary is not available.
+ * Thrown when hnswlib-node native binary is not available.
  * The HnswAdapter factory catches this to fall back to the JS backend.
  */
 export declare class NativeHnswUnavailableError extends Error {
@@ -31,6 +58,17 @@ export declare function resetNativeModuleLoader(): void;
  * Check if the native module is available without throwing.
  */
 export declare function isNativeModuleAvailable(): boolean;
+/**
+ * Performance metrics for a NativeHnswBackend instance.
+ *
+ * Note on legacy fields: `fallbackSearchCount`, `bruteForceSearchCount`,
+ * `fallbackRate`, and `allSearchesBruteForce` are retained for backward
+ * compatibility with consumers that referenced them during the
+ * @ruvector/router era. Under hnswlib-node there is no native→brute-force
+ * fallback path (the library always returns correct results), so these
+ * fields will always be 0/false. They will be removed in a future major
+ * version.
+ */
 export interface NativeHnswMetrics {
     totalSearches: number;
     totalAdds: number;
@@ -38,40 +76,61 @@ export interface NativeHnswMetrics {
     avgSearchLatencyMs: number;
     maxSearchLatencyMs: number;
     lastSearchLatencyMs: number;
+    /** Always 0 under hnswlib-node — kept for backward compatibility. */
     fallbackSearchCount: number;
+    /** Always 0 under hnswlib-node — kept for backward compatibility. */
     bruteForceSearchCount: number;
+    /** Equals totalSearches under hnswlib-node (every search is native). */
     nativeSearchCount: number;
+    /** Always 0 under hnswlib-node. */
     fallbackRate: number;
+    /** Always false under hnswlib-node. */
     allSearchesBruteForce: boolean;
 }
 /**
- * Native HNSW backend using @ruvector/router VectorDb.
+ * Native HNSW backend using hnswlib-node HierarchicalNSW.
  *
  * Provides the same IHnswIndexProvider interface as ProgressiveHnswBackend
- * but delegates all vector operations to a Rust-based HNSW implementation
- * for improved performance.
+ * but delegates all vector operations to the C++ Hnswlib reference
+ * implementation for sublinear search latency at scale.
  *
- * Note: @ruvector/router VectorDb uses string IDs while IHnswIndexProvider
- * uses numeric IDs. Conversion is done via String(id) and Number(id).
- * VectorDb search returns distance scores (lower = closer) which are
- * converted to similarity scores (higher = closer) for consistency.
+ * Key differences from the previous @ruvector/router-backed implementation:
+ *   - No local vectorStore mirror — hnswlib-node returns correct distances
+ *     directly, so re-scoring is unnecessary.
+ *   - No process-wide singleton lock — multiple instances coexist.
+ *   - No vectors.db pollution — persistence is opt-in via writeIndex().
+ *   - resizeIndex() doubling on overflow — grows past initial maxElements.
  */
 export declare class NativeHnswBackend implements IHnswIndexProvider {
     private readonly config;
-    private nativeDb;
+    private nativeIndex;
+    private currentMaxElements;
+    /** Tracks live ids so size() and remove() can correctly distinguish soft-deleted slots. */
+    private readonly liveIds;
+    /** Optional metadata mirror — only stored when callers attach metadata. */
     private readonly metadataStore;
-    private readonly vectorStore;
-    private readonly normStore;
-    private operationLock;
-    private highFallbackWarningEmitted;
     private _metrics;
     /**
      * Create a NativeHnswBackend.
      *
      * @param config - HNSW configuration overrides
-     * @throws {NativeHnswUnavailableError} If @ruvector/router is not available
+     * @throws {NativeHnswUnavailableError} If hnswlib-node is not available
      */
     constructor(config?: Partial<HnswConfig>);
+    /**
+     * Build a fresh HierarchicalNSW with the configured parameters.
+     *
+     * Used both at construction time and on `clear()` to guarantee a clean
+     * graph. hnswlib-node's `markDelete` is a soft tombstone — it leaves the
+     * graph node in place and only hides it from search. Across many cycles
+     * of `clear() → re-add` (which happens during long test runs that share
+     * the singleton HnswAdapter registry), the tombstoned slots can interact
+     * pathologically with `addPoint(label, replaceDeleted=true)` and produce
+     * duplicate-label results from `searchKnn`. Recreating the underlying
+     * index on `clear()` is the only way to guarantee O(1) clean state
+     * regardless of how the wrapper is used over time.
+     */
+    private createFreshIndex;
     add(id: number, vector: Float32Array, metadata?: Record<string, unknown>): void;
     search(query: Float32Array, k: number): SearchResult[];
     remove(id: number): boolean;
@@ -92,18 +151,27 @@ export declare class NativeHnswBackend implements IHnswIndexProvider {
     getConfig(): Readonly<HnswConfig>;
     /**
      * Clear all vectors from the index.
-     * Deletes all entries from the native VectorDb.
+     *
+     * Recreates the underlying HierarchicalNSW from scratch rather than
+     * `markDelete`-ing each label. The tombstone-only approach leaks state
+     * across `clear() → re-add` cycles (the C++ graph keeps every deleted
+     * slot, and `addPoint(label, replaceDeleted=true)` can produce
+     * duplicate-label results from `searchKnn` after many cycles). A fresh
+     * index is the only guarantee of O(1) clean state and is what callers
+     * who use `clear()` actually expect.
      */
     clear(): void;
     /**
-     * Dispose of the native VectorDb handle so NAPI garbage collection can
-     * reclaim the underlying Rust-side index. After dispose(), this backend
-     * instance must not be used. The HnswAdapter registry is expected to
-     * drop its reference as part of `HnswAdapter.close(name)`.
+     * Dispose of the native index handle.
      *
-     * This is the load-bearing fix for the v3.9.1 regression where
-     * `resetUnifiedMemory()` would null the UnifiedMemoryManager but leave
-     * a stale NativeHnswBackend alive in the HnswAdapter registry.
+     * hnswlib-node holds the C++ index in JS-managed memory; setting the
+     * reference to null lets V8 GC reclaim it on the next collection cycle.
+     * Unlike the previous @ruvector/router VectorDb, there is no file lock
+     * to release and no synchronous teardown required.
+     *
+     * After dispose(), this backend instance must not be used. The
+     * HnswAdapter registry is expected to drop its reference as part of
+     * `HnswAdapter.close(name)`.
      */
     dispose(): void;
     /**
@@ -112,10 +180,5 @@ export declare class NativeHnswBackend implements IHnswIndexProvider {
     isNativeAvailable(): boolean;
     private normalizeVector;
     private updateSearchMetrics;
-    /**
-     * Brute-force fallback search over locally stored vectors.
-     * Used when native search throws an unexpected error.
-     */
-    private bruteForceSearch;
 }
 //# sourceMappingURL=native-hnsw-backend.d.ts.map