native-vector-store 0.3.9 → 0.4.0

package/deps/parallel_hashmap/phmap_utils.h

@@ -0,0 +1,407 @@
+#if !defined(phmap_utils_h_guard_)
+#define phmap_utils_h_guard_
+
+// ---------------------------------------------------------------------------
+// Copyright (c) 2019, Gregory Popovitch - greg7mdp@gmail.com
+//
+//       minimal header providing phmap::HashState
+//
+//       use as:  phmap::HashState().combine(0, _first_name, _last_name, _age);
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ---------------------------------------------------------------------------
+
+#ifdef _MSC_VER
+    #pragma warning(push)  
+    #pragma warning(disable : 4514) // unreferenced inline function has been removed
+    #pragma warning(disable : 4710) // function not inlined
+    #pragma warning(disable : 4711) // selected for automatic inline expansion
+#endif
+
+#include <cstdint>
+#include <functional>
+#include <tuple>
+#include "phmap_bits.h"
+
+// ---------------------------------------------------------------
+// Absl forward declaration requires global scope.
+// ---------------------------------------------------------------
+#if defined(PHMAP_USE_ABSL_HASH) && !defined(phmap_fwd_decl_h_guard_) && !defined(ABSL_HASH_HASH_H_)
+    namespace absl { template <class T> struct Hash; };
+#endif
+
+namespace phmap
+{
+
+// ---------------------------------------------------------------
+// ---------------------------------------------------------------
+template<int n> 
+struct phmap_mix
+{
+    inline size_t operator()(size_t) const;
+};
+
+template<>
+struct phmap_mix<4>
+{
+    inline size_t operator()(size_t a) const
+    {
+        static constexpr uint64_t kmul = 0xcc9e2d51UL;
+        uint64_t l = a * kmul;
+        return static_cast<size_t>(l ^ (l >> 32));
+    }
+};
+
+#if defined(PHMAP_HAS_UMUL128)
+    template<>
+    struct phmap_mix<8>
+    {
+        // Very fast mixing (similar to Abseil)
+        inline size_t operator()(size_t a) const
+        {
+            static constexpr uint64_t k = 0xde5fb9d2630458e9ULL;
+            uint64_t h;
+            uint64_t l = umul128(a, k, &h);
+            return static_cast<size_t>(h + l);
+        }
+    };
+#else
+    template<>
+    struct phmap_mix<8>
+    {
+        inline size_t operator()(size_t a) const
+        {
+            a = (~a) + (a << 21); // a = (a << 21) - a - 1;
+            a = a ^ (a >> 24);
+            a = (a + (a << 3)) + (a << 8); // a * 265
+            a = a ^ (a >> 14);
+            a = (a + (a << 2)) + (a << 4); // a * 21
+            a = a ^ (a >> 28);
+            a = a + (a << 31);
+            return static_cast<size_t>(a);
+        }
+    };
+#endif
+
+// --------------------------------------------
+template<int n> 
+struct fold_if_needed
+{
+    inline size_t operator()(uint64_t) const;
+};
+
+template<>
+struct fold_if_needed<4>
+{
+    inline size_t operator()(uint64_t a) const
+    {
+        return static_cast<size_t>(a ^ (a >> 32));
+    }
+};
+
+template<>
+struct fold_if_needed<8>
+{
+    inline size_t operator()(uint64_t a) const
+    {
+        return static_cast<size_t>(a);
+    }
+};
+
+// ---------------------------------------------------------------
+// see if class T has a hash_value() friend method
+// ---------------------------------------------------------------
+template<typename T>
+struct has_hash_value
+{
+private:
+    typedef std::true_type yes;
+    typedef std::false_type no;
+
+    template<typename U> static auto test(int) -> decltype(hash_value(std::declval<const U&>()) == 1, yes());
+
+    template<typename> static no test(...);
+
+public:
+    static constexpr bool value = std::is_same<decltype(test<T>(0)), yes>::value;
+};
+
+#if defined(PHMAP_USE_ABSL_HASH) && !defined(phmap_fwd_decl_h_guard_)
+    template <class T> using Hash = ::absl::Hash<T>;
+#elif !defined(PHMAP_USE_ABSL_HASH)
+// ---------------------------------------------------------------
+//               phmap::Hash
+// ---------------------------------------------------------------
+template <class T>
+struct Hash
+{
+    template <class U, typename std::enable_if<has_hash_value<U>::value, int>::type = 0>
+    size_t _hash(const T& val) const
+    {
+        return hash_value(val);
+    }
+ 
+    template <class U, typename std::enable_if<!has_hash_value<U>::value, int>::type = 0>
+    size_t _hash(const T& val) const
+    {
+        return std::hash<T>()(val);
+    }
+ 
+    inline size_t operator()(const T& val) const
+    {
+        return _hash<T>(val);
+    }
+};
+ 
+template<class ArgumentType, class ResultType>
+struct phmap_unary_function
+{
+    typedef ArgumentType argument_type;
+    typedef ResultType result_type;
+};
+
+template <>
+struct Hash<bool> : public phmap_unary_function<bool, size_t>
+{
+    inline size_t operator()(bool val) const noexcept
+    { return static_cast<size_t>(val); }
+};
+
+template <>
+struct Hash<char> : public phmap_unary_function<char, size_t>
+{
+    inline size_t operator()(char val) const noexcept
+    { return static_cast<size_t>(val); }
+};
+
+template <>
+struct Hash<signed char> : public phmap_unary_function<signed char, size_t>
+{
+    inline size_t operator()(signed char val) const noexcept
+    { return static_cast<size_t>(val); }
+};
+
+template <>
+struct Hash<unsigned char> : public phmap_unary_function<unsigned char, size_t>
+{
+    inline size_t operator()(unsigned char val) const noexcept
+    { return static_cast<size_t>(val); }
+};
+
+#ifdef PHMAP_HAS_NATIVE_WCHAR_T
+template <>
+struct Hash<wchar_t> : public phmap_unary_function<wchar_t, size_t>
+{
+    inline size_t operator()(wchar_t val) const noexcept
+    { return static_cast<size_t>(val); }
+};
+#endif
+
+template <>
+struct Hash<int16_t> : public phmap_unary_function<int16_t, size_t>
+{
+    inline size_t operator()(int16_t val) const noexcept
+    { return static_cast<size_t>(val); }
+};
+
+template <>
+struct Hash<uint16_t> : public phmap_unary_function<uint16_t, size_t>
+{
+    inline size_t operator()(uint16_t val) const noexcept
+    { return static_cast<size_t>(val); }
+};
+
+template <>
+struct Hash<int32_t> : public phmap_unary_function<int32_t, size_t>
+{
+    inline size_t operator()(int32_t val) const noexcept
+    { return static_cast<size_t>(val); }
+};
+
+template <>
+struct Hash<uint32_t> : public phmap_unary_function<uint32_t, size_t>
+{
+    inline size_t operator()(uint32_t val) const noexcept
+    { return static_cast<size_t>(val); }
+};
+
+template <>
+struct Hash<int64_t> : public phmap_unary_function<int64_t, size_t>
+{
+    inline size_t operator()(int64_t val) const noexcept
+    { return fold_if_needed<sizeof(size_t)>()(static_cast<uint64_t>(val)); }
+};
+
+template <>
+struct Hash<uint64_t> : public phmap_unary_function<uint64_t, size_t>
+{
+    inline size_t operator()(uint64_t val) const noexcept
+    { return fold_if_needed<sizeof(size_t)>()(val); }
+};
+
+template <>
+struct Hash<float> : public phmap_unary_function<float, size_t>
+{
+    inline size_t operator()(float val) const noexcept
+    {
+        // -0.0 and 0.0 should return same hash
+        uint32_t *as_int = reinterpret_cast<uint32_t *>(&val);
+        return (val == 0) ? static_cast<size_t>(0) : 
+                            static_cast<size_t>(*as_int);
+    }
+};
+
+template <>
+struct Hash<double> : public phmap_unary_function<double, size_t>
+{
+    inline size_t operator()(double val) const noexcept
+    {
+        // -0.0 and 0.0 should return same hash
+        uint64_t *as_int = reinterpret_cast<uint64_t *>(&val);
+        return (val == 0) ? static_cast<size_t>(0) : 
+                            fold_if_needed<sizeof(size_t)>()(*as_int);
+    }
+};
+
+#endif
+
+#if defined(_MSC_VER)
+#   define PHMAP_HASH_ROTL32(x, r) _rotl(x,r)
+#else
+#   define PHMAP_HASH_ROTL32(x, r) (x << r) | (x >> (32 - r))
+#endif
+
+
+template <class H, int sz> struct Combiner
+{
+    H operator()(H seed, size_t value);
+};
+
+template <class H> struct Combiner<H, 4>
+{
+    H operator()(H h1, size_t k1)
+    {
+        // Copyright 2005-2014 Daniel James.
+        // Distributed under the Boost Software License, Version 1.0. (See accompanying
+        // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+        
+        const uint32_t c1 = 0xcc9e2d51;
+        const uint32_t c2 = 0x1b873593;
+
+        k1 *= c1;
+        k1 = PHMAP_HASH_ROTL32(k1,15);
+        k1 *= c2;
+
+        h1 ^= k1;
+        h1 = PHMAP_HASH_ROTL32(h1,13);
+        h1 = h1*5+0xe6546b64;
+
+        return h1;
+    }
+};
+
+template <class H> struct Combiner<H, 8>
+{
+    H operator()(H h, size_t k)
+    {
+        // Copyright 2005-2014 Daniel James.
+        // Distributed under the Boost Software License, Version 1.0. (See accompanying
+        // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+        const uint64_t m = (uint64_t(0xc6a4a793) << 32) + 0x5bd1e995;
+        const int r = 47;
+
+        k *= m;
+        k ^= k >> r;
+        k *= m;
+
+        h ^= k;
+        h *= m;
+
+        // Completely arbitrary number, to prevent 0's
+        // from hashing to 0.
+        h += 0xe6546b64;
+
+        return h;
+    }
+};
+
+// define HashState to combine member hashes... see example below
+// -----------------------------------------------------------------------------
+template <typename H>
+class HashStateBase {
+public:
+    template <typename T, typename... Ts>
+    static H combine(H state, const T& value, const Ts&... values);
+
+    static H combine(H state) { return state; }
+};
+
+template <typename H>
+template <typename T, typename... Ts>
+H HashStateBase<H>::combine(H seed, const T& v, const Ts&... vs)
+{
+    return HashStateBase<H>::combine(Combiner<H, sizeof(H)>()(
+                                         seed, phmap::Hash<T>()(v)), 
+                                     vs...);
+}
+
+using HashState = HashStateBase<size_t>;
+
+// -----------------------------------------------------------------------------
+
+#if !defined(PHMAP_USE_ABSL_HASH)
+
+// define Hash for std::pair
+// -------------------------
+template<class T1, class T2> 
+struct Hash<std::pair<T1, T2>> {
+    size_t operator()(std::pair<T1, T2> const& p) const noexcept {
+        return phmap::HashState().combine(phmap::Hash<T1>()(p.first), p.second);
+    }
+};
+
+// define Hash for std::tuple
+// --------------------------
+template<class... T> 
+struct Hash<std::tuple<T...>> {
+    size_t operator()(std::tuple<T...> const& t) const noexcept {
+        size_t seed = 0;
+        return _hash_helper(seed, t);
+    }
+
+private:
+    template<size_t I = 0, class TUP>
+    typename std::enable_if<I == std::tuple_size<TUP>::value, size_t>::type
+    _hash_helper(size_t seed, const TUP &) const noexcept { return seed; }
+
+    template<size_t I = 0, class TUP>
+    typename std::enable_if<I < std::tuple_size<TUP>::value, size_t>::type
+    _hash_helper(size_t seed, const TUP &t) const noexcept {
+        const auto &el = std::get<I>(t);
+        using el_type = typename std::remove_cv<typename std::remove_reference<decltype(el)>::type>::type;
+        seed = Combiner<size_t, sizeof(size_t)>()(seed, phmap::Hash<el_type>()(el));
+        return _hash_helper<I + 1>(seed, t);
+    }
+};
+
+
+#endif
+
+
+}  // namespace phmap
+
+#ifdef _MSC_VER
+     #pragma warning(pop)  
+#endif
+
+#endif // phmap_utils_h_guard_

package/docs/index.html

@@ -60,8 +60,9 @@
 <li><strong>🚀 High Performance</strong>: C++ implementation with OpenMP SIMD optimization</li>
 <li><strong>📦 Arena Allocation</strong>: Memory-efficient storage with 64MB chunks</li>
 <li><strong>⚡ Fast Search</strong>: Sub-10ms similarity search for large document collections</li>
+<li><strong>🔍 Hybrid Search</strong>: Combines vector similarity (semantic) with BM25 text search (lexical)</li>
 <li><strong>🔧 MCP Integration</strong>: Built for Model Context Protocol servers</li>
-<li><strong>🌐 Cross-Platform</strong>: Works on Linux, macOS, and Windows</li>
+<li><strong>🌐 Cross-Platform</strong>: Works on Linux and macOS (Windows users: use WSL)</li>
 <li><strong>📊 TypeScript Support</strong>: Full type definitions included</li>
 <li><strong>🔄 Producer-Consumer Loading</strong>: Parallel document loading at 178k+ docs/sec</li>
 </ul>
@@ -85,7 +86,7 @@
 <li><strong>Linux</strong>: <code>sudo apt-get install libgomp1</code> (Ubuntu/Debian) or <code>dnf install libgomp</code> (Fedora)</li>
 <li><strong>Alpine</strong>: <code>apk add libgomp</code></li>
 <li><strong>macOS</strong>: <code>brew install libomp</code></li>
-<li><strong>Windows</strong>: Included with Visual C++ runtime</li>
+<li><strong>Windows</strong>: Use WSL (Windows Subsystem for Linux)</li>
 </ul>
 </li>
 </ul>
@@ -93,7 +94,6 @@
 <ul>
 <li>Linux (x64, arm64, musl/Alpine) - x64 builds are AWS Lambda compatible (no AVX-512)</li>
 <li>macOS (x64, arm64/Apple Silicon)</li>
-<li>Windows (x64)</li>
 </ul>
 <p>If building from source, you'll need:</p>
 <ul>
@@ -125,8 +125,16 @@ store.finalize(); // Must call before searching!
 
 // Search for similar documents
 const queryEmbedding = new Float32Array(1536);
+
+// Option 1: Vector-only search (traditional)
 const results = store.search(queryEmbedding, 5); // Top 5 results
 
+// Option 2: Hybrid search (NEW - combines vector + BM25 text search)
+const hybridResults = store.search(queryEmbedding, 5, &quot;your search query text&quot;);
+
+// Option 3: BM25 text-only search
+const textResults = store.searchBM25(&quot;your search query&quot;, 5);
+
 // Results format - array of SearchResult objects, sorted by score (highest first):
 console.log(results);
 // [
@@ -284,6 +292,47 @@ if (process.env.NODE_ENV === 'development') {
   fs.watch('./documents', { recursive: true }, reloadStore);
 }
 </code></pre>
+<h2 id="hybrid-search">Hybrid Search</h2>
+<p>The vector store now supports hybrid search, combining semantic similarity (vector search) with lexical matching (BM25 text search) for improved retrieval accuracy:</p>
+<pre class="prettyprint source lang-javascript"><code>const { VectorStore } = require('native-vector-store');
+
+const store = new VectorStore(1536);
+store.loadDir('./documents');
+
+// Hybrid search automatically combines vector and text search
+const queryEmbedding = new Float32Array(1536);
+const results = store.search(
+  queryEmbedding, 
+  10,                               // Top 10 results
+  &quot;machine learning algorithms&quot;    // Query text for BM25
+);
+
+// You can also use individual search methods
+const vectorResults = store.searchVector(queryEmbedding, 10);
+const textResults = store.searchBM25(&quot;machine learning&quot;, 10);
+
+// Or explicitly control the hybrid weights
+const customResults = store.searchHybrid(
+  queryEmbedding,
+  &quot;machine learning&quot;,
+  10,
+  0.3,  // Vector weight (30%)
+  0.7   // BM25 weight (70%)
+);
+
+// Tune BM25 parameters for your corpus
+store.setBM25Parameters(
+  1.2,  // k1: Term frequency saturation (default: 1.2)
+  0.75, // b: Document length normalization (default: 0.75)
+  1.0   // delta: Smoothing parameter (default: 1.0)
+);
+</code></pre>
+<p>Hybrid search is particularly effective for:</p>
+<ul>
+<li><strong>Question answering</strong>: BM25 finds documents with exact terms while vectors capture semantic meaning</li>
+<li><strong>Knowledge retrieval</strong>: Combines conceptual similarity with keyword matching</li>
+<li><strong>Multi-lingual search</strong>: Vectors handle cross-language similarity while BM25 matches exact terms</li>
+</ul>
 <h2 id="mcp-server-integration">MCP Server Integration</h2>
 <p>Perfect for building local RAG capabilities in MCP servers:</p>
 <pre class="prettyprint source lang-javascript"><code>const { MCPVectorServer } = require('native-vector-store/examples/mcp-server');

package/lib/index.d.ts

@@ -31,11 +31,45 @@ export class VectorStore {
   
   /**
    * Search for k most similar documents
+   * Uses hybrid search if queryText is provided, otherwise vector-only search
+   * @param query - Query embedding vector
+   * @param k - Number of results to return
+   * @param queryText - Optional text query for hybrid search (BM25 + vector)
+   */
+  search(query: Float32Array, k: number, queryText?: string): SearchResult[];
+  
+  /**
+   * Pure vector similarity search
    * @param query - Query embedding vector
    * @param k - Number of results to return
    * @param normalizeQuery - Whether to L2 normalize the query (default: true)
    */
-  search(query: Float32Array, k: number, normalizeQuery?: boolean): SearchResult[];
+  searchVector(query: Float32Array, k: number, normalizeQuery?: boolean): SearchResult[];
+  
+  /**
+   * Pure BM25 text search
+   * @param queryText - Text query or array of query terms
+   * @param k - Number of results to return
+   */
+  searchBM25(queryText: string | string[], k: number): SearchResult[];
+  
+  /**
+   * Hybrid search combining vector similarity and BM25 text search
+   * @param query - Query embedding vector
+   * @param queryText - Text query for BM25 component
+   * @param k - Number of results to return
+   * @param vectorWeight - Weight for vector similarity (default: 0.5)
+   * @param bm25Weight - Weight for BM25 score (default: 0.5)
+   */
+  searchHybrid(query: Float32Array, queryText: string, k: number, vectorWeight?: number, bm25Weight?: number): SearchResult[];
+  
+  /**
+   * Set BM25 parameters for text search
+   * @param k1 - Controls term frequency saturation (default: 1.2)
+   * @param b - Controls document length normalization (default: 0.75)
+   * @param delta - Smoothing parameter (default: 1.0)
+   */
+  setBM25Parameters(k1: number, b: number, delta?: number): void;
   
   /**
    * Normalize all stored embeddings

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "native-vector-store",
-  "version": "0.3.9",
+  "version": "0.4.0",
   "description": "High-performance local vector store with SIMD optimization for MCP servers",
   "main": "index.js",
   "types": "lib/index.d.ts",

package/src/Makefile

@@ -14,7 +14,7 @@ CXXFLAGS = -std=c++17 -g -O0 -fno-omit-frame-pointer -DDEBUG
 # OS-specific configuration
 ifeq ($(UNAME_S),Darwin)
     # macOS configuration
-    INCLUDES = -I. -I../deps/simdjson -I../deps/atomic_queue -I/opt/homebrew/opt/libomp/include
+    INCLUDES = -I. -I../deps/simdjson -I../deps/atomic_queue -I../deps -I/opt/homebrew/opt/libomp/include
     LDFLAGS = -L/opt/homebrew/opt/libomp/lib
     LIBS = -lomp
     # Add flags for better debugging with lldb
@@ -23,7 +23,7 @@ ifeq ($(UNAME_S),Darwin)
     CXXFLAGS += -Xpreprocessor -fopenmp
 else
     # Linux configuration
-    INCLUDES = -I. -I../deps/simdjson -I../deps/atomic_queue
+    INCLUDES = -I. -I../deps/simdjson -I../deps/atomic_queue -I../deps
     LDFLAGS = -L/usr/lib
     LIBS = -lgomp
     # OpenMP flags for Linux
@@ -32,12 +32,18 @@ endif
 
 TARGET = test_vector_store
 STRESS_TARGET = test_stress
-SOURCES = test_main.cpp vector_store.cpp ../deps/simdjson/simdjson.cpp
-STRESS_SOURCES = test_stress.cpp vector_store.cpp vector_store_loader.cpp vector_store_loader_mmap.cpp vector_store_loader_adaptive.cpp ../deps/simdjson/simdjson.cpp
+BM25_TARGET = test_bm25
+HYBRID_TARGET = test_hybrid_search
+SOURCES = test_main.cpp vector_store.cpp simple_tokenizer.cpp ../deps/simdjson/simdjson.cpp
+STRESS_SOURCES = test_stress.cpp vector_store.cpp simple_tokenizer.cpp vector_store_loader.cpp vector_store_loader_mmap.cpp vector_store_loader_adaptive.cpp ../deps/simdjson/simdjson.cpp
+BM25_SOURCES = test_bm25.cpp vector_store.cpp simple_tokenizer.cpp ../deps/simdjson/simdjson.cpp
+HYBRID_SOURCES = test_hybrid_search.cpp vector_store.cpp simple_tokenizer.cpp ../deps/simdjson/simdjson.cpp
 OBJECTS = $(SOURCES:.cpp=.o)
 STRESS_OBJECTS = $(STRESS_SOURCES:.cpp=.o)
+BM25_OBJECTS = $(BM25_SOURCES:.cpp=.o)
+HYBRID_OBJECTS = $(HYBRID_SOURCES:.cpp=.o)
 
-all: $(TARGET) $(STRESS_TARGET)
+all: $(TARGET) $(STRESS_TARGET) $(BM25_TARGET) $(HYBRID_TARGET)
 
 $(TARGET): $(OBJECTS)
 	$(CXX) $(CXXFLAGS) $(OBJECTS) -o $(TARGET) $(LDFLAGS) $(LIBS)
@@ -45,11 +51,25 @@ $(TARGET): $(OBJECTS)
 $(STRESS_TARGET): $(STRESS_OBJECTS)
 	$(CXX) $(CXXFLAGS) $(STRESS_OBJECTS) -o $(STRESS_TARGET) $(LDFLAGS) $(LIBS)
 
+$(BM25_TARGET): $(BM25_OBJECTS)
+	$(CXX) $(CXXFLAGS) $(BM25_OBJECTS) -o $(BM25_TARGET) $(LDFLAGS) $(LIBS)
+
+$(HYBRID_TARGET): $(HYBRID_OBJECTS)
+	$(CXX) $(CXXFLAGS) $(HYBRID_OBJECTS) -o $(HYBRID_TARGET) $(LDFLAGS) $(LIBS)
+
 %.o: %.cpp
 	$(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@
 
 clean:
-	rm -f $(OBJECTS) $(STRESS_OBJECTS) $(TARGET) $(STRESS_TARGET)
+	rm -f $(OBJECTS) $(STRESS_OBJECTS) $(BM25_OBJECTS) $(HYBRID_OBJECTS) $(TARGET) $(STRESS_TARGET) $(BM25_TARGET) $(HYBRID_TARGET)
+
+# BM25 test
+bm25: $(BM25_TARGET)
+	./$(BM25_TARGET)
+
+# Hybrid search test
+hybrid: $(HYBRID_TARGET)
+	./$(HYBRID_TARGET)
 
 # Force rebuild (useful for CI)
 rebuild: clean all