native-vector-store 0.1.0 → 0.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "native-vector-store",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "High-performance local vector store with SIMD optimization for MCP servers",
   "main": "index.js",
   "types": "lib/index.d.ts",
@@ -36,17 +36,36 @@
   "dependencies": {
     "dotenv": "^17.2.0",
     "node-addon-api": "^7.0.0",
-    "node-gyp-build": "^4.8.0",
+    "node-gyp-build": "^4.8.4",
     "openai": "^5.8.3"
   },
+  "optionalDependencies": {
+    "@modelcontextprotocol/sdk": "^1.15.1"
+  },
   "devDependencies": {
     "node-gyp": "^10.0.0",
-    "prebuildify": "^6.0.0"
+    "prebuildify": "^6.0.0",
+    "prebuildify-cross": "^5.0.0"
   },
-  "os": ["darwin", "linux", "win32"],
-  "cpu": ["x64", "arm64"],
+  "os": [
+    "darwin",
+    "linux",
+    "win32"
+  ],
+  "cpu": [
+    "x64",
+    "arm64"
+  ],
   "engines": {
     "node": ">=14.0.0"
   },
-  "gypfile": true
+  "gypfile": true,
+  "files": [
+    "index.js",
+    "lib/",
+    "src/",
+    "deps/",
+    "binding.gyp",
+    "prebuilds/"
+  ]
 }

package/src/Makefile

@@ -0,0 +1,87 @@
+# Detect OS
+UNAME_S := $(shell uname -s)
+
+# Compiler
+ifeq ($(UNAME_S),Darwin)
+    CXX = clang++
+else
+    CXX = g++
+endif
+
+# Base flags
+CXXFLAGS = -std=c++17 -g -O0 -fno-omit-frame-pointer -DDEBUG
+
+# OS-specific configuration
+ifeq ($(UNAME_S),Darwin)
+    # macOS configuration
+    INCLUDES = -I. -I../deps -I/opt/homebrew/opt/libomp/include
+    LDFLAGS = -L/opt/homebrew/opt/libomp/lib
+    LIBS = -lomp
+    # Add flags for better debugging with lldb
+    CXXFLAGS += -glldb -gdwarf-4
+    # OpenMP flags for macOS
+    CXXFLAGS += -Xpreprocessor -fopenmp
+else
+    # Linux configuration
+    INCLUDES = -I. -I../deps
+    LDFLAGS = -L/usr/lib
+    LIBS = -lgomp
+    # OpenMP flags for Linux
+    CXXFLAGS += -fopenmp
+endif
+
+TARGET = test_vector_store
+STRESS_TARGET = test_stress
+SOURCES = test_main.cpp vector_store.cpp ../deps/simdjson.cpp
+STRESS_SOURCES = test_stress.cpp vector_store.cpp vector_store_loader.cpp vector_store_loader_mmap.cpp vector_store_loader_adaptive.cpp ../deps/simdjson.cpp
+OBJECTS = $(SOURCES:.cpp=.o)
+STRESS_OBJECTS = $(STRESS_SOURCES:.cpp=.o)
+
+all: $(TARGET) $(STRESS_TARGET)
+
+$(TARGET): $(OBJECTS)
+	$(CXX) $(CXXFLAGS) $(OBJECTS) -o $(TARGET) $(LDFLAGS) $(LIBS)
+
+$(STRESS_TARGET): $(STRESS_OBJECTS)
+	$(CXX) $(CXXFLAGS) $(STRESS_OBJECTS) -o $(STRESS_TARGET) $(LDFLAGS) $(LIBS)
+
+%.o: %.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@
+
+clean:
+	rm -f $(OBJECTS) $(STRESS_OBJECTS) $(TARGET) $(STRESS_TARGET)
+
+# Force rebuild (useful for CI)
+rebuild: clean all
+
+run: $(TARGET)
+	./$(TARGET) ../test
+
+debug: $(TARGET)
+	lldb ./$(TARGET)
+
+# Sanitizer builds
+test-asan: CXXFLAGS += -fsanitize=address -fno-sanitize-recover=all
+test-asan: LDFLAGS += -fsanitize=address
+test-asan: clean $(STRESS_TARGET)
+	./$(STRESS_TARGET)
+
+test-tsan: CXXFLAGS += -fsanitize=thread -fno-sanitize-recover=all
+test-tsan: LDFLAGS += -fsanitize=thread
+test-tsan: clean $(STRESS_TARGET)
+	./$(STRESS_TARGET)
+
+# Run stress tests with sanitizers by default
+# Use SANITIZER=none to disable, SANITIZER=thread for TSan
+SANITIZER ?= address
+
+ifeq ($(SANITIZER),address)
+stress: test-asan
+else ifeq ($(SANITIZER),thread)
+stress: test-tsan
+else
+stress: $(STRESS_TARGET)
+	./$(STRESS_TARGET)
+endif
+
+.PHONY: all clean run debug test-asan test-tsan stress

package/src/test_main.cpp

@@ -0,0 +1,173 @@
+#include "vector_store.h"
+#include <iostream>
+#include <filesystem>
+#include <vector>
+#include <simdjson.h>
+#include <cctype>
+
+void test_single_document() {
+    std::cout << "=== Testing Single Document ===" << std::endl;
+    
+    VectorStore store(20);
+    
+    // Create a test document manually
+    std::string json_str = R"({
+        "id": "test1",
+        "text": "Test document for debugging",
+        "metadata": {
+            "embedding": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 
+                         0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
+            "category": "test"
+        }
+    })";
+    
+    try {
+        simdjson::ondemand::parser parser;
+        simdjson::padded_string padded(json_str);
+        simdjson::ondemand::document doc;
+        auto parse_error = parser.iterate(padded).get(doc);
+        if (parse_error) {
+            std::cerr << "JSON parse error: " << simdjson::error_message(parse_error) << std::endl;
+            return;
+        }
+        
+        std::cout << "Adding document..." << std::endl;
+        auto add_error = store.add_document(doc);
+        if (add_error) {
+            std::cerr << "Document add error: " << simdjson::error_message(add_error) << std::endl;
+            return;
+        }
+        std::cout << "Document added successfully. Store size: " << store.size() << std::endl;
+        
+        // Test search
+        float query[20] = {0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0,
+                          0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0};
+        
+        auto results = store.search(query, 1);
+        std::cout << "Search completed. Found " << results.size() << " results" << std::endl;
+        if (!results.empty()) {
+            std::cout << "Top result score: " << results[0].first << std::endl;
+        }
+        
+    } catch (const std::exception& e) {
+        std::cerr << "Error: " << e.what() << std::endl;
+    }
+}
+
+void test_load_directory(const std::string& path) {
+    std::cout << "\n=== Testing Load Directory ===" << std::endl;
+    std::cout << "Loading from: " << path << std::endl;
+    
+    VectorStore store(20);
+    
+    try {
+        // Collect JSON files
+        std::vector<std::filesystem::path> json_files;
+        for (const auto& entry : std::filesystem::directory_iterator(path)) {
+            if (entry.path().extension() == ".json") {
+                std::cout << "Found JSON file: " << entry.path() << std::endl;
+                json_files.push_back(entry.path());
+            }
+        }
+        
+        std::cout << "Total JSON files found: " << json_files.size() << std::endl;
+        
+        // Process files
+        simdjson::ondemand::parser file_parser;
+        for (size_t i = 0; i < json_files.size(); ++i) {
+            std::cout << "Processing file " << (i+1) << "/" << json_files.size() 
+                      << ": " << json_files[i].filename() << std::endl;
+            
+            try {
+                std::cout << "  Loading file..." << std::endl;
+                auto json = simdjson::padded_string::load(json_files[i].string()).value_unsafe();
+                std::cout << "  File loaded, size: " << json.size() << " bytes" << std::endl;
+                
+                std::cout << "  Parsing JSON..." << std::endl;
+                auto json_doc = file_parser.iterate(json).value_unsafe();
+                std::cout << "  JSON parsed successfully" << std::endl;
+                
+                // Check first character to determine if it's an array
+                const char* json_start = json.data();
+                while (json_start && *json_start && std::isspace(*json_start)) {
+                    json_start++;
+                }
+                
+                if (json_start && *json_start == '[') {
+                    // It's an array
+                    std::cout << "  Detected array of documents" << std::endl;
+                    simdjson::ondemand::array doc_array;
+                    auto array_error = json_doc.get_array().get(doc_array);
+                    if (array_error) {
+                        std::cerr << "  Error getting array: " << simdjson::error_message(array_error) << std::endl;
+                        continue;
+                    }
+                    
+                    size_t doc_count = 0;
+                    size_t error_count = 0;
+                    for (auto doc_element : doc_array) {
+                        simdjson::ondemand::object doc_obj;
+                        auto obj_error = doc_element.get_object().get(doc_obj);
+                        if (obj_error) {
+                            std::cerr << "  Error getting object from array element: " << simdjson::error_message(obj_error) << std::endl;
+                            error_count++;
+                            continue;
+                        }
+                        
+                        auto add_error = store.add_document(doc_obj);
+                        if (add_error) {
+                            std::cerr << "  Error adding document: " << simdjson::error_message(add_error) << std::endl;
+                            error_count++;
+                        } else {
+                            doc_count++;
+                        }
+                    }
+                    std::cout << "  Added " << doc_count << " documents";
+                    if (error_count > 0) {
+                        std::cout << " (with " << error_count << " errors)";
+                    }
+                    std::cout << ". Current store size: " << store.size() << std::endl;
+                } else {
+                    // Single document
+                    std::cout << "  Detected single document" << std::endl;
+                    std::cout << "  Adding to store..." << std::endl;
+                    auto add_error = store.add_document(json_doc);
+                    if (add_error) {
+                        std::cerr << "  Error adding document: " << simdjson::error_message(add_error) << std::endl;
+                    } else {
+                        std::cout << "  Document added successfully";
+                    }
+                    std::cout << ". Current store size: " << store.size() << std::endl;
+                }
+                
+            } catch (const std::exception& e) {
+                std::cerr << "  Error processing file: " << e.what() << std::endl;
+            }
+        }
+        
+        std::cout << "\nAll files processed. Final store size: " << store.size() << std::endl;
+        
+        // Test normalization
+        std::cout << "Testing normalization..." << std::endl;
+        store.normalize_all();
+        std::cout << "Normalization completed" << std::endl;
+        
+    } catch (const std::exception& e) {
+        std::cerr << "Error in load directory: " << e.what() << std::endl;
+    }
+}
+
+int main(int argc, char** argv) {
+    std::cout << "Vector Store C++ Test Program" << std::endl;
+    std::cout << "=============================" << std::endl;
+    
+    // Test single document first
+    test_single_document();
+    
+    // Test directory loading
+    std::string test_dir = (argc > 1) ? argv[1] : "test";
+    test_load_directory(test_dir);
+    
+    std::cout << "\nAll tests completed!" << std::endl;
+    return 0;
+}