native-vector-store 0.1.0

package/src/binding.cc

@@ -0,0 +1,151 @@
+#include <napi.h>
+#include "vector_store.h"
+#include "vector_store_loader.h"
+#include <cmath>
+
+class VectorStoreWrapper : public Napi::ObjectWrap<VectorStoreWrapper> {
+    std::unique_ptr<VectorStore> store_;
+    size_t dim_;
+    
+public:
+    static Napi::Object Init(Napi::Env env, Napi::Object exports) {
+        Napi::Function func = DefineClass(env, "VectorStore", {
+            InstanceMethod("loadDir", &VectorStoreWrapper::LoadDir),
+            InstanceMethod("loadDirMMap", &VectorStoreWrapper::LoadDirMMap),
+            InstanceMethod("loadDirAdaptive", &VectorStoreWrapper::LoadDirAdaptive),
+            InstanceMethod("addDocument", &VectorStoreWrapper::AddDocument),
+            InstanceMethod("search", &VectorStoreWrapper::Search),
+            InstanceMethod("normalize", &VectorStoreWrapper::Normalize),
+            InstanceMethod("finalize", &VectorStoreWrapper::FinalizeStore),
+            InstanceMethod("isFinalized", &VectorStoreWrapper::IsFinalized),
+            InstanceMethod("size", &VectorStoreWrapper::Size)
+        });
+        
+        exports.Set("VectorStore", func);
+        return exports;
+    }
+    
+    VectorStoreWrapper(const Napi::CallbackInfo& info) 
+        : Napi::ObjectWrap<VectorStoreWrapper>(info) {
+        dim_ = info[0].As<Napi::Number>().Uint32Value();
+        store_ = std::make_unique<VectorStore>(dim_);
+    }
+    
+    void LoadDir(const Napi::CallbackInfo& info) {
+        std::string path = info[0].As<Napi::String>();
+        // Use adaptive loader as default for best performance
+        VectorStoreLoader::loadDirectoryAdaptive(store_.get(), path);
+    }
+    
+    void LoadDirMMap(const Napi::CallbackInfo& info) {
+        std::string path = info[0].As<Napi::String>();
+        VectorStoreLoader::loadDirectoryMMap(store_.get(), path);
+    }
+    
+    void LoadDirAdaptive(const Napi::CallbackInfo& info) {
+        std::string path = info[0].As<Napi::String>();
+        VectorStoreLoader::loadDirectoryAdaptive(store_.get(), path);
+    }
+    
+    void AddDocument(const Napi::CallbackInfo& info) {
+        Napi::Object doc = info[0].As<Napi::Object>();
+        
+        // Convert JS object to JSON string
+        std::string json_str = "{";
+        json_str += "\"id\":\"" + doc.Get("id").As<Napi::String>().Utf8Value() + "\",";
+        json_str += "\"text\":\"" + doc.Get("text").As<Napi::String>().Utf8Value() + "\",";
+        json_str += "\"metadata\":{\"embedding\":[";
+        
+        // Get embedding from metadata
+        Napi::Object metadata = doc.Get("metadata").As<Napi::Object>();
+        Napi::Array embedding = metadata.Get("embedding").As<Napi::Array>();
+        
+        for (uint32_t i = 0; i < embedding.Length(); ++i) {
+            if (i > 0) json_str += ",";
+            json_str += std::to_string(embedding.Get(i).As<Napi::Number>().DoubleValue());
+        }
+        json_str += "]}}";
+        
+        // Parse and add
+        simdjson::ondemand::parser parser;
+        simdjson::padded_string padded(json_str);
+        simdjson::ondemand::document json_doc;
+        auto parse_error = parser.iterate(padded).get(json_doc);
+        if (parse_error) {
+            Napi::Error::New(info.Env(), 
+                std::string("JSON parse error: ") + simdjson::error_message(parse_error))
+                .ThrowAsJavaScriptException();
+            return;
+        }
+        
+        auto add_error = store_->add_document(json_doc);
+        if (add_error) {
+            Napi::Error::New(info.Env(), 
+                std::string("Document add error: ") + simdjson::error_message(add_error))
+                .ThrowAsJavaScriptException();
+            return;
+        }
+    }
+    
+    Napi::Value Search(const Napi::CallbackInfo& info) {
+        Napi::Env env = info.Env();
+        Napi::Float32Array query_array = info[0].As<Napi::Float32Array>();
+        size_t k = info[1].As<Napi::Number>().Uint32Value();
+        
+        // Normalize query if requested
+        bool normalize_query = info.Length() > 2 ? info[2].As<Napi::Boolean>() : true;
+        
+        std::vector<float> query(query_array.Data(), 
+                                 query_array.Data() + query_array.ElementLength());
+        
+        if (normalize_query) {
+            float sum = 0.0f;
+            for (float v : query) sum += v * v;
+            if (sum > 1e-10f) {
+                float inv_norm = 1.0f / std::sqrt(sum);
+                for (float& v : query) v *= inv_norm;
+            }
+        }
+        
+        auto results = store_->search(query.data(), k);
+        
+        Napi::Array output = Napi::Array::New(env, results.size());
+        for (size_t i = 0; i < results.size(); ++i) {
+            const auto& entry = store_->get_entry(results[i].second);
+            
+            Napi::Object result = Napi::Object::New(env);
+            result.Set("score", results[i].first);
+            result.Set("id", std::string(entry.doc.id));
+            result.Set("text", std::string(entry.doc.text));
+            
+            // For now, return metadata as JSON string
+            result.Set("metadata_json", std::string(entry.doc.metadata_json));
+            
+            output[i] = result;
+        }
+        
+        return output;
+    }
+    
+    void Normalize(const Napi::CallbackInfo& info) {
+        store_->normalize_all();
+    }
+    
+    void FinalizeStore(const Napi::CallbackInfo& info) {
+        store_->finalize();
+    }
+    
+    Napi::Value IsFinalized(const Napi::CallbackInfo& info) {
+        return Napi::Boolean::New(info.Env(), store_->is_finalized());
+    }
+    
+    Napi::Value Size(const Napi::CallbackInfo& info) {
+        return Napi::Number::New(info.Env(), store_->size());
+    }
+};
+
+Napi::Object Init(Napi::Env env, Napi::Object exports) {
+    return VectorStoreWrapper::Init(env, exports);
+}
+
+NODE_API_MODULE(vector_store, Init)

package/src/defs.h

@@ -0,0 +1,107 @@
+/* -*- mode: c++; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 4 -*- */
+#ifndef ATOMIC_QUEUE_DEFS_H_INCLUDED
+#define ATOMIC_QUEUE_DEFS_H_INCLUDED
+
+// Copyright (c) 2019 Maxim Egorushkin. MIT License. See the full licence in file LICENSE.
+
+#include <atomic>
+
+#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)
+#include <emmintrin.h>
+namespace atomic_queue {
+constexpr int CACHE_LINE_SIZE = 64;
+static inline void spin_loop_pause() noexcept {
+    _mm_pause();
+}
+} // namespace atomic_queue
+#elif defined(__arm__) || defined(__aarch64__) || defined(_M_ARM64)
+namespace atomic_queue {
+constexpr int CACHE_LINE_SIZE = 64;
+static inline void spin_loop_pause() noexcept {
+#if (defined(__ARM_ARCH_6K__) || \
+     defined(__ARM_ARCH_6Z__) || \
+     defined(__ARM_ARCH_6ZK__) || \
+     defined(__ARM_ARCH_6T2__) || \
+     defined(__ARM_ARCH_7__) || \
+     defined(__ARM_ARCH_7A__) || \
+     defined(__ARM_ARCH_7R__) || \
+     defined(__ARM_ARCH_7M__) || \
+     defined(__ARM_ARCH_7S__) || \
+     defined(__ARM_ARCH_8A__) || \
+     defined(__aarch64__))
+    asm volatile ("yield" ::: "memory");
+#elif defined(_M_ARM64)
+    __yield();
+#else
+    asm volatile ("nop" ::: "memory");
+#endif
+}
+} // namespace atomic_queue
+#elif defined(__ppc64__) || defined(__powerpc64__)
+namespace atomic_queue {
+constexpr int CACHE_LINE_SIZE = 128; // TODO: Review that this is the correct value.
+static inline void spin_loop_pause() noexcept {
+    asm volatile("or 31,31,31 # very low priority"); // TODO: Review and benchmark that this is the right instruction.
+}
+} // namespace atomic_queue
+#elif defined(__s390x__)
+namespace atomic_queue {
+constexpr int CACHE_LINE_SIZE = 256; // TODO: Review that this is the correct value.
+static inline void spin_loop_pause() noexcept {} // TODO: Find the right instruction to use here, if any.
+} // namespace atomic_queue
+#elif defined(__riscv)
+namespace atomic_queue {
+constexpr int CACHE_LINE_SIZE = 64;
+static inline void spin_loop_pause() noexcept {
+    asm volatile (".insn i 0x0F, 0, x0, x0, 0x010");
+}
+} // namespace atomic_queue
+#elif defined(__loongarch__)
+namespace atomic_queue {
+constexpr int CACHE_LINE_SIZE = 64;
+static inline void spin_loop_pause() noexcept
+{
+    asm volatile("nop \n nop \n nop \n nop \n nop \n nop \n nop \n nop");
+}
+} // namespace atomic_queue
+#else
+#ifdef _MSC_VER
+#pragma message("Unknown CPU architecture. Using L1 cache line size of 64 bytes and no spinloop pause instruction.")
+#else
+#warning "Unknown CPU architecture. Using L1 cache line size of 64 bytes and no spinloop pause instruction."
+#endif
+namespace atomic_queue {
+constexpr int CACHE_LINE_SIZE = 64; // TODO: Review that this is the correct value.
+static inline void spin_loop_pause() noexcept {}
+} // namespace atomic_queue
+#endif
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace atomic_queue {
+
+#if defined(__GNUC__) || defined(__clang__)
+#define ATOMIC_QUEUE_LIKELY(expr) __builtin_expect(static_cast<bool>(expr), 1)
+#define ATOMIC_QUEUE_UNLIKELY(expr) __builtin_expect(static_cast<bool>(expr), 0)
+#define ATOMIC_QUEUE_NOINLINE __attribute__((noinline))
+#else
+#define ATOMIC_QUEUE_LIKELY(expr) (expr)
+#define ATOMIC_QUEUE_UNLIKELY(expr) (expr)
+#define ATOMIC_QUEUE_NOINLINE
+#endif
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+auto constexpr A = std::memory_order_acquire;
+auto constexpr R = std::memory_order_release;
+auto constexpr X = std::memory_order_relaxed;
+auto constexpr C = std::memory_order_seq_cst;
+auto constexpr AR = std::memory_order_acq_rel;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace atomic_queue
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#endif // ATOMIC_QUEUE_DEFS_H_INCLUDED

package/src/mmap_file.h

@@ -0,0 +1,159 @@
+#pragma once
+#include <string>
+#include <memory>
+
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#endif
+
+// Cross-platform memory-mapped file wrapper
+class MMapFile {
+public:
+    MMapFile() = default;
+    ~MMapFile() { close(); }
+    
+    // Disable copy, enable move
+    MMapFile(const MMapFile&) = delete;
+    MMapFile& operator=(const MMapFile&) = delete;
+    MMapFile(MMapFile&& other) noexcept { *this = std::move(other); }
+    MMapFile& operator=(MMapFile&& other) noexcept {
+        if (this != &other) {
+            close();
+            data_ = other.data_;
+            size_ = other.size_;
+            #ifdef _WIN32
+            file_handle_ = other.file_handle_;
+            map_handle_ = other.map_handle_;
+            other.file_handle_ = INVALID_HANDLE_VALUE;
+            other.map_handle_ = nullptr;
+            #else
+            fd_ = other.fd_;
+            other.fd_ = -1;
+            #endif
+            other.data_ = nullptr;
+            other.size_ = 0;
+        }
+        return *this;
+    }
+    
+    bool open(const std::string& filepath) {
+        close();
+        
+        #ifdef _WIN32
+        // Windows implementation
+        file_handle_ = CreateFileA(filepath.c_str(), GENERIC_READ, FILE_SHARE_READ,
+                                   nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
+        if (file_handle_ == INVALID_HANDLE_VALUE) {
+            return false;
+        }
+        
+        LARGE_INTEGER file_size;
+        if (!GetFileSizeEx(file_handle_, &file_size)) {
+            CloseHandle(file_handle_);
+            file_handle_ = INVALID_HANDLE_VALUE;
+            return false;
+        }
+        size_ = static_cast<size_t>(file_size.QuadPart);
+        
+        if (size_ == 0) {
+            return true; // Empty file, nothing to map
+        }
+        
+        map_handle_ = CreateFileMappingA(file_handle_, nullptr, PAGE_READONLY, 0, 0, nullptr);
+        if (!map_handle_) {
+            CloseHandle(file_handle_);
+            file_handle_ = INVALID_HANDLE_VALUE;
+            return false;
+        }
+        
+        data_ = MapViewOfFile(map_handle_, FILE_MAP_READ, 0, 0, 0);
+        if (!data_) {
+            CloseHandle(map_handle_);
+            CloseHandle(file_handle_);
+            map_handle_ = nullptr;
+            file_handle_ = INVALID_HANDLE_VALUE;
+            return false;
+        }
+        #else
+        // POSIX implementation
+        fd_ = ::open(filepath.c_str(), O_RDONLY);
+        if (fd_ < 0) {
+            return false;
+        }
+        
+        struct stat st;
+        if (fstat(fd_, &st) < 0) {
+            ::close(fd_);
+            fd_ = -1;
+            return false;
+        }
+        size_ = static_cast<size_t>(st.st_size);
+        
+        if (size_ == 0) {
+            return true; // Empty file, nothing to map
+        }
+        
+        data_ = mmap(nullptr, size_, PROT_READ, MAP_PRIVATE, fd_, 0);
+        if (data_ == MAP_FAILED) {
+            ::close(fd_);
+            fd_ = -1;
+            data_ = nullptr;
+            return false;
+        }
+        
+        // Advise kernel about access pattern
+        madvise(data_, size_, MADV_SEQUENTIAL);
+        #endif
+        
+        return true;
+    }
+    
+    void close() {
+        if (data_) {
+            #ifdef _WIN32
+            UnmapViewOfFile(data_);
+            #else
+            munmap(data_, size_);
+            #endif
+            data_ = nullptr;
+        }
+        
+        #ifdef _WIN32
+        if (map_handle_) {
+            CloseHandle(map_handle_);
+            map_handle_ = nullptr;
+        }
+        if (file_handle_ != INVALID_HANDLE_VALUE) {
+            CloseHandle(file_handle_);
+            file_handle_ = INVALID_HANDLE_VALUE;
+        }
+        #else
+        if (fd_ >= 0) {
+            ::close(fd_);
+            fd_ = -1;
+        }
+        #endif
+        
+        size_ = 0;
+    }
+    
+    const char* data() const { return static_cast<const char*>(data_); }
+    size_t size() const { return size_; }
+    bool is_open() const { return data_ != nullptr || size_ == 0; }
+    
+private:
+    void* data_ = nullptr;
+    size_t size_ = 0;
+    
+    #ifdef _WIN32
+    HANDLE file_handle_ = INVALID_HANDLE_VALUE;
+    HANDLE map_handle_ = nullptr;
+    #else
+    int fd_ = -1;
+    #endif
+};