native-vector-store 0.3.7 → 0.3.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "native-vector-store",
-  "version": "0.3.7",
+  "version": "0.3.9",
   "description": "High-performance local vector store with SIMD optimization for MCP servers",
   "main": "index.js",
   "types": "lib/index.d.ts",

package/src/vector_store_loader.cpp

@@ -92,13 +92,16 @@ void VectorStoreLoader::loadDirectory(VectorStore* store, const std::string& pat
     
     for (size_t w = 0; w < num_workers; ++w) {
         consumers.emplace_back([&]() {
-            // Each thread needs its own parser with large capacity for big files
-            simdjson::ondemand::parser doc_parser(16 * 1024 * 1024); // 16MB capacity
+            // Each thread needs its own parser with initial capacity
+            simdjson::ondemand::parser doc_parser(1 * 1024 * 1024 * 1024); // 16MB initial capacity
+            // Set a larger maximum capacity for very large files (up to 512MB)
+            doc_parser.allocate(512 * 1024 * 1024);
             FileData* data = nullptr;
             
             while (true) {
                 // Try to get work from queue
                 if (queue.try_pop(data)) {
+                    
                     // Process the file
                     simdjson::padded_string json(data->content);
                     
@@ -193,4 +196,4 @@ void VectorStoreLoader::loadDirectory(VectorStore* store, const std::string& pat
     
     // Finalize after batch load - normalize and switch to serving phase
     store->finalize();
-}
+}

package/src/vector_store_loader_adaptive.cpp

@@ -132,8 +132,10 @@ void VectorStoreLoader::loadDirectoryAdaptive(VectorStore* store, const std::str
     
     for (size_t w = 0; w < num_workers; ++w) {
         consumers.emplace_back([&]() {
-            // Each thread needs its own parser with large capacity for big files
-            simdjson::ondemand::parser doc_parser(16 * 1024 * 1024); // 16MB capacity
+            // Each thread needs its own parser with initial capacity
+            simdjson::ondemand::parser doc_parser(1 * 1024 * 1024 * 1024); // 16MB initial capacity
+            // Set a larger maximum capacity for very large files (up to 512MB)
+            doc_parser.allocate(512 * 1024 * 1024);
             MixedFileData* data = nullptr;
             
             while (true) {
@@ -144,6 +146,7 @@ void VectorStoreLoader::loadDirectoryAdaptive(VectorStore* store, const std::str
                         ? simdjson::padded_string(data->mmap->data(), data->mmap->size())
                         : simdjson::padded_string(data->content);
                     
+                    
                     // Check if it's an array or object
                     const char* json_start = json.data();
                     while (json_start && *json_start && std::isspace(*json_start)) {
@@ -217,4 +220,4 @@ void VectorStoreLoader::loadDirectoryAdaptive(VectorStore* store, const std::str
     
     // Finalize after batch load - normalize and switch to serving phase
     store->finalize();
-}
+}

package/src/vector_store_loader_mmap.cpp

@@ -67,13 +67,16 @@ void VectorStoreLoader::loadDirectoryMMap(VectorStore* store, const std::string&
     
     for (size_t w = 0; w < num_workers; ++w) {
         consumers.emplace_back([&]() {
-            // Each thread needs its own parser with large capacity for big files
-            simdjson::ondemand::parser doc_parser(16 * 1024 * 1024); // 16MB capacity
+            // Each thread needs its own parser with initial capacity
+            simdjson::ondemand::parser doc_parser(1 * 1024 * 1024 * 1024); // 16MB initial capacity
+            // Set a larger maximum capacity for very large files (up to 512MB)
+            doc_parser.set_max_capacity(512 * 1024 * 1024);
             MMapFileData* data = nullptr;
             
             while (true) {
                 // Try to get work from queue
                 if (queue.try_pop(data)) {
+                    
                     // Process the memory-mapped file
                     // For mmap, we need to copy to ensure padding
                     simdjson::padded_string json(data->mmap->data(), data->mmap->size());
@@ -151,4 +154,4 @@ void VectorStoreLoader::loadDirectoryMMap(VectorStore* store, const std::string&
     
     // Finalize after batch load - normalize and switch to serving phase
     store->finalize();
-}
+}