@faiss-node/native 0.1.4

package/package.json

@@ -0,0 +1,69 @@
+{
+  "name": "@faiss-node/native",
+  "version": "0.1.4",
+  "description": "High-performance Node.js native bindings for Facebook FAISS - the fastest vector similarity search library. Supports FLAT_L2, IVF_FLAT, and HNSW index types with async operations, persistence, and batch search.",
+  "main": "src/js/index.js",
+  "types": "src/js/types.d.ts",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/anupammaurya6767/faiss-node-native.git"
+  },
+  "homepage": "https://github.com/anupammaurya6767/faiss-node-native#readme",
+  "bugs": {
+    "url": "https://github.com/anupammaurya6767/faiss-node-native/issues"
+  },
+  "author": "Anupam Maurya <anupammaurya6767@gmail.com>",
+  "license": "MIT",
+  "keywords": [
+    "faiss",
+    "vector",
+    "similarity",
+    "search",
+    "embedding",
+    "nearest-neighbor",
+    "ann",
+    "approximate-nearest-neighbor",
+    "machine-learning",
+    "ai",
+    "semantic-search",
+    "rag",
+    "vector-database",
+    "hnsw",
+    "ivf",
+    "native",
+    "performance"
+  ],
+  "scripts": {
+    "build": "node-gyp rebuild",
+    "clean": "node-gyp clean",
+    "test": "jest",
+    "test:ci": "jest --ci --coverage --config=jest.ci.config.js --testPathIgnorePatterns=manual",
+    "test:unit": "jest test/unit",
+    "test:integration": "jest test/integration",
+    "test:watch": "jest --watch",
+    "test:memory": "node --expose-gc test/memory.test.js",
+    "test:local": "npm run clean && npm run build && npm test",
+    "ci:status": "bash scripts/check-ci-status.sh",
+    "docs": "npm run docs:js && npm run docs:cpp",
+    "docs:js": "typedoc",
+    "docs:cpp": "doxygen Doxyfile",
+    "docs:watch": "npm run docs:js -- --watch",
+    "docs:serve": "cd docs/api && python3 -m http.server 8000",
+    "docs:serve:cpp": "cd docs/html && python3 -m http.server 8001",
+    "docs:auto": "npm run docs && npm run docs:serve",
+    "prepublishOnly": "npm run docs",
+    "publish:local": "bash -c 'export NODE_AUTH_TOKEN=$(grep NPM_TOKEN .env | cut -d \\\"=\\\" -f2) && npm config set //registry.npmjs.org/:_authToken $NODE_AUTH_TOKEN && npm publish --access public'",
+    "publish:dry-run": "bash -c 'export NODE_AUTH_TOKEN=$(grep NPM_TOKEN .env | cut -d \\\"=\\\" -f2) && npm config set //registry.npmjs.org/:_authToken $NODE_AUTH_TOKEN && npm publish --access public --dry-run'"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "devDependencies": {
+    "express": "^4.22.1",
+    "jest": "^29.7.0",
+    "typedoc": "^0.25.0"
+  },
+  "dependencies": {
+    "node-addon-api": "^7.0.0"
+  }
+}

package/src/cpp/faiss_index.cpp

@@ -0,0 +1,300 @@
+// Include FAISS headers FIRST, before our header
+// This ensures all FAISS types are properly defined
+#include <faiss/impl/FaissAssert.h>
+#include <faiss/MetricType.h>
+#include <faiss/Index.h>
+#include <faiss/IndexFlat.h>
+#include <faiss/IndexIVF.h>
+#include <faiss/index_factory.h>
+#include <faiss/index_io.h>
+#include <faiss/impl/io.h>
+#include <fstream>
+#include <sstream>
+#include <cstdio>
+#include <cstring>
+
+// Now include our header
+#include "faiss_index.h"
+#include <stdexcept>
+
+FaissIndexWrapper::FaissIndexWrapper(int dims, const std::string& indexDescription, int metric) 
+    : dims_(dims), disposed_(false) {
+    if (dims <= 0) {
+        throw std::invalid_argument("Dimensions must be positive");
+    }
+    
+    // Create index using index_factory
+    // Examples: "Flat" -> IndexFlatL2, "IVF100,Flat" -> IndexIVFFlat, "HNSW32" -> IndexHNSW
+    faiss::MetricType metricType = static_cast<faiss::MetricType>(metric);
+    index_ = std::unique_ptr<faiss::Index>(faiss::index_factory(dims, indexDescription.c_str(), metricType));
+}
+
+FaissIndexWrapper::FaissIndexWrapper(int dims) 
+    : FaissIndexWrapper(dims, "Flat", 1) {  // Default to IndexFlatL2 with L2 metric
+}
+
+FaissIndexWrapper::~FaissIndexWrapper() {
+    if (!disposed_) {
+        Dispose();
+    }
+}
+
+void FaissIndexWrapper::Add(const float* vectors, size_t n) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    
+    if (disposed_) {
+        throw std::runtime_error("Index has been disposed");
+    }
+    
+    if (vectors == nullptr) {
+        throw std::invalid_argument("Vectors pointer cannot be null");
+    }
+    
+    if (n == 0) {
+        return; // Nothing to add
+    }
+    
+    // FAISS expects vectors as a flat array: [v1[0..d-1], v2[0..d-1], ...]
+    // This matches how Float32Array is laid out in memory
+    index_->add(n, vectors);
+}
+
+void FaissIndexWrapper::Search(const float* query, int k, float* distances, int64_t* labels) const {
+    std::lock_guard<std::mutex> lock(mutex_);
+    
+    if (disposed_) {
+        throw std::runtime_error("Index has been disposed");
+    }
+    
+    if (query == nullptr) {
+        throw std::invalid_argument("Query pointer cannot be null");
+    }
+    
+    if (distances == nullptr || labels == nullptr) {
+        throw std::invalid_argument("Output arrays cannot be null");
+    }
+    
+    if (k <= 0) {
+        throw std::invalid_argument("k must be positive");
+    }
+    
+    size_t ntotal = index_->ntotal;
+    if (ntotal == 0) {
+        throw std::runtime_error("Cannot search empty index");
+    }
+    
+    // Clamp k to available vectors
+    int actual_k = (k > static_cast<int>(ntotal)) ? static_cast<int>(ntotal) : k;
+    
+    // FAISS search: nq=1 (single query), k neighbors
+    // Cast labels to faiss::idx_t* for FAISS API
+    index_->search(1, query, actual_k, distances, reinterpret_cast<faiss::idx_t*>(labels));
+}
+
+void FaissIndexWrapper::SearchBatch(const float* queries, size_t nq, int k, float* distances, int64_t* labels) const {
+    std::lock_guard<std::mutex> lock(mutex_);
+    
+    if (disposed_) {
+        throw std::runtime_error("Index has been disposed");
+    }
+    
+    if (queries == nullptr) {
+        throw std::invalid_argument("Queries pointer cannot be null");
+    }
+    
+    if (distances == nullptr || labels == nullptr) {
+        throw std::invalid_argument("Output arrays cannot be null");
+    }
+    
+    if (nq == 0) {
+        throw std::invalid_argument("Number of queries must be positive");
+    }
+    
+    if (k <= 0) {
+        throw std::invalid_argument("k must be positive");
+    }
+    
+    size_t ntotal = index_->ntotal;
+    if (ntotal == 0) {
+        throw std::runtime_error("Cannot search empty index");
+    }
+    
+    // Clamp k to available vectors
+    int actual_k = (k > static_cast<int>(ntotal)) ? static_cast<int>(ntotal) : k;
+    
+    // FAISS batch search: nq queries, k neighbors per query
+    // Results are stored as: [q1_results, q2_results, ..., qn_results]
+    // Each query's results: [k distances, k labels]
+    // Cast labels to faiss::idx_t* for FAISS API
+    index_->search(nq, queries, actual_k, distances, reinterpret_cast<faiss::idx_t*>(labels));
+}
+
+size_t FaissIndexWrapper::GetTotalVectors() const {
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (disposed_) {
+        return 0;
+    }
+    return index_->ntotal;
+}
+
+int FaissIndexWrapper::GetDimensions() const {
+    return dims_;
+}
+
+void FaissIndexWrapper::Train(const float* vectors, size_t n) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    
+    if (disposed_) {
+        throw std::runtime_error("Index has been disposed");
+    }
+    
+    if (vectors == nullptr) {
+        throw std::invalid_argument("Vectors pointer cannot be null");
+    }
+    
+    if (n == 0) {
+        throw std::invalid_argument("Number of training vectors must be positive");
+    }
+    
+    index_->train(n, vectors);
+}
+
+void FaissIndexWrapper::SetNprobe(int nprobe) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    
+    if (disposed_) {
+        throw std::runtime_error("Index has been disposed");
+    }
+    
+    // Try to cast to IndexIVF to set nprobe
+    // This is safe even if not an IVF index (will just do nothing)
+    faiss::IndexIVF* ivf_index = dynamic_cast<faiss::IndexIVF*>(index_.get());
+    if (ivf_index) {
+        ivf_index->nprobe = nprobe;
+    }
+}
+
+bool FaissIndexWrapper::IsTrained() const {
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (disposed_) {
+        return false;
+    }
+    return index_->is_trained;
+}
+
+void FaissIndexWrapper::Dispose() {
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (disposed_) {
+        return;
+    }
+    
+    disposed_ = true;
+    index_.reset();
+}
+
+void FaissIndexWrapper::Save(const std::string& filename) const {
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (disposed_) {
+        throw std::runtime_error("Index has been disposed");
+    }
+    
+    if (filename.empty()) {
+        throw std::invalid_argument("Filename cannot be empty");
+    }
+    
+    try {
+        faiss::write_index(index_.get(), filename.c_str());
+    } catch (const std::exception& e) {
+        throw std::runtime_error(std::string("Failed to save index: ") + e.what());
+    }
+}
+
+std::unique_ptr<FaissIndexWrapper> FaissIndexWrapper::Load(const std::string& filename) {
+    if (filename.empty()) {
+        throw std::invalid_argument("Filename cannot be empty");
+    }
+    
+    try {
+        faiss::Index* loaded_index = faiss::read_index(filename.c_str());
+        
+        // Create wrapper with loaded index (supports any index type)
+        auto wrapper = std::make_unique<FaissIndexWrapper>(loaded_index->d);
+        wrapper->index_.reset(loaded_index);
+        wrapper->dims_ = loaded_index->d;
+        
+        return wrapper;
+    } catch (const std::exception& e) {
+        throw std::runtime_error(std::string("Failed to load index: ") + e.what());
+    }
+}
+
+std::vector<uint8_t> FaissIndexWrapper::ToBuffer() const {
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (disposed_) {
+        throw std::runtime_error("Index has been disposed");
+    }
+    
+    try {
+        // Use FAISS VectorIOWriter for direct memory serialization (no temp files)
+        // This is the same approach used by ewfian/faiss-node
+        faiss::VectorIOWriter writer;
+        faiss::write_index(index_.get(), &writer);
+        
+        // Return the buffer directly
+        return writer.data;
+    } catch (const std::exception& e) {
+        throw std::runtime_error(std::string("Failed to serialize index: ") + e.what());
+    }
+}
+
+std::unique_ptr<FaissIndexWrapper> FaissIndexWrapper::FromBuffer(const uint8_t* data, size_t length) {
+    if (data == nullptr || length == 0) {
+        throw std::invalid_argument("Invalid buffer data");
+    }
+    
+    try {
+        // Use FAISS VectorIOReader for direct memory deserialization (no temp files)
+        // This is the same approach used by ewfian/faiss-node
+        faiss::VectorIOReader reader;
+        reader.data.assign(data, data + length);
+        
+        faiss::Index* loaded_index = faiss::read_index(&reader);
+        
+        // Create wrapper with loaded index (supports any index type)
+        auto wrapper = std::make_unique<FaissIndexWrapper>(loaded_index->d);
+        wrapper->index_.reset(loaded_index);
+        wrapper->dims_ = loaded_index->d;
+        
+        return wrapper;
+    } catch (const std::exception& e) {
+        throw std::runtime_error(std::string("Failed to deserialize index: ") + e.what());
+    }
+}
+
+void FaissIndexWrapper::MergeFrom(const FaissIndexWrapper& other) {
+    // Lock both mutexes to prevent deadlock (always lock in same order)
+    // We'll use a simple approach: lock this first, then other
+    // Note: This could deadlock if two threads merge in opposite directions
+    // In practice, this is unlikely, but we should document it
+    std::lock_guard<std::mutex> lock1(mutex_);
+    std::lock_guard<std::mutex> lock2(other.mutex_);
+    
+    if (disposed_) {
+        throw std::runtime_error("Index has been disposed");
+    }
+    
+    if (other.disposed_) {
+        throw std::runtime_error("Cannot merge from disposed index");
+    }
+    
+    if (other.dims_ != dims_) {
+        throw std::invalid_argument("Merging index must have the same dimensions");
+    }
+    
+    try {
+        // FAISS merge_from copies all vectors from other index
+        index_->merge_from(*(other.index_));
+    } catch (const std::exception& e) {
+        throw std::runtime_error(std::string("Failed to merge index: ") + e.what());
+    }
+}

package/src/cpp/faiss_index.h

@@ -0,0 +1,99 @@
+#ifndef FAISS_NODE_INDEX_H
+#define FAISS_NODE_INDEX_H
+
+#include <memory>
+#include <cstdint>
+#include <string>
+#include <vector>
+#include <mutex>
+
+// Forward declarations - full definitions in .cpp file
+namespace faiss {
+    struct Index;  // FAISS defines Index as struct, not class
+    using idx_t = int64_t;  // idx_t is int64_t (defined in MetricType.h)
+}
+
+/**
+ * Wrapper class for FAISS index that manages memory and provides
+ * a clean interface for N-API bindings.
+ */
+class FaissIndexWrapper {
+public:
+    // Constructor: creates index using index_factory string
+    // Examples: "Flat" for IndexFlatL2, "IVF100,Flat" for IndexIVFFlat, "HNSW32" for IndexHNSW
+    FaissIndexWrapper(int dims, const std::string& indexDescription, int metric = 1);
+    
+    // Constructor: creates IndexFlatL2 (for backward compatibility)
+    explicit FaissIndexWrapper(int dims);
+    
+    // Destructor: automatic cleanup via RAII
+    ~FaissIndexWrapper();
+    
+    // Prevent copying (we own the FAISS index)
+    FaissIndexWrapper(const FaissIndexWrapper&) = delete;
+    FaissIndexWrapper& operator=(const FaissIndexWrapper&) = delete;
+    
+    // Add vectors to the index
+    // vectors: pointer to float array (n * dims elements)
+    // n: number of vectors
+    void Add(const float* vectors, size_t n);
+    
+    // Search for k nearest neighbors (single query)
+    // query: pointer to query vector (dims elements)
+    // k: number of neighbors to return
+    // distances: output array (k elements) - caller must allocate
+    // labels: output array (k elements) - caller must allocate
+    void Search(const float* query, int k, float* distances, int64_t* labels) const;
+    
+    // Batch search for k nearest neighbors (multiple queries)
+    // queries: pointer to query vectors (nq * dims elements)
+    // nq: number of queries
+    // k: number of neighbors to return per query
+    // distances: output array (nq * k elements) - caller must allocate
+    // labels: output array (nq * k elements) - caller must allocate
+    void SearchBatch(const float* queries, size_t nq, int k, float* distances, int64_t* labels) const;
+    
+    // Train the index (required for IVF indexes)
+    void Train(const float* vectors, size_t n);
+    
+    // Get index statistics
+    size_t GetTotalVectors() const;
+    int GetDimensions() const;
+    bool IsTrained() const;
+    
+    // Set nprobe for IVF indexes
+    void SetNprobe(int nprobe);
+    
+    // Dispose: explicitly free resources
+    void Dispose();
+    
+    // Check if disposed (thread-safe)
+    bool IsDisposed() const {
+        std::lock_guard<std::mutex> lock(mutex_);
+        return disposed_;
+    }
+    
+    // Save index to file
+    void Save(const std::string& filename) const;
+    
+    // Load index from file (static factory method)
+    static std::unique_ptr<FaissIndexWrapper> Load(const std::string& filename);
+    
+    // Serialize index to buffer
+    std::vector<uint8_t> ToBuffer() const;
+    
+    // Deserialize index from buffer (static factory method)
+    static std::unique_ptr<FaissIndexWrapper> FromBuffer(const uint8_t* data, size_t length);
+    
+    // Merge vectors from another index
+    // other: reference to another FaissIndexWrapper
+    void MergeFrom(const FaissIndexWrapper& other);
+
+private:
+    std::unique_ptr<faiss::Index> index_;  // Base Index pointer (can hold any index type)
+    int dims_;
+    bool disposed_;
+    mutable std::mutex mutex_;  // Protect concurrent access
+};
+
+#endif // FAISS_NODE_INDEX_H