faiss 0.5.3 → 0.6.1

data/vendor/faiss/faiss/IndexHNSW.h

@@ -9,13 +9,16 @@
 
 #pragma once
 
+#include <optional>
 #include <vector>
-#include "faiss/Index.h"
 
+#include <faiss/Index.h>
 #include <faiss/IndexFlat.h>
 #include <faiss/IndexPQ.h>
 #include <faiss/IndexScalarQuantizer.h>
 #include <faiss/impl/HNSW.h>
+#include <faiss/impl/Panorama.h>
+#include <faiss/impl/hnsw/LockVector.h>
 #include <faiss/utils/utils.h>
 
 namespace faiss {
@@ -26,7 +29,7 @@ struct IndexHNSW;
  * link structure built on top */
 
 struct IndexHNSW : Index {
-    typedef HNSW::storage_idx_t storage_idx_t;
+    using storage_idx_t = HNSW::storage_idx_t;
 
     // the link structure
     HNSW hnsw;
@@ -47,11 +50,20 @@ struct IndexHNSW : Index {
     // used when GpuIndexCagra::copyFrom(IndexHNSWCagra*) is invoked.
     bool keep_max_size_level0 = false;
 
+    // See impl/VisitedTable.h.
+    std::optional<bool> use_visited_hashset;
+
+    // Per-node locks for HNSW graph construction.
+    LockVector locks;
+    // locks are freed after each call to add() unless this flag is set.
+    bool retain_locks = false;
+
     explicit IndexHNSW(int d = 0, int M = 32, MetricType metric = METRIC_L2);
     explicit IndexHNSW(Index* storage, int M = 32);
 
     ~IndexHNSW() override;
 
+    /// Adds vectors to the index. May not be called concurrently.
     void add(idx_t n, const float* x) override;
 
     /// Trains the storage if needed
@@ -73,6 +85,12 @@ struct IndexHNSW : Index {
             RangeSearchResult* result,
             const SearchParameters* params = nullptr) const override;
 
+    /** search one vector with a custom result handler */
+    void search1(
+            const float* x,
+            ResultHandler& handler,
+            SearchParameters* params = nullptr) const override;
+
     void reconstruct(idx_t key, float* recons) const override;
 
     void reset() override;
@@ -164,20 +182,11 @@ struct IndexHNSWFlatPanorama : IndexHNSWFlat {
 
     /// Inline for performance - called frequently in search hot path.
     const float* get_cum_sum(idx_t i) const {
-        return cum_sums.data() + i * (num_panorama_levels + 1);
+        return cum_sums.data() + i * (pano.n_levels + 1);
     }
 
-    /// Compute cumulative sums for a vector (used both for database points and
-    /// queries).
-    static void compute_cum_sums(
-            const float* x,
-            float* dst_cum_sums,
-            int d,
-            int num_panorama_levels,
-            int panorama_level_width);
-
     std::vector<float> cum_sums;
-    const size_t panorama_level_width;
+    Panorama pano;
     const size_t num_panorama_levels;
 };
 
@@ -257,9 +266,16 @@ struct IndexHNSWCagra : IndexHNSW {
             idx_t* labels,
             const SearchParameters* params = nullptr) const override;
 
+    void range_search(
+            idx_t n,
+            const float* x,
+            float radius,
+            RangeSearchResult* result,
+            const SearchParameters* params = nullptr) const override;
+
     faiss::NumericType get_numeric_type() const;
     void set_numeric_type(faiss::NumericType numeric_type);
-    NumericType numeric_type_;
+    NumericType numeric_type_ = Float32;
 };
 
 } // namespace faiss

data/vendor/faiss/faiss/IndexIDMap.cpp

@@ -12,7 +12,7 @@
 #include <cinttypes>
 #include <cstdint>
 #include <cstdio>
-#include "faiss/Index.h"
+#include <stdexcept>
 
 #include <faiss/impl/AuxIndexStructures.h>
 #include <faiss/impl/FaissAssert.h>
@@ -24,7 +24,7 @@ namespace {
 
 // IndexBinary needs to update the code_size when d is set...
 
-void sync_d(Index* index) {}
+void sync_d(Index* /* index */) {}
 
 void sync_d(IndexBinary* index) {
     FAISS_THROW_IF_NOT(index->d % 8 == 0);
@@ -49,12 +49,12 @@ NumericType component_t_to_numeric() {
  *******************************************************/
 
 template <typename IndexT>
-IndexIDMapTemplate<IndexT>::IndexIDMapTemplate(IndexT* index) : index(index) {
-    FAISS_THROW_IF_NOT_MSG(index->ntotal == 0, "index must be empty on input");
-    this->is_trained = index->is_trained;
-    this->metric_type = index->metric_type;
-    this->verbose = index->verbose;
-    this->d = index->d;
+IndexIDMapTemplate<IndexT>::IndexIDMapTemplate(IndexT* index_) : index(index_) {
+    FAISS_THROW_IF_NOT_MSG(index_->ntotal == 0, "index must be empty on input");
+    this->is_trained = index_->is_trained;
+    this->metric_type = index_->metric_type;
+    this->verbose = index_->verbose;
+    this->d = index_->d;
     sync_d(this);
 }
 
@@ -62,7 +62,7 @@ template <typename IndexT>
 void IndexIDMapTemplate<IndexT>::add_ex(
         idx_t,
         const void*,
-        NumericType numeric_type) {
+        NumericType /* numeric_type */) {
     FAISS_THROW_MSG(
             "add does not make sense with IndexIDMap, "
             "use add_with_ids");
@@ -196,9 +196,10 @@ void IndexIDMapTemplate<IndexT>::search_ex(
     }
     index->search_ex(n, x, numeric_type, k, distances, labels, params);
     idx_t* li = labels;
+    const idx_t id_map_size = static_cast<idx_t>(id_map.size());
 #pragma omp parallel for
     for (idx_t i = 0; i < n * k; i++) {
-        li[i] = li[i] < 0 ? li[i] : id_map[li[i]];
+        li[i] = (li[i] < 0 || li[i] >= id_map_size) ? idx_t(-1) : id_map[li[i]];
     }
 }
 
@@ -237,10 +238,12 @@ void IndexIDMapTemplate<IndexT>::range_search(
         index->range_search(n, x, radius, result);
     }
 
+    const idx_t id_map_size = static_cast<idx_t>(id_map.size());
 #pragma omp parallel for
-    for (idx_t i = 0; i < result->lims[result->nq]; i++) {
-        result->labels[i] = result->labels[i] < 0 ? result->labels[i]
-                                                  : id_map[result->labels[i]];
+    for (idx_t i = 0; i < static_cast<idx_t>(result->lims[result->nq]); i++) {
+        const idx_t label = result->labels[i];
+        result->labels[i] =
+                (label < 0 || label >= id_map_size) ? idx_t(-1) : id_map[label];
     }
 }
 
@@ -298,8 +301,8 @@ IndexIDMapTemplate<IndexT>::~IndexIDMapTemplate() {
  *******************************************************/
 
 template <typename IndexT>
-IndexIDMap2Template<IndexT>::IndexIDMap2Template(IndexT* index)
-        : IndexIDMapTemplate<IndexT>(index) {}
+IndexIDMap2Template<IndexT>::IndexIDMap2Template(IndexT* index_)
+        : IndexIDMapTemplate<IndexT>(index_) {}
 
 template <typename IndexT>
 void IndexIDMap2Template<IndexT>::add_with_ids_ex(
@@ -307,9 +310,9 @@ void IndexIDMap2Template<IndexT>::add_with_ids_ex(
         const void* x,
         NumericType numeric_type,
         const idx_t* xids) {
-    size_t prev_ntotal = this->ntotal;
+    idx_t prev_ntotal = this->ntotal;
     IndexIDMapTemplate<IndexT>::add_with_ids_ex(n, x, numeric_type, xids);
-    for (size_t i = prev_ntotal; i < this->ntotal; i++) {
+    for (idx_t i = prev_ntotal; i < this->ntotal; i++) {
         rev_map[this->id_map[i]] = i;
     }
 }
@@ -329,8 +332,9 @@ void IndexIDMap2Template<IndexT>::add_with_ids(
 template <typename IndexT>
 void IndexIDMap2Template<IndexT>::check_consistency() const {
     FAISS_THROW_IF_NOT(rev_map.size() == this->id_map.size());
-    FAISS_THROW_IF_NOT(this->id_map.size() == this->ntotal);
-    for (size_t i = 0; i < this->ntotal; i++) {
+    FAISS_THROW_IF_NOT(
+            this->id_map.size() == static_cast<size_t>(this->ntotal));
+    for (idx_t i = 0; i < this->ntotal; i++) {
         idx_t ii = rev_map.at(this->id_map[i]);
         FAISS_THROW_IF_NOT(ii == i);
     }
@@ -338,9 +342,9 @@ void IndexIDMap2Template<IndexT>::check_consistency() const {
 
 template <typename IndexT>
 void IndexIDMap2Template<IndexT>::merge_from(IndexT& otherIndex, idx_t add_id) {
-    size_t prev_ntotal = this->ntotal;
+    idx_t prev_ntotal = this->ntotal;
     IndexIDMapTemplate<IndexT>::merge_from(otherIndex, add_id);
-    for (size_t i = prev_ntotal; i < this->ntotal; i++) {
+    for (idx_t i = prev_ntotal; i < this->ntotal; i++) {
         rev_map[this->id_map[i]] = i;
     }
     static_cast<IndexIDMap2Template<IndexT>&>(otherIndex).rev_map.clear();
@@ -349,7 +353,7 @@ void IndexIDMap2Template<IndexT>::merge_from(IndexT& otherIndex, idx_t add_id) {
 template <typename IndexT>
 void IndexIDMap2Template<IndexT>::construct_rev_map() {
     rev_map.clear();
-    for (size_t i = 0; i < this->ntotal; i++) {
+    for (idx_t i = 0; i < this->ntotal; i++) {
         rev_map[this->id_map[i]] = i;
     }
 }

data/vendor/faiss/faiss/IndexIDMap.h

@@ -132,15 +132,17 @@ struct IDSelectorTranslated : IDSelector {
     const IDSelector* sel;
 
     IDSelectorTranslated(
-            const std::vector<int64_t>& id_map,
-            const IDSelector* sel)
-            : id_map(id_map), sel(sel) {}
+            const std::vector<int64_t>& id_map_in,
+            const IDSelector* sel_in)
+            : id_map(id_map_in), sel(sel_in) {}
 
-    IDSelectorTranslated(IndexBinaryIDMap& index_idmap, const IDSelector* sel)
-            : id_map(index_idmap.id_map), sel(sel) {}
+    IDSelectorTranslated(
+            IndexBinaryIDMap& index_idmap,
+            const IDSelector* sel_in)
+            : id_map(index_idmap.id_map), sel(sel_in) {}
 
-    IDSelectorTranslated(IndexIDMap& index_idmap, const IDSelector* sel)
-            : id_map(index_idmap.id_map), sel(sel) {}
+    IDSelectorTranslated(IndexIDMap& index_idmap, const IDSelector* sel_in)
+            : id_map(index_idmap.id_map), sel(sel_in) {}
 
     bool is_member(idx_t id) const override {
         return sel->is_member(id_map[id]);