faiss 0.4.3 → 0.5.0

data/vendor/faiss/faiss/IndexIDMap.cpp

@@ -59,7 +59,7 @@ IndexIDMapTemplate<IndexT>::IndexIDMapTemplate(IndexT* index) : index(index) {
 }
 
 template <typename IndexT>
-void IndexIDMapTemplate<IndexT>::addEx(
+void IndexIDMapTemplate<IndexT>::add_ex(
         idx_t,
         const void*,
         NumericType numeric_type) {
@@ -78,11 +78,11 @@ void IndexIDMapTemplate<IndexT>::add(
 }
 
 template <typename IndexT>
-void IndexIDMapTemplate<IndexT>::trainEx(
+void IndexIDMapTemplate<IndexT>::train_ex(
         idx_t n,
         const void* x,
         NumericType numeric_type) {
-    index->trainEx(n, x, numeric_type);
+    index->train_ex(n, x, numeric_type);
     this->is_trained = index->is_trained;
 }
 
@@ -90,7 +90,8 @@ template <typename IndexT>
 void IndexIDMapTemplate<IndexT>::train(
         idx_t n,
         const typename IndexT::component_t* x) {
-    trainEx(n,
+    train_ex(
+            n,
             static_cast<const void*>(x),
             component_t_to_numeric<typename IndexT::component_t>());
 }
@@ -103,12 +104,12 @@ void IndexIDMapTemplate<IndexT>::reset() {
 }
 
 template <typename IndexT>
-void IndexIDMapTemplate<IndexT>::add_with_idsEx(
+void IndexIDMapTemplate<IndexT>::add_with_ids_ex(
         idx_t n,
         const void* x,
         NumericType numeric_type,
         const idx_t* xids) {
-    index->addEx(n, x, numeric_type);
+    index->add_ex(n, x, numeric_type);
     for (idx_t i = 0; i < n; i++) {
         id_map.push_back(xids[i]);
     }
@@ -120,7 +121,7 @@ void IndexIDMapTemplate<IndexT>::add_with_ids(
         idx_t n,
         const typename IndexT::component_t* x,
         const idx_t* xids) {
-    add_with_idsEx(
+    add_with_ids_ex(
             n,
             static_cast<const void*>(x),
             component_t_to_numeric<typename IndexT::component_t>(),
@@ -166,7 +167,7 @@ struct ScopedSelChange {
 } // namespace
 
 template <typename IndexT>
-void IndexIDMapTemplate<IndexT>::searchEx(
+void IndexIDMapTemplate<IndexT>::search_ex(
         idx_t n,
         const void* x,
         NumericType numeric_type,
@@ -193,7 +194,7 @@ void IndexIDMapTemplate<IndexT>::searchEx(
             sel_change.set(params_non_const, &this_idtrans);
         }
     }
-    index->searchEx(n, x, numeric_type, k, distances, labels, params);
+    index->search_ex(n, x, numeric_type, k, distances, labels, params);
     idx_t* li = labels;
 #pragma omp parallel for
     for (idx_t i = 0; i < n * k; i++) {
@@ -209,7 +210,7 @@ void IndexIDMapTemplate<IndexT>::search(
         typename IndexT::distance_t* distances,
         idx_t* labels,
         const SearchParameters* params) const {
-    searchEx(
+    search_ex(
             n,
             static_cast<const void*>(x),
             component_t_to_numeric<typename IndexT::component_t>(),
@@ -301,13 +302,13 @@ IndexIDMap2Template<IndexT>::IndexIDMap2Template(IndexT* index)
         : IndexIDMapTemplate<IndexT>(index) {}
 
 template <typename IndexT>
-void IndexIDMap2Template<IndexT>::add_with_idsEx(
+void IndexIDMap2Template<IndexT>::add_with_ids_ex(
         idx_t n,
         const void* x,
         NumericType numeric_type,
         const idx_t* xids) {
     size_t prev_ntotal = this->ntotal;
-    IndexIDMapTemplate<IndexT>::add_with_idsEx(n, x, numeric_type, xids);
+    IndexIDMapTemplate<IndexT>::add_with_ids_ex(n, x, numeric_type, xids);
     for (size_t i = prev_ntotal; i < this->ntotal; i++) {
         rev_map[this->id_map[i]] = i;
     }
@@ -318,7 +319,7 @@ void IndexIDMap2Template<IndexT>::add_with_ids(
         idx_t n,
         const typename IndexT::component_t* x,
         const idx_t* xids) {
-    add_with_idsEx(
+    add_with_ids_ex(
             n,
             static_cast<const void*>(x),
             component_t_to_numeric<typename IndexT::component_t>(),

data/vendor/faiss/faiss/IndexIDMap.h

@@ -23,7 +23,7 @@ struct IndexIDMapTemplate : IndexT {
     using distance_t = typename IndexT::distance_t;
 
     IndexT* index = nullptr; ///! the sub-index
-    bool own_fields = false; ///! whether pointers are deleted in destructo
+    bool own_fields = false; ///! whether pointers are deleted in destructor
     std::vector<idx_t> id_map;
 
     explicit IndexIDMapTemplate(IndexT* index);
@@ -31,7 +31,7 @@ struct IndexIDMapTemplate : IndexT {
     /// @param xids if non-null, ids to store for the vectors (size n)
     void add_with_ids(idx_t n, const component_t* x, const idx_t* xids)
             override;
-    void add_with_idsEx(
+    void add_with_ids_ex(
             idx_t n,
             const void* x,
             NumericType numeric_type,
@@ -39,7 +39,7 @@ struct IndexIDMapTemplate : IndexT {
 
     /// this will fail. Use add_with_ids
     void add(idx_t n, const component_t* x) override;
-    void addEx(idx_t n, const void* x, NumericType numeric_type) override;
+    void add_ex(idx_t n, const void* x, NumericType numeric_type) override;
 
     void search(
             idx_t n,
@@ -48,7 +48,7 @@ struct IndexIDMapTemplate : IndexT {
             distance_t* distances,
             idx_t* labels,
             const SearchParameters* params = nullptr) const override;
-    void searchEx(
+    void search_ex(
             idx_t n,
             const void* x,
             NumericType numeric_type,
@@ -58,7 +58,7 @@ struct IndexIDMapTemplate : IndexT {
             const SearchParameters* params = nullptr) const override;
 
     void train(idx_t n, const component_t* x) override;
-    void trainEx(idx_t n, const void* x, NumericType numeric_type) override;
+    void train_ex(idx_t n, const void* x, NumericType numeric_type) override;
 
     void reset() override;
 
@@ -104,7 +104,7 @@ struct IndexIDMap2Template : IndexIDMapTemplate<IndexT> {
 
     void add_with_ids(idx_t n, const component_t* x, const idx_t* xids)
             override;
-    void add_with_idsEx(
+    void add_with_ids_ex(
             idx_t n,
             const void* x,
             NumericType numeric_type,

data/vendor/faiss/faiss/IndexIVF.cpp

@@ -506,7 +506,7 @@ void IndexIVF::search_preassigned(
         };
 
         // single list scan using the current scanner (with query
-        // set porperly) and storing results in simi and idxi
+        // set properly) and storing results in simi and idxi
         auto scan_one_list = [&](idx_t key,
                                  float coarse_dis_i,
                                  float* simi,

data/vendor/faiss/faiss/IndexIVF.h

@@ -160,7 +160,7 @@ struct IndexIVFInterface : Level1Quantizer {
  * index maps to a list (aka inverted list or posting list), where the
  * id of the vector is stored.
  *
- * The inverted list object is required only after trainng. If none is
+ * The inverted list object is required only after training. If none is
  * set externally, an ArrayInvertedLists is used automatically.
  *
  * At search time, the vector to be searched is also quantized, and
@@ -171,7 +171,7 @@ struct IndexIVFInterface : Level1Quantizer {
  * lists are visited.
  *
  * Sub-classes implement a post-filtering of the index that refines
- * the distance estimation from the query to databse vectors.
+ * the distance estimation from the query to database vectors.
  */
 struct IndexIVF : Index, IndexIVFInterface {
     /// Access to the actual data
@@ -497,12 +497,12 @@ struct InvertedListScanner {
     /// compute a single query-to-code distance
     virtual float distance_to_code(const uint8_t* code) const = 0;
 
-    /** scan a set of codes, compute distances to current query and
+    /** scan a set of codes, compute distances to current query, and
      * update heap of results if necessary. Default implementation
      * calls distance_to_code.
      *
-     * @param n      number of codes to scan
-     * @param codes  codes to scan (n * code_size)
+     * @param n          number of codes to scan
+     * @param codes      codes to scan (n * code_size)
      * @param ids        corresponding ids (ignored if store_pairs)
      * @param distances  heap distances (size k)
      * @param labels     heap labels (size k)

data/vendor/faiss/faiss/IndexIVFAdditiveQuantizer.cpp

@@ -198,7 +198,7 @@ struct AQInvertedListScanner : InvertedListScanner {
         }
     }
 
-    ~AQInvertedListScanner() = default;
+    ~AQInvertedListScanner() override = default;
 };
 
 template <bool is_IP>

data/vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.cpp

@@ -14,6 +14,7 @@
 
 #include <faiss/impl/AuxIndexStructures.h>
 #include <faiss/impl/FaissAssert.h>
+#include <faiss/impl/FastScanDistancePostProcessing.h>
 #include <faiss/impl/LookupTableScaler.h>
 #include <faiss/impl/pq4_fast_scan.h>
 #include <faiss/invlists/BlockInvertedLists.h>
@@ -212,7 +213,9 @@ void IndexIVFAdditiveQuantizerFastScan::estimate_norm_scale(
     size_t index_nprobe = nprobe;
     nprobe = 1;
     CoarseQuantized cq{index_nprobe, coarse_dis.data(), coarse_ids.data()};
-    compute_LUT(n, x, cq, dis_tables, biases);
+    FastScanDistancePostProcessing empty_context{};
+
+    compute_LUT(n, x, cq, dis_tables, biases, empty_context);
     nprobe = index_nprobe;
 
     float scale = 0;
@@ -314,8 +317,10 @@ void IndexIVFAdditiveQuantizerFastScan::search(
     }
 
     NormTableScaler scaler(norm_scale);
+    FastScanDistancePostProcessing context;
+    context.norm_scaler = &scaler;
     IndexIVFFastScan::CoarseQuantized cq{nprobe};
-    search_dispatch_implem(n, x, k, distances, labels, cq, &scaler);
+    search_dispatch_implem(n, x, k, distances, labels, cq, context);
 }
 
 /*********************************************************
@@ -383,7 +388,8 @@ void IndexIVFAdditiveQuantizerFastScan::compute_LUT(
         const float* x,
         const CoarseQuantized& cq,
         AlignedTable<float>& dis_tables,
-        AlignedTable<float>& biases) const {
+        AlignedTable<float>& biases,
+        const FastScanDistancePostProcessing&) const {
     const size_t dim12 = ksub * M;
     const size_t ip_dim12 = aq->M * ksub;
     const size_t nprobe = cq.nprobe;

data/vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.h

@@ -12,6 +12,7 @@
 #include <faiss/IndexIVFAdditiveQuantizer.h>
 #include <faiss/IndexIVFFastScan.h>
 #include <faiss/impl/AdditiveQuantizer.h>
+#include <faiss/impl/FastScanDistancePostProcessing.h>
 #include <faiss/impl/ProductAdditiveQuantizer.h>
 #include <faiss/utils/AlignedTable.h>
 
@@ -101,7 +102,8 @@ struct IndexIVFAdditiveQuantizerFastScan : IndexIVFFastScan {
             const float* x,
             const CoarseQuantized& cq,
             AlignedTable<float>& dis_tables,
-            AlignedTable<float>& biases) const override;
+            AlignedTable<float>& biases,
+            const FastScanDistancePostProcessing& context) const override;
 };
 
 struct IndexIVFLocalSearchQuantizerFastScan