faiss 0.2.4 → 0.2.5

data/vendor/faiss/faiss/IndexIVFPQFastScan.h

@@ -9,6 +9,7 @@
 
 #include <memory>
 
+#include <faiss/IndexIVFFastScan.h>
 #include <faiss/IndexIVFPQ.h>
 #include <faiss/impl/ProductQuantizer.h>
 #include <faiss/utils/AlignedTable.h>
@@ -31,36 +32,20 @@ namespace faiss {
  * 13: idem, collect results in reservoir
  */
 
-struct IndexIVFPQFastScan : IndexIVF {
-    bool by_residual;    ///< Encode residual or plain vector?
+struct IndexIVFPQFastScan : IndexIVFFastScan {
     ProductQuantizer pq; ///< produces the codes
 
-    // size of the kernel
-    int bbs; // set at build time
-
-    // M rounded up to a multiple of 2
-    size_t M2;
-
     /// precomputed tables management
     int use_precomputed_table = 0;
     /// if use_precompute_table size (nlist, pq.M, pq.ksub)
     AlignedTable<float> precomputed_table;
 
-    // search-time implementation
-    int implem = 0;
-    // skip some parts of the computation (for timing)
-    int skip = 0;
-
-    // batching factors at search time (0 = default)
-    int qbs = 0;
-    size_t qbs2 = 0;
-
     IndexIVFPQFastScan(
             Index* quantizer,
             size_t d,
             size_t nlist,
             size_t M,
-            size_t nbits_per_idx,
+            size_t nbits,
             MetricType metric = METRIC_L2,
             int bbs = 32);
 
@@ -69,9 +54,6 @@ struct IndexIVFPQFastScan : IndexIVF {
     // built from an IndexIVFPQ
     explicit IndexIVFPQFastScan(const IndexIVFPQ& orig, int bbs = 32);
 
-    /// orig's inverted lists (for debugging)
-    InvertedLists* orig_invlists = nullptr;
-
     void train_residual(idx_t n, const float* x) override;
 
     /// build precomputed table, possibly updating use_precomputed_table
@@ -86,106 +68,19 @@ struct IndexIVFPQFastScan : IndexIVF {
             uint8_t* codes,
             bool include_listno = false) const override;
 
-    void add_with_ids(idx_t n, const float* x, const idx_t* xids) override;
-
-    void search(
-            idx_t n,
-            const float* x,
-            idx_t k,
-            float* distances,
-            idx_t* labels) const override;
-
     // prepare look-up tables
 
+    bool lookup_table_is_3d() const override;
+
     void compute_LUT(
             size_t n,
             const float* x,
             const idx_t* coarse_ids,
             const float* coarse_dis,
             AlignedTable<float>& dis_tables,
-            AlignedTable<float>& biases) const;
-
-    void compute_LUT_uint8(
-            size_t n,
-            const float* x,
-            const idx_t* coarse_ids,
-            const float* coarse_dis,
-            AlignedTable<uint8_t>& dis_tables,
-            AlignedTable<uint16_t>& biases,
-            float* normalizers) const;
-
-    // internal search funcs
+            AlignedTable<float>& biases) const override;
 
-    template <bool is_max>
-    void search_dispatch_implem(
-            idx_t n,
-            const float* x,
-            idx_t k,
-            float* distances,
-            idx_t* labels) const;
-
-    template <class C>
-    void search_implem_1(
-            idx_t n,
-            const float* x,
-            idx_t k,
-            float* distances,
-            idx_t* labels) const;
-
-    template <class C>
-    void search_implem_2(
-            idx_t n,
-            const float* x,
-            idx_t k,
-            float* distances,
-            idx_t* labels) const;
-
-    // implem 10 and 12 are not multithreaded internally, so
-    // export search stats
-    template <class C>
-    void search_implem_10(
-            idx_t n,
-            const float* x,
-            idx_t k,
-            float* distances,
-            idx_t* labels,
-            int impl,
-            size_t* ndis_out,
-            size_t* nlist_out) const;
-
-    template <class C>
-    void search_implem_12(
-            idx_t n,
-            const float* x,
-            idx_t k,
-            float* distances,
-            idx_t* labels,
-            int impl,
-            size_t* ndis_out,
-            size_t* nlist_out) const;
+    void sa_decode(idx_t n, const uint8_t* bytes, float* x) const override;
 };
 
-struct IVFFastScanStats {
-    uint64_t times[10];
-    uint64_t t_compute_distance_tables, t_round;
-    uint64_t t_copy_pack, t_scan, t_to_flat;
-    uint64_t reservoir_times[4];
-
-    double Mcy_at(int i) {
-        return times[i] / (1000 * 1000.0);
-    }
-
-    double Mcy_reservoir_at(int i) {
-        return reservoir_times[i] / (1000 * 1000.0);
-    }
-    IVFFastScanStats() {
-        reset();
-    }
-    void reset() {
-        memset(this, 0, sizeof(*this));
-    }
-};
-
-FAISS_API extern IVFFastScanStats IVFFastScan_stats;
-
 } // namespace faiss

data/vendor/faiss/faiss/IndexIVFPQR.cpp

@@ -201,11 +201,11 @@ void IndexIVFPQR::reconstruct_from_offset(
     }
 }
 
-void IndexIVFPQR::merge_from(IndexIVF& other_in, idx_t add_id) {
-    IndexIVFPQR* other = dynamic_cast<IndexIVFPQR*>(&other_in);
+void IndexIVFPQR::merge_from(Index& otherIndex, idx_t add_id) {
+    IndexIVFPQR* other = dynamic_cast<IndexIVFPQR*>(&otherIndex);
     FAISS_THROW_IF_NOT(other);
 
-    IndexIVF::merge_from(other_in, add_id);
+    IndexIVF::merge_from(otherIndex, add_id);
 
     refine_codes.insert(
             refine_codes.end(),

data/vendor/faiss/faiss/IndexIVFPQR.h

@@ -51,7 +51,7 @@ struct IndexIVFPQR : IndexIVFPQ {
     void reconstruct_from_offset(int64_t list_no, int64_t offset, float* recons)
             const override;
 
-    void merge_from(IndexIVF& other, idx_t add_id) override;
+    void merge_from(Index& otherIndex, idx_t add_id) override;
 
     void search_preassigned(
             idx_t n,

data/vendor/faiss/faiss/IndexIVFSpectralHash.cpp

@@ -293,7 +293,9 @@ struct IVFScanner : InvertedListScanner {
 } // anonymous namespace
 
 InvertedListScanner* IndexIVFSpectralHash::get_InvertedListScanner(
-        bool store_pairs) const {
+        bool store_pairs,
+        const IDSelector* sel) const {
+    FAISS_THROW_IF_NOT(!sel);
     switch (code_size) {
 #define HANDLE_CODE_SIZE(cs) \
     case cs:                 \

data/vendor/faiss/faiss/IndexIVFSpectralHash.h

@@ -70,7 +70,8 @@ struct IndexIVFSpectralHash : IndexIVF {
             bool include_listnos = false) const override;
 
     InvertedListScanner* get_InvertedListScanner(
-            bool store_pairs) const override;
+            bool store_pairs,
+            const IDSelector* sel) const override;
 
     /** replace the vector transform for an empty (and possibly untrained) index
      */

data/vendor/faiss/faiss/IndexLSH.cpp

@@ -103,9 +103,11 @@ void IndexLSH::search(
         const float* x,
         idx_t k,
         float* distances,
-        idx_t* labels) const {
+        idx_t* labels,
+        const SearchParameters* params) const {
+    FAISS_THROW_IF_NOT_MSG(
+            !params, "search params not supported for this index");
     FAISS_THROW_IF_NOT(k > 0);
-
     FAISS_THROW_IF_NOT(is_trained);
     const float* xt = apply_preprocess(n, x);
     ScopeDeleter<float> del(xt == x ? nullptr : xt);

data/vendor/faiss/faiss/IndexLSH.h

@@ -49,7 +49,8 @@ struct IndexLSH : IndexFlatCodes {
             const float* x,
             idx_t k,
             float* distances,
-            idx_t* labels) const override;
+            idx_t* labels,
+            const SearchParameters* params = nullptr) const override;
 
     /// transfer the thresholds to a pre-processing stage (and unset
     /// train_thresholds)

data/vendor/faiss/faiss/IndexLattice.cpp

@@ -118,7 +118,13 @@ void IndexLattice::add(idx_t, const float*) {
     FAISS_THROW_MSG("not implemented");
 }
 
-void IndexLattice::search(idx_t, const float*, idx_t, float*, idx_t*) const {
+void IndexLattice::search(
+        idx_t,
+        const float*,
+        idx_t,
+        float*,
+        idx_t*,
+        const SearchParameters*) const {
     FAISS_THROW_MSG("not implemented");
 }
 

data/vendor/faiss/faiss/IndexLattice.h

@@ -54,7 +54,9 @@ struct IndexLattice : Index {
             const float* x,
             idx_t k,
             float* distances,
-            idx_t* labels) const override;
+            idx_t* labels,
+            const SearchParameters* params = nullptr) const override;
+
     void reset() override;
 };
 

data/vendor/faiss/faiss/IndexNNDescent.cpp

@@ -135,9 +135,10 @@ void IndexNNDescent::search(
         const float* x,
         idx_t k,
         float* distances,
-        idx_t* labels) const
-
-{
+        idx_t* labels,
+        const SearchParameters* params) const {
+    FAISS_THROW_IF_NOT_MSG(
+            !params, "search params not supported for this index");
     FAISS_THROW_IF_NOT_MSG(
             storage,
             "Please use IndexNNDescentFlat (or variants) "

data/vendor/faiss/faiss/IndexNNDescent.h

@@ -53,7 +53,8 @@ struct IndexNNDescent : Index {
             const float* x,
             idx_t k,
             float* distances,
-            idx_t* labels) const override;
+            idx_t* labels,
+            const SearchParameters* params = nullptr) const override;
 
     void reconstruct(idx_t key, float* recons) const override;
 

data/vendor/faiss/faiss/IndexNSG.cpp

@@ -78,9 +78,10 @@ void IndexNSG::search(
         const float* x,
         idx_t k,
         float* distances,
-        idx_t* labels) const
-
-{
+        idx_t* labels,
+        const SearchParameters* params) const {
+    FAISS_THROW_IF_NOT_MSG(
+            !params, "search params not supported for this index");
     FAISS_THROW_IF_NOT_MSG(
             storage,
             "Please use IndexNSGFlat (or variants) instead of IndexNSG directly");
@@ -298,4 +299,37 @@ IndexNSGFlat::IndexNSGFlat(int d, int R, MetricType metric)
     is_trained = true;
 }
 
+/**************************************************************
+ * IndexNSGPQ implementation
+ **************************************************************/
+
+IndexNSGPQ::IndexNSGPQ() {}
+
+IndexNSGPQ::IndexNSGPQ(int d, int pq_m, int M)
+        : IndexNSG(new IndexPQ(d, pq_m, 8), M) {
+    own_fields = true;
+    is_trained = false;
+}
+
+void IndexNSGPQ::train(idx_t n, const float* x) {
+    IndexNSG::train(n, x);
+    (dynamic_cast<IndexPQ*>(storage))->pq.compute_sdc_table();
+}
+
+/**************************************************************
+ * IndexNSGSQ implementation
+ **************************************************************/
+
+IndexNSGSQ::IndexNSGSQ(
+        int d,
+        ScalarQuantizer::QuantizerType qtype,
+        int M,
+        MetricType metric)
+        : IndexNSG(new IndexScalarQuantizer(d, qtype, metric), M) {
+    is_trained = false;
+    own_fields = true;
+}
+
+IndexNSGSQ::IndexNSGSQ() {}
+
 } // namespace faiss

data/vendor/faiss/faiss/IndexNSG.h

@@ -13,6 +13,8 @@
 
 #include <faiss/IndexFlat.h>
 #include <faiss/IndexNNDescent.h>
+#include <faiss/IndexPQ.h>
+#include <faiss/IndexScalarQuantizer.h>
 #include <faiss/impl/NSG.h>
 #include <faiss/utils/utils.h>
 
@@ -64,7 +66,8 @@ struct IndexNSG : Index {
             const float* x,
             idx_t k,
             float* distances,
-            idx_t* labels) const override;
+            idx_t* labels,
+            const SearchParameters* params = nullptr) const override;
 
     void reconstruct(idx_t key, float* recons) const override;
 
@@ -82,4 +85,25 @@ struct IndexNSGFlat : IndexNSG {
     IndexNSGFlat(int d, int R, MetricType metric = METRIC_L2);
 };
 
+/** PQ index topped with with a NSG structure to access elements
+ *  more efficiently.
+ */
+struct IndexNSGPQ : IndexNSG {
+    IndexNSGPQ();
+    IndexNSGPQ(int d, int pq_m, int M);
+    void train(idx_t n, const float* x) override;
+};
+
+/** SQ index topped with with a NSG structure to access elements
+ *  more efficiently.
+ */
+struct IndexNSGSQ : IndexNSG {
+    IndexNSGSQ();
+    IndexNSGSQ(
+            int d,
+            ScalarQuantizer::QuantizerType qtype,
+            int M,
+            MetricType metric = METRIC_L2);
+};
+
 } // namespace faiss