faiss 0.3.0 → 0.3.1

data/vendor/faiss/faiss/index_factory.cpp

@@ -216,7 +216,7 @@ VectorTransform* parse_VectorTransform(const std::string& description, int d) {
         return new RemapDimensionsTransform(d, std::max(d_out, d), false);
     }
     return nullptr;
-};
+}
 
 /***************************************************************
  * Parse IndexIVF
@@ -440,11 +440,13 @@ IndexHNSW* parse_IndexHNSW(
     if (match("Flat|")) {
         return new IndexHNSWFlat(d, hnsw_M, mt);
     }
-    if (match("PQ([0-9]+)(np)?")) {
+
+    if (match("PQ([0-9]+)(x[0-9]+)?(np)?")) {
         int M = std::stoi(sm[1].str());
-        IndexHNSWPQ* ipq = new IndexHNSWPQ(d, M, hnsw_M);
+        int nbit = mres_to_int(sm[2], 8, 1);
+        IndexHNSWPQ* ipq = new IndexHNSWPQ(d, M, hnsw_M, nbit);
         dynamic_cast<IndexPQ*>(ipq->storage)->do_polysemous_training =
-                sm[2].str() != "np";
+                sm[3].str() != "np";
         return ipq;
     }
     if (match(sq_pattern)) {
@@ -490,11 +492,12 @@ IndexNSG* parse_IndexNSG(
     if (match("Flat|")) {
         return new IndexNSGFlat(d, nsg_R, mt);
     }
-    if (match("PQ([0-9]+)(np)?")) {
+    if (match("PQ([0-9]+)(x[0-9]+)?(np)?")) {
         int M = std::stoi(sm[1].str());
-        IndexNSGPQ* ipq = new IndexNSGPQ(d, M, nsg_R);
+        int nbit = mres_to_int(sm[2], 8, 1);
+        IndexNSGPQ* ipq = new IndexNSGPQ(d, M, nsg_R, nbit);
         dynamic_cast<IndexPQ*>(ipq->storage)->do_polysemous_training =
-                sm[2].str() != "np";
+                sm[3].str() != "np";
         return ipq;
     }
     if (match(sq_pattern)) {

data/vendor/faiss/faiss/invlists/DirectMap.cpp

@@ -199,7 +199,7 @@ size_t DirectMap::remove_ids(const IDSelector& sel, InvertedLists* invlists) {
                             last_id,
                             ScopedCodes(invlists, list_no, last).get());
                     // update hash entry for last element
-                    hashtable[last_id] = list_no << 32 | offset;
+                    hashtable[last_id] = lo_build(list_no, offset);
                 }
                 invlists->resize(list_no, last);
                 nremove++;

data/vendor/faiss/faiss/invlists/InvertedLists.cpp

@@ -28,11 +28,12 @@ InvertedLists::InvertedLists(size_t nlist, size_t code_size)
 
 InvertedLists::~InvertedLists() {}
 
-bool InvertedLists::is_empty(size_t list_no) const {
-    return use_iterator
-            ? !std::unique_ptr<InvertedListsIterator>(get_iterator(list_no))
-                       ->is_available()
-            : list_size(list_no) == 0;
+bool InvertedLists::is_empty(size_t list_no, void* inverted_list_context)
+        const {
+    return use_iterator ? !std::unique_ptr<InvertedListsIterator>(
+                                   get_iterator(list_no, inverted_list_context))
+                                   ->is_available()
+                        : list_size(list_no) == 0;
 }
 
 idx_t InvertedLists::get_single_id(size_t list_no, size_t offset) const {
@@ -58,7 +59,8 @@ const uint8_t* InvertedLists::get_single_code(size_t list_no, size_t offset)
 size_t InvertedLists::add_entry(
         size_t list_no,
         idx_t theid,
-        const uint8_t* code) {
+        const uint8_t* code,
+        void* /*inverted_list_context*/) {
     return add_entries(list_no, 1, &theid, code);
 }
 
@@ -76,7 +78,9 @@ void InvertedLists::reset() {
     }
 }
 
-InvertedListsIterator* InvertedLists::get_iterator(size_t /*list_no*/) const {
+InvertedListsIterator* InvertedLists::get_iterator(
+        size_t /*list_no*/,
+        void* /*inverted_list_context*/) const {
     FAISS_THROW_MSG("get_iterator is not supported");
 }
 
@@ -287,6 +291,20 @@ void ArrayInvertedLists::update_entries(
     memcpy(&codes[list_no][offset * code_size], codes_in, code_size * n_entry);
 }
 
+void ArrayInvertedLists::permute_invlists(const idx_t* map) {
+    std::vector<std::vector<uint8_t>> new_codes(nlist);
+    std::vector<std::vector<idx_t>> new_ids(nlist);
+
+    for (size_t i = 0; i < nlist; i++) {
+        size_t o = map[i];
+        FAISS_THROW_IF_NOT(o < nlist);
+        std::swap(new_codes[i], codes[o]);
+        std::swap(new_ids[i], ids[o]);
+    }
+    std::swap(codes, new_codes);
+    std::swap(ids, new_ids);
+}
+
 ArrayInvertedLists::~ArrayInvertedLists() {}
 
 /*****************************************************************
@@ -423,7 +441,7 @@ idx_t translate_list_no(const SliceInvertedLists* sil, idx_t list_no) {
     return list_no + sil->i0;
 }
 
-}; // namespace
+} // namespace
 
 SliceInvertedLists::SliceInvertedLists(
         const InvertedLists* il,
@@ -508,7 +526,7 @@ idx_t sum_il_sizes(int nil, const InvertedLists** ils_in) {
     return tot;
 }
 
-}; // namespace
+} // namespace
 
 VStackInvertedLists::VStackInvertedLists(int nil, const InvertedLists** ils_in)
         : ReadOnlyInvertedLists(

data/vendor/faiss/faiss/invlists/InvertedLists.h

@@ -51,13 +51,15 @@ struct InvertedLists {
      *  Read only functions */
 
     // check if the list is empty
-    bool is_empty(size_t list_no) const;
+    bool is_empty(size_t list_no, void* inverted_list_context) const;
 
     /// get the size of a list
     virtual size_t list_size(size_t list_no) const = 0;
 
     /// get iterable for lists that use_iterator
-    virtual InvertedListsIterator* get_iterator(size_t list_no) const;
+    virtual InvertedListsIterator* get_iterator(
+            size_t list_no,
+            void* inverted_list_context) const;
 
     /** get the codes for an inverted list
      * must be released by release_codes
@@ -94,7 +96,11 @@ struct InvertedLists {
      * writing functions     */
 
     /// add one entry to an inverted list
-    virtual size_t add_entry(size_t list_no, idx_t theid, const uint8_t* code);
+    virtual size_t add_entry(
+            size_t list_no,
+            idx_t theid,
+            const uint8_t* code,
+            void* inverted_list_context = nullptr);
 
     virtual size_t add_entries(
             size_t list_no,
@@ -253,6 +259,9 @@ struct ArrayInvertedLists : InvertedLists {
 
     void resize(size_t list_no, size_t new_size) override;
 
+    /// permute the inverted lists, map maps new_id to old_id
+    void permute_invlists(const idx_t* map);
+
     ~ArrayInvertedLists() override;
 };
 

data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.cpp

@@ -407,7 +407,7 @@ void OnDiskInvertedLists::update_entries(
     FAISS_THROW_IF_NOT(!read_only);
     if (n_entry == 0)
         return;
-    const List& l = lists[list_no];
+    [[maybe_unused]] const List& l = lists[list_no];
     assert(n_entry + offset <= l.size);
     idx_t* ids = const_cast<idx_t*>(get_ids(list_no));
     memcpy(ids + offset, ids_in, sizeof(ids_in[0]) * n_entry);
@@ -524,7 +524,7 @@ void OnDiskInvertedLists::free_slot(size_t offset, size_t capacity) {
         it++;
     }
 
-    size_t inf = 1UL << 60;
+    size_t inf = ((size_t)1) << 60;
 
     size_t end_prev = inf;
     if (it != slots.begin()) {
@@ -533,7 +533,7 @@ void OnDiskInvertedLists::free_slot(size_t offset, size_t capacity) {
         end_prev = prev->offset + prev->capacity;
     }
 
-    size_t begin_next = 1L << 60;
+    size_t begin_next = ((size_t)1) << 60;
     if (it != slots.end()) {
         begin_next = it->offset;
     }

data/vendor/faiss/faiss/python/python_callbacks.cpp

@@ -22,7 +22,7 @@ struct PyThreadLock {
     }
 };
 
-}; // namespace
+} // namespace
 
 /***********************************************************
  * Callbacks for IO reader and writer

data/vendor/faiss/faiss/utils/Heap.cpp

@@ -93,7 +93,7 @@ void HeapArray<C>::addn_query_subset_with_ids(
     }
 #pragma omp parallel for if (nsubset * nj > 100000)
     for (int64_t si = 0; si < nsubset; si++) {
-        T i = subset[si];
+        TI i = subset[si];
         T* __restrict simi = get_val(i);
         TI* __restrict idxi = get_ids(i);
         const T* ip_line = vin + si * nj;
@@ -136,6 +136,8 @@ void HeapArray<C>::per_line_extrema(T* out_val, TI* out_ids) const {
 
 template struct HeapArray<CMin<float, int64_t>>;
 template struct HeapArray<CMax<float, int64_t>>;
+template struct HeapArray<CMin<float, int32_t>>;
+template struct HeapArray<CMax<float, int32_t>>;
 template struct HeapArray<CMin<int, int64_t>>;
 template struct HeapArray<CMax<int, int64_t>>;
 

data/vendor/faiss/faiss/utils/WorkerThread.h

@@ -9,6 +9,7 @@
 
 #include <condition_variable>
 #include <deque>
+#include <functional>
 #include <future>
 #include <thread>