faiss 0.5.3 → 0.6.1

data/vendor/faiss/faiss/impl/svs_io.cpp

@@ -50,7 +50,8 @@ int WriterStreambuf::overflow(int ch) {
     return wrote == 1 ? ch : traits_type::eof();
 }
 
-ReaderStreambuf::ReaderStreambuf(IOReader* rr) : r(rr), single_char_buffer(0) {
+ReaderStreambuf::ReaderStreambuf(IOReader* rr, size_t limit)
+        : r(rr), single_char_buffer(0), per_read_byte_limit(limit) {
     // Initialize with empty get area
     setg(nullptr, nullptr, nullptr);
 }
@@ -77,9 +78,17 @@ std::streamsize ReaderStreambuf::xsgetn(char* s, std::streamsize n) {
     if (n <= 0) {
         return 0;
     }
+    size_t nbytes = n;
 
-    size_t got = (*r)(s, 1, n);
-    return static_cast<std::streamsize>(got);
+    // Per-read byte limit: reject individual reads that exceed the limit,
+    // matching READVECTOR semantics where each vector allocation is
+    // independently checked against deserialization_vector_byte_limit.
+    if (per_read_byte_limit > 0 && nbytes >= per_read_byte_limit) {
+        return 0;
+    }
+
+    size_t got = (*r)(s, 1, nbytes);
+    return got;
 }
 
 } // namespace svs_io

data/vendor/faiss/faiss/impl/svs_io.h

@@ -53,9 +53,15 @@ struct WriterStreambuf : std::streambuf {
 // without intermediate buffering.
 struct ReaderStreambuf : std::streambuf {
     IOReader* r;
-    char single_char_buffer; // Single-byte buffer for underflow() operations
+    char single_char_buffer;    // Single-byte buffer for underflow() operations
+    size_t per_read_byte_limit; // Max bytes per individual read (0 = unlimited)
 
-    explicit ReaderStreambuf(IOReader* rr);
+    /// Construct with an optional per-read byte limit matching READVECTOR
+    /// semantics. Each individual xsgetn call that meets or exceeds the
+    /// limit returns 0, matching how READVECTOR rejects individual vectors
+    /// whose allocation would exceed deserialization_vector_byte_limit.
+    /// A limit of 0 means unlimited.
+    explicit ReaderStreambuf(IOReader* rr, size_t limit = 0);
     ~ReaderStreambuf() override;
 
    protected:

data/vendor/faiss/faiss/index_factory.cpp

@@ -45,7 +45,6 @@
 #include <faiss/IndexRefine.h>
 #include <faiss/IndexRowwiseMinMax.h>
 #include <faiss/IndexScalarQuantizer.h>
-#include <faiss/MetaIndexes.h>
 #include <faiss/VectorTransform.h>
 
 #include <faiss/IndexBinaryFlat.h>
@@ -55,6 +54,9 @@
 
 #ifdef FAISS_ENABLE_SVS
 #include <faiss/svs/IndexSVSFlat.h>
+#include <faiss/svs/IndexSVSIVF.h>
+#include <faiss/svs/IndexSVSIVFLVQ.h>
+#include <faiss/svs/IndexSVSIVFLeanVec.h>
 #include <faiss/svs/IndexSVSVamana.h>
 #include <faiss/svs/IndexSVSVamanaLVQ.h>
 #include <faiss/svs/IndexSVSVamanaLeanVec.h>
@@ -91,10 +93,10 @@ void find_matching_parentheses(
         int begin = 0) {
     int st = 0;
     i0 = i1 = 0;
-    for (int i = begin; i < s.length(); i++) {
+    for (size_t i = begin; i < s.length(); i++) {
         if (s[i] == '(') {
             if (st == 0) {
-                i0 = i;
+                i0 = static_cast<int>(i);
             }
             st++;
         }
@@ -102,7 +104,7 @@ void find_matching_parentheses(
         if (s[i] == ')') {
             st--;
             if (st == 0) {
-                i1 = i;
+                i1 = static_cast<int>(i);
                 return;
             }
             if (st < 0) {
@@ -128,6 +130,11 @@ char get_trains_alone(const Index* coarse_quantizer) {
     if (dynamic_cast<const IndexHNSWFlat*>(coarse_quantizer)) {
         return 2;
     }
+#ifdef FAISS_ENABLE_SVS
+    if (dynamic_cast<const IndexSVSVamana*>(coarse_quantizer)) {
+        return 2;
+    }
+#endif
     return 2; // for complicated indexes, we assume they can't be used as a
               // kmeans index
 }
@@ -155,9 +162,15 @@ std::map<std::string, ScalarQuantizer::QuantizerType> sq_types = {
         {"SQbf16", ScalarQuantizer::QT_bf16},
         {"SQ8_direct_signed", ScalarQuantizer::QT_8bit_direct_signed},
         {"SQ8_direct", ScalarQuantizer::QT_8bit_direct},
+        {"SQ0", ScalarQuantizer::QT_0bit},
+        {"SQtqmse1", ScalarQuantizer::QT_1bit_tqmse},
+        {"SQtqmse2", ScalarQuantizer::QT_2bit_tqmse},
+        {"SQtqmse3", ScalarQuantizer::QT_3bit_tqmse},
+        {"SQtqmse4", ScalarQuantizer::QT_4bit_tqmse},
+        {"SQtqmse8", ScalarQuantizer::QT_8bit_tqmse},
 };
 const std::string sq_pattern =
-        "(SQ4|SQ8|SQ6|SQfp16|SQbf16|SQ8_direct_signed|SQ8_direct)";
+        "(SQ0|SQ4|SQ8|SQ6|SQfp16|SQbf16|SQ8_direct_signed|SQ8_direct|SQtqmse1|SQtqmse2|SQtqmse3|SQtqmse4|SQtqmse8)";
 
 std::map<std::string, AdditiveQuantizer::Search_type_t> aq_search_type = {
         {"_Nfloat", AdditiveQuantizer::ST_norm_float},
@@ -183,7 +196,7 @@ AdditiveQuantizer::Search_type_t aq_parse_search_type(
         return metric == METRIC_L2 ? AdditiveQuantizer::ST_decompress
                                    : AdditiveQuantizer::ST_LUT_nonorm;
     }
-    int pos = stok.rfind('_');
+    size_t pos = stok.rfind('_');
     return aq_search_type[stok.substr(pos)];
 }
 
@@ -206,7 +219,7 @@ std::vector<size_t> aq_parse_nbits(std::string stok) {
 
 VectorTransform* parse_VectorTransform(const std::string& description, int d) {
     std::smatch sm;
-    auto match = [&sm, description](std::string pattern) {
+    auto match = [&sm, description](const std::string& pattern) {
         return re_match(description, pattern, sm);
     };
     if (match("PCA(W?)(R?)([0-9]+)")) {
@@ -220,6 +233,9 @@ VectorTransform* parse_VectorTransform(const std::string& description, int d) {
     if (match("RR([0-9]+)?")) {
         return new RandomRotationMatrix(d, mres_to_int(sm[1], d));
     }
+    if (match("HR([0-9]+)?")) {
+        return new HadamardRotation(d, mres_to_int(sm[1], 12345));
+    }
     if (match("ITQ([0-9]+)?")) {
         return new ITQTransform(d, mres_to_int(sm[1], d), sm[1].length() > 0);
     }
@@ -261,7 +277,7 @@ Index* parse_coarse_quantizer(
         size_t& nlist,
         bool& use_2layer) {
     std::smatch sm;
-    auto match = [&sm, description](std::string pattern) {
+    auto match = [&sm, description](const std::string& pattern) {
         return re_match(description, pattern, sm);
     };
     use_2layer = false;
@@ -288,10 +304,34 @@ Index* parse_coarse_quantizer(
         int R = std::stoi(sm[2]);
         return new IndexNSGFlat(d, R, mt);
     }
+#ifdef FAISS_ENABLE_SVS
+    if (match("IVF([0-9]+[kM]?)_SVSVamana([0-9]*)(_.+)?")) {
+        nlist = parse_nlist(sm[1].str());
+        int degree = sm[2].length() > 0 ? std::stoi(sm[2]) : 32;
+        SVSStorageKind storage = SVSStorageKind::SVS_FP32;
+        if (sm[3].matched) {
+            std::string s = sm[3].str().substr(1);
+            if (s == "SQI8") {
+                storage = SVSStorageKind::SVS_SQI8;
+            } else if (s == "FP16") {
+                storage = SVSStorageKind::SVS_FP16;
+            } else if (s == "FP32") {
+                storage = SVSStorageKind::SVS_FP32;
+            } else {
+                FAISS_THROW_FMT(
+                        "unsupported SVSVamana coarse quantizer storage: %s",
+                        s.c_str());
+            }
+        }
+        return new IndexSVSVamana(d, degree, mt, storage);
+    }
+#endif
     if (match("IVF([0-9]+[kM]?)\\(Index([0-9])\\)")) {
         nlist = parse_nlist(sm[1].str());
         int no = std::stoi(sm[2].str());
-        FAISS_ASSERT(no >= 0 && no < parenthesis_indexes.size());
+        FAISS_ASSERT(
+                no >= 0 &&
+                static_cast<size_t>(no) < parenthesis_indexes.size());
         return parenthesis_indexes[no].release();
     }
 
@@ -337,9 +377,11 @@ IndexIVF* parse_IndexIVF(
     if (match("FlatDedup")) {
         return new IndexIVFFlatDedup(get_q(), d, nlist, mt, own_il);
     }
-    if (match("FlatPanorama([0-9]+)?")) {
+    if (match("FlatPanorama([0-9]+)?(_([0-9]+))?")) {
         int nlevels = mres_to_int(sm[1], 8); // default to 8 levels
-        return new IndexIVFFlatPanorama(get_q(), d, nlist, nlevels, mt, own_il);
+        int bs = mres_to_int(sm[3], 128);
+        return new IndexIVFFlatPanorama(
+                get_q(), d, nlist, nlevels, mt, own_il, bs);
     }
     if (match(sq_pattern)) {
         return new IndexIVFScalarQuantizer(
@@ -585,7 +627,10 @@ SVSStorageKind parse_lvq(const std::string& lvq_string) {
     if (lvq_string == "LVQ4x8") {
         return SVSStorageKind::SVS_LVQ4x8;
     }
-    FAISS_ASSERT(!"not supported SVS LVQ level");
+    if (lvq_string == "LVQ8x0") {
+        return SVSStorageKind::SVS_LVQ8x0;
+    }
+    FAISS_ASSERT(false && "not supported SVS LVQ level");
 }
 
 SVSStorageKind parse_leanvec(const std::string& leanvec_string) {
@@ -598,7 +643,7 @@ SVSStorageKind parse_leanvec(const std::string& leanvec_string) {
     if (leanvec_string == "LeanVec8x8") {
         return SVSStorageKind::SVS_LeanVec8x8;
     }
-    FAISS_ASSERT(!"not supported SVS Leanvec level");
+    FAISS_ASSERT(false && "not supported SVS Leanvec level");
 }
 
 Index* parse_svs_datatype(
@@ -610,43 +655,72 @@ Index* parse_svs_datatype(
     std::smatch sm;
 
     if (datatype_string.empty()) {
-        if (index_type == "Vamana")
+        if (index_type == "Vamana") {
             return new IndexSVSVamana(d, std::stoul(arg_string), mt);
-        if (index_type == "Flat")
+        }
+        if (index_type == "Flat") {
             return new IndexSVSFlat(d, mt);
-        FAISS_ASSERT(!"Unspported SVS index type");
+        }
+        if (index_type == "IVF") {
+            return new IndexSVSIVF(d, std::stoul(arg_string), mt);
+        }
+        FAISS_ASSERT(false && "Unsupported SVS index type");
     }
     if (re_match(datatype_string, "FP16", sm)) {
-        if (index_type == "Vamana")
+        if (index_type == "Vamana") {
             return new IndexSVSVamana(
                     d, std::stoul(arg_string), mt, SVSStorageKind::SVS_FP16);
-        FAISS_ASSERT(!"Unspported SVS index type for Float16");
+        }
+        if (index_type == "IVF") {
+            return new IndexSVSIVF(
+                    d, std::stoul(arg_string), mt, SVSStorageKind::SVS_FP16);
+        }
+        FAISS_ASSERT(false && "Unsupported SVS index type for Float16");
     }
     if (re_match(datatype_string, "SQI8", sm)) {
-        if (index_type == "Vamana")
+        if (index_type == "Vamana") {
             return new IndexSVSVamana(
                     d, std::stoul(arg_string), mt, SVSStorageKind::SVS_SQI8);
-        FAISS_ASSERT(!"Unspported SVS index type for SQI8");
+        }
+        if (index_type == "IVF") {
+            return new IndexSVSIVF(
+                    d, std::stoul(arg_string), mt, SVSStorageKind::SVS_SQI8);
+        }
+        FAISS_ASSERT(false && "Unsupported SVS index type for SQI8");
     }
     if (re_match(datatype_string, "(LVQ[0-9]+x[0-9]+)", sm)) {
-        if (index_type == "Vamana")
+        if (index_type == "Vamana") {
             return new IndexSVSVamanaLVQ(
                     d, std::stoul(arg_string), mt, parse_lvq(sm[0].str()));
-        FAISS_ASSERT(!"Unspported SVS index type for LVQ");
+        }
+        if (index_type == "IVF") {
+            return new IndexSVSIVFLVQ(
+                    d, std::stoul(arg_string), mt, parse_lvq(sm[0].str()));
+        }
+        FAISS_ASSERT(false && "Unsupported SVS index type for LVQ");
     }
     if (re_match(datatype_string, "(LeanVec[0-9]+x[0-9]+)(_[0-9]+)?", sm)) {
         std::string leanvec_d_string =
                 sm[2].length() > 0 ? sm[2].str().substr(1) : "0";
-        int leanvec_d = std::stoul(leanvec_d_string);
+        int leanvec_d = static_cast<int>(std::stoul(leanvec_d_string));
 
-        if (index_type == "Vamana")
+        if (index_type == "Vamana") {
             return new IndexSVSVamanaLeanVec(
                     d,
                     std::stoul(arg_string),
                     mt,
                     leanvec_d,
                     parse_leanvec(sm[1].str()));
-        FAISS_ASSERT(!"Unspported SVS index type for LeanVec");
+        }
+        if (index_type == "IVF") {
+            return new IndexSVSIVFLeanVec(
+                    d,
+                    std::stoul(arg_string),
+                    mt,
+                    leanvec_d,
+                    parse_leanvec(sm[1].str()));
+        }
+        FAISS_ASSERT(false && "Unsupported SVS index type for LeanVec");
     }
     return nullptr;
 }
@@ -659,7 +733,6 @@ Index* parse_IndexSVS(const std::string& code_string, int d, MetricType mt) {
         return parse_svs_datatype("Flat", "", datatype_string, d, mt);
     }
     if (re_match(code_string, "Vamana([0-9]+)(,.+)?", sm)) {
-        Index* index{nullptr};
         std::string degree_string = sm[1].str();
         std::string datatype_string =
                 sm[2].length() > 0 ? sm[2].str().substr(1) : "";
@@ -667,7 +740,10 @@ Index* parse_IndexSVS(const std::string& code_string, int d, MetricType mt) {
                 "Vamana", degree_string, datatype_string, d, mt);
     }
     if (re_match(code_string, "IVF([0-9]+)(,.+)?", sm)) {
-        FAISS_ASSERT(!"Unspported SVS index type");
+        std::string nlist_string = sm[1].str();
+        std::string datatype_string =
+                sm[2].length() > 0 ? sm[2].str().substr(1) : "";
+        return parse_svs_datatype("IVF", nlist_string, datatype_string, d, mt);
     }
     return nullptr;
 }
@@ -703,6 +779,17 @@ Index* parse_other_indexes(
         }
     }
 
+    // IndexFlatIPPanorama
+    if (match("FlatIPPanorama([0-9]+)(_[0-9]+)?")) {
+        FAISS_THROW_IF_NOT(metric == METRIC_INNER_PRODUCT);
+        int nlevels = std::stoi(sm[1].str());
+        if (sm[2].length() == 0) {
+            return new IndexFlatIPPanorama(d, nlevels);
+        }
+        int batch_size = std::stoi(sm[2].str().substr(1));
+        return new IndexFlatIPPanorama(d, nlevels, (size_t)batch_size);
+    }
+
     // IndexLSH
     if (match("LSH([0-9]*)(r?)(t?)")) {
         int nbits = sm[1].length() > 0 ? std::stoi(sm[1].str()) : d;
@@ -945,7 +1032,7 @@ std::unique_ptr<Index> index_factory_sub(
         int i0, i1;
         find_matching_parentheses(description, i0, i1, begin);
         std::string sub_description = description.substr(i0 + 1, i1 - i0 - 1);
-        int no = parenthesis_indexes.size();
+        size_t no = parenthesis_indexes.size();
         parenthesis_indexes.push_back(
                 index_factory_sub(d, sub_description, metric));
         description = description.substr(0, i0 + 1) + "Index" +

data/vendor/faiss/faiss/index_io.h

@@ -11,13 +11,19 @@
 #define FAISS_INDEX_IO_H
 
 #include <cstdio>
+#include <memory>
+
+#include <faiss/impl/platform_macros.h>
 
 /** I/O functions can read/write to a filename, a file handle or to an
  * object that abstracts the medium.
  *
- * The read functions return objects that should be deallocated with
- * delete. All references within these objects are owned by the
- * object.
+ * The read functions come in two forms:
+ * - read_*_up() returns a std::unique_ptr that owns the result.
+ * - read_*() returns a raw pointer for backward compatibility.
+ *   The caller is responsible for deleting the returned object.
+ *
+ * All references within these objects are owned by the object.
  */
 
 namespace faiss {
@@ -64,29 +70,73 @@ const int IO_FLAG_MMAP = IO_FLAG_SKIP_IVF_DATA | 0x646f0000;
 //   after OnDiskInvertedLists get properly updated.
 const int IO_FLAG_MMAP_IFC = 1 << 9;
 
+FAISS_API extern bool index_read_warn_on_null_invlists;
+
 Index* read_index(const char* fname, int io_flags = 0);
 Index* read_index(FILE* f, int io_flags = 0);
 Index* read_index(IOReader* reader, int io_flags = 0);
 
+std::unique_ptr<Index> read_index_up(const char* fname, int io_flags = 0);
+std::unique_ptr<Index> read_index_up(FILE* f, int io_flags = 0);
+std::unique_ptr<Index> read_index_up(IOReader* reader, int io_flags = 0);
+
 IndexBinary* read_index_binary(const char* fname, int io_flags = 0);
 IndexBinary* read_index_binary(FILE* f, int io_flags = 0);
 IndexBinary* read_index_binary(IOReader* reader, int io_flags = 0);
 
+std::unique_ptr<IndexBinary> read_index_binary_up(
+        const char* fname,
+        int io_flags = 0);
+std::unique_ptr<IndexBinary> read_index_binary_up(FILE* f, int io_flags = 0);
+std::unique_ptr<IndexBinary> read_index_binary_up(
+        IOReader* reader,
+        int io_flags = 0);
+
 void write_VectorTransform(const VectorTransform* vt, const char* fname);
 void write_VectorTransform(const VectorTransform* vt, IOWriter* f);
 
 VectorTransform* read_VectorTransform(const char* fname);
 VectorTransform* read_VectorTransform(IOReader* f);
 
+std::unique_ptr<VectorTransform> read_VectorTransform_up(const char* fname);
+std::unique_ptr<VectorTransform> read_VectorTransform_up(IOReader* f);
+
 ProductQuantizer* read_ProductQuantizer(const char* fname);
 ProductQuantizer* read_ProductQuantizer(IOReader* reader);
 
+std::unique_ptr<ProductQuantizer> read_ProductQuantizer_up(const char* fname);
+std::unique_ptr<ProductQuantizer> read_ProductQuantizer_up(IOReader* reader);
+
 void write_ProductQuantizer(const ProductQuantizer* pq, const char* fname);
 void write_ProductQuantizer(const ProductQuantizer* pq, IOWriter* f);
 
 void write_InvertedLists(const InvertedLists* ils, IOWriter* f);
 InvertedLists* read_InvertedLists(IOReader* reader, int io_flags = 0);
 
+std::unique_ptr<InvertedLists> read_InvertedLists_up(
+        IOReader* reader,
+        int io_flags = 0);
+
+// Returns the current deserialization loop limit.
+// When nonzero, deserialization rejects loop-driving fields (nlist,
+// nsplits, VT chain length, nhash, etc.) that exceed this value.
+// Default: 0 (no limit).
+size_t get_deserialization_loop_limit();
+
+// Sets the deserialization loop limit.
+// NOT thread-safe: set before any concurrent deserialization calls
+// and do not modify while deserialization is in progress on other threads.
+void set_deserialization_loop_limit(size_t value);
+
+// Returns the maximum number of bytes that a single READVECTOR call
+// may allocate.  Default: 1 TB (1 << 40).
+size_t get_deserialization_vector_byte_limit();
+
+// Sets the per-vector byte limit for deserialization.
+// NOT thread-safe: set before any concurrent deserialization calls
+// and do not modify while deserialization is in progress on other threads.
+void set_deserialization_vector_byte_limit(size_t value);
+
 } // namespace faiss
 
 #endif

data/vendor/faiss/faiss/invlists/BlockInvertedLists.cpp

@@ -5,8 +5,11 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+#include <faiss/index_io.h>
 #include <faiss/invlists/BlockInvertedLists.h>
 
+#include <memory>
+
 #include <faiss/impl/CodePacker.h>
 #include <faiss/impl/FaissAssert.h>
 #include <faiss/impl/IDSelector.h>
@@ -17,23 +20,25 @@
 namespace faiss {
 
 BlockInvertedLists::BlockInvertedLists(
-        size_t nlist,
-        size_t n_per_block,
-        size_t block_size)
-        : InvertedLists(nlist, InvertedLists::INVALID_CODE_SIZE),
-          n_per_block(n_per_block),
-          block_size(block_size) {
-    ids.resize(nlist);
-    codes.resize(nlist);
+        size_t nlist_in,
+        size_t n_per_block_in,
+        size_t block_size_in)
+        : InvertedLists(nlist_in, InvertedLists::INVALID_CODE_SIZE),
+          n_per_block(n_per_block_in),
+          block_size(block_size_in) {
+    ids.resize(nlist_in);
+    codes.resize(nlist_in);
 }
 
-BlockInvertedLists::BlockInvertedLists(size_t nlist, const CodePacker* packer)
-        : InvertedLists(nlist, InvertedLists::INVALID_CODE_SIZE),
-          n_per_block(packer->nvec),
-          block_size(packer->block_size),
-          packer(packer) {
-    ids.resize(nlist);
-    codes.resize(nlist);
+BlockInvertedLists::BlockInvertedLists(
+        size_t nlist_in,
+        const CodePacker* packer_in)
+        : InvertedLists(nlist_in, InvertedLists::INVALID_CODE_SIZE),
+          n_per_block(packer_in->nvec),
+          block_size(packer_in->block_size),
+          packer(packer_in) {
+    ids.resize(nlist_in);
+    codes.resize(nlist_in);
 }
 
 BlockInvertedLists::BlockInvertedLists()
@@ -81,8 +86,8 @@ const uint8_t* BlockInvertedLists::get_codes(size_t list_no) const {
 
 size_t BlockInvertedLists::remove_ids(const IDSelector& sel) {
     idx_t nremove = 0;
-#pragma omp parallel for
-    for (idx_t i = 0; i < nlist; i++) {
+#pragma omp parallel for reduction(+ : nremove)
+    for (idx_t i = 0; i < static_cast<idx_t>(nlist); i++) {
         std::vector<uint8_t> buffer(packer->code_size);
         idx_t l = ids[i].size(), j = 0;
         while (j < l) {
@@ -95,8 +100,9 @@ size_t BlockInvertedLists::remove_ids(const IDSelector& sel) {
                 j++;
             }
         }
+        idx_t orig_size = ids[i].size();
         resize(i, l);
-        nremove += ids[i].size() - l;
+        nremove += orig_size - l;
     }
 
     return nremove;
@@ -160,21 +166,68 @@ void BlockInvertedListsIOHook::write(const InvertedLists* ils_in, IOWriter* f)
 
 InvertedLists* BlockInvertedListsIOHook::read(IOReader* f, int /* io_flags */)
         const {
-    BlockInvertedLists* il = new BlockInvertedLists();
+    auto il = std::make_unique<BlockInvertedLists>();
     READ1(il->nlist);
+    {
+        auto limit_ = get_deserialization_loop_limit();
+        if (limit_ > 0) {
+            FAISS_THROW_IF_NOT_FMT(
+                    static_cast<size_t>(il->nlist) <= limit_,
+                    "BlockInvertedLists nlist=%zd exceeds "
+                    "deserialization_loop_limit of %zd",
+                    static_cast<size_t>(il->nlist),
+                    limit_);
+        }
+    }
     READ1(il->code_size);
     READ1(il->n_per_block);
     READ1(il->block_size);
 
+    {
+        auto limit = get_deserialization_loop_limit();
+        if (limit > 0) {
+            FAISS_THROW_IF_NOT_FMT(
+                    il->nlist <= limit,
+                    "BlockInvertedLists nlist=%zd exceeds "
+                    "deserialization_loop_limit of %zd",
+                    il->nlist,
+                    limit);
+        }
+    }
+
+    FAISS_THROW_IF_NOT_FMT(
+            il->n_per_block > 0,
+            "invalid BlockInvertedLists n_per_block %zd (must be > 0)",
+            il->n_per_block);
+    FAISS_THROW_IF_NOT_FMT(
+            il->block_size > 0,
+            "invalid BlockInvertedLists block_size %zd (must be > 0)",
+            il->block_size);
+
     il->ids.resize(il->nlist);
     il->codes.resize(il->nlist);
 
     for (size_t i = 0; i < il->nlist; i++) {
         READVECTOR(il->ids[i]);
         READVECTOR(il->codes[i]);
+        size_t n_ids = il->ids[i].size();
+        size_t n_block = (n_ids + il->n_per_block - 1) / il->n_per_block;
+        size_t expected_codes_size = mul_no_overflow(
+                n_block, il->block_size, "BlockInvertedLists codes");
+        FAISS_THROW_IF_NOT_FMT(
+                il->codes[i].size() == expected_codes_size,
+                "BlockInvertedLists list %zd: codes size %zd does not "
+                "match expected %zd (ids=%zd, n_per_block=%zd, "
+                "block_size=%zd)",
+                i,
+                il->codes[i].size(),
+                expected_codes_size,
+                n_ids,
+                il->n_per_block,
+                il->block_size);
     }
 
-    return il;
+    return il.release();
 }
 
 } // namespace faiss