faiss 0.5.2 → 0.6.0

data/vendor/faiss/faiss/impl/index_read.cpp

@@ -12,14 +12,19 @@
 
 #include <cstdio>
 #include <cstdlib>
+#include <cstring>
+#include <memory>
 #include <optional>
 
 #include <faiss/impl/FaissAssert.h>
+#include <faiss/impl/RaBitQUtils.h>
 #include <faiss/impl/io.h>
 #include <faiss/utils/hamming.h>
 
 #include <faiss/invlists/InvertedListsIOHook.h>
 
+#include <faiss/invlists/BlockInvertedLists.h>
+
 #include <faiss/Index2Layer.h>
 #include <faiss/IndexAdditiveQuantizer.h>
 #include <faiss/IndexAdditiveQuantizerFastScan.h>
@@ -205,34 +210,41 @@ void read_xb_vector(VectorT& target, IOReader* f) {
  * Read
  **************************************************************/
 
-void read_index_header(Index* idx, IOReader* f) {
-    READ1(idx->d);
-    READ1(idx->ntotal);
+static void read_index_header(Index& idx, IOReader* f) {
+    READ1(idx.d);
+    READ1(idx.ntotal);
+    FAISS_CHECK_RANGE(idx.d, 0, (1 << 20) + 1);
+    FAISS_THROW_IF_NOT_FMT(
+            idx.ntotal >= 0,
+            "invalid ntotal %" PRId64 " read from index",
+            (int64_t)idx.ntotal);
     idx_t dummy;
     READ1(dummy);
     READ1(dummy);
-    READ1(idx->is_trained);
-    READ1(idx->metric_type);
-    if (idx->metric_type > 1) {
-        READ1(idx->metric_arg);
+    READ1(idx.is_trained);
+    int metric_type_int;
+    READ1(metric_type_int);
+    idx.metric_type = metric_type_from_int(metric_type_int);
+    if (idx.metric_type > 1) {
+        READ1(idx.metric_arg);
     }
-    idx->verbose = false;
+    idx.verbose = false;
 }
 
-VectorTransform* read_VectorTransform(IOReader* f) {
+std::unique_ptr<VectorTransform> read_VectorTransform_up(IOReader* f) {
     uint32_t h;
     READ1(h);
-    VectorTransform* vt = nullptr;
+    std::unique_ptr<VectorTransform> vt;
 
     if (h == fourcc("rrot") || h == fourcc("PCAm") || h == fourcc("LTra") ||
         h == fourcc("PcAm") || h == fourcc("Viqm") || h == fourcc("Pcam")) {
-        LinearTransform* lt = nullptr;
+        std::unique_ptr<LinearTransform> lt;
         if (h == fourcc("rrot")) {
-            lt = new RandomRotationMatrix();
+            lt = std::make_unique<RandomRotationMatrix>();
         } else if (
                 h == fourcc("PCAm") || h == fourcc("PcAm") ||
                 h == fourcc("Pcam")) {
-            PCAMatrix* pca = new PCAMatrix();
+            auto pca = std::make_unique<PCAMatrix>();
             READ1(pca->eigen_power);
             if (h == fourcc("Pcam")) {
                 READ1(pca->epsilon);
@@ -244,14 +256,14 @@ VectorTransform* read_VectorTransform(IOReader* f) {
             READVECTOR(pca->mean);
             READVECTOR(pca->eigenvalues);
             READVECTOR(pca->PCAMat);
-            lt = pca;
+            lt = std::move(pca);
         } else if (h == fourcc("Viqm")) {
-            ITQMatrix* itqm = new ITQMatrix();
+            auto itqm = std::make_unique<ITQMatrix>();
             READ1(itqm->max_iter);
             READ1(itqm->seed);
-            lt = itqm;
+            lt = std::move(itqm);
         } else if (h == fourcc("LTra")) {
-            lt = new LinearTransform();
+            lt = std::make_unique<LinearTransform>();
         }
         READ1(lt->have_bias);
         READVECTOR(lt->A);
@@ -259,38 +271,43 @@ VectorTransform* read_VectorTransform(IOReader* f) {
         FAISS_THROW_IF_NOT(lt->A.size() >= lt->d_in * lt->d_out);
         FAISS_THROW_IF_NOT(!lt->have_bias || lt->b.size() >= lt->d_out);
         lt->set_is_orthonormal();
-        vt = lt;
+        vt = std::move(lt);
     } else if (h == fourcc("RmDT")) {
-        RemapDimensionsTransform* rdt = new RemapDimensionsTransform();
+        auto rdt = std::make_unique<RemapDimensionsTransform>();
         READVECTOR(rdt->map);
-        vt = rdt;
+        vt = std::move(rdt);
     } else if (h == fourcc("VNrm")) {
-        NormalizationTransform* nt = new NormalizationTransform();
+        auto nt = std::make_unique<NormalizationTransform>();
         READ1(nt->norm);
-        vt = nt;
+        vt = std::move(nt);
     } else if (h == fourcc("VCnt")) {
-        CenteringTransform* ct = new CenteringTransform();
+        auto ct = std::make_unique<CenteringTransform>();
         READVECTOR(ct->mean);
-        vt = ct;
+        vt = std::move(ct);
     } else if (h == fourcc("Viqt")) {
-        ITQTransform* itqt = new ITQTransform();
+        auto itqt = std::make_unique<ITQTransform>();
 
         READVECTOR(itqt->mean);
         READ1(itqt->do_pca);
         {
-            ITQMatrix* itqm = dynamic_cast<ITQMatrix*>(read_VectorTransform(f));
+            // Read, dereference, discard.
+            auto sub_vt = read_VectorTransform_up(f);
+            ITQMatrix* itqm = dynamic_cast<ITQMatrix*>(sub_vt.get());
             FAISS_THROW_IF_NOT(itqm);
             itqt->itq = *itqm;
-            delete itqm;
         }
         {
-            LinearTransform* pi =
-                    dynamic_cast<LinearTransform*>(read_VectorTransform(f));
+            // Read, dereference, discard.
+            auto sub_vt = read_VectorTransform_up(f);
+            LinearTransform* pi = dynamic_cast<LinearTransform*>(sub_vt.get());
             FAISS_THROW_IF_NOT(pi);
             itqt->pca_then_itq = *pi;
-            delete pi;
         }
-        vt = itqt;
+        vt = std::move(itqt);
+    } else if (h == fourcc("HRot")) {
+        auto hr = std::make_unique<HadamardRotation>();
+        READ1(hr->seed);
+        vt = std::move(hr);
     } else {
         FAISS_THROW_FMT(
                 "fourcc %ud (\"%s\") not recognized in %s",
@@ -301,9 +318,17 @@ VectorTransform* read_VectorTransform(IOReader* f) {
     READ1(vt->d_in);
     READ1(vt->d_out);
     READ1(vt->is_trained);
+    if (h == fourcc("HRot")) {
+        auto* hr = dynamic_cast<HadamardRotation*>(vt.get());
+        hr->init(hr->seed);
+    }
     return vt;
 }
 
+VectorTransform* read_VectorTransform(IOReader* f) {
+    return read_VectorTransform_up(f).release();
+}
+
 static void read_ArrayInvertedLists_sizes(
         IOReader* f,
         std::vector<size_t>& sizes) {
@@ -316,6 +341,10 @@ static void read_ArrayInvertedLists_sizes(
     } else if (list_type == fourcc("sprs")) {
         std::vector<size_t> idsizes;
         READVECTOR(idsizes);
+        FAISS_THROW_IF_NOT_FMT(
+                idsizes.size() % 2 == 0,
+                "invalid sparse inverted list size: %zd (must be even)",
+                idsizes.size());
         for (size_t j = 0; j < idsizes.size(); j += 2) {
             FAISS_THROW_IF_NOT(idsizes[j] < sizes.size());
             sizes[idsizes[j]] = idsizes[j + 1];
@@ -328,7 +357,9 @@ static void read_ArrayInvertedLists_sizes(
     }
 }
 
-InvertedLists* read_InvertedLists(IOReader* f, int io_flags) {
+std::unique_ptr<InvertedLists> read_InvertedLists_up(
+        IOReader* f,
+        int io_flags) {
     uint32_t h;
     READ1(h);
     if (h == fourcc("il00")) {
@@ -341,7 +372,8 @@ InvertedLists* read_InvertedLists(IOReader* f, int io_flags) {
         READ1(nlist);
         READ1(code_size);
         READ1(n_levels);
-        auto ailp = new ArrayInvertedListsPanorama(nlist, code_size, n_levels);
+        auto ailp = std::make_unique<ArrayInvertedListsPanorama>(
+                nlist, code_size, n_levels);
         std::vector<size_t> sizes(nlist);
         read_ArrayInvertedLists_sizes(f, sizes);
         for (size_t i = 0; i < nlist; i++) {
@@ -365,7 +397,7 @@ InvertedLists* read_InvertedLists(IOReader* f, int io_flags) {
         }
         return ailp;
     } else if (h == fourcc("ilar") && !(io_flags & IO_FLAG_SKIP_IVF_DATA)) {
-        auto ails = new ArrayInvertedLists(0, 0);
+        auto ails = std::make_unique<ArrayInvertedLists>(0, 0);
         READ1(ails->nlist);
         READ1(ails->code_size);
         ails->ids.resize(ails->nlist);
@@ -374,13 +406,17 @@ InvertedLists* read_InvertedLists(IOReader* f, int io_flags) {
         read_ArrayInvertedLists_sizes(f, sizes);
         for (size_t i = 0; i < ails->nlist; i++) {
             ails->ids[i].resize(sizes[i]);
-            ails->codes[i].resize(sizes[i] * ails->code_size);
+            ails->codes[i].resize(mul_no_overflow(
+                    sizes[i], ails->code_size, "inverted list codes"));
         }
         for (size_t i = 0; i < ails->nlist; i++) {
             size_t n = ails->ids[i].size();
             if (n > 0) {
                 read_vector_with_known_size(
-                        ails->codes[i], f, n * ails->code_size);
+                        ails->codes[i],
+                        f,
+                        mul_no_overflow(
+                                n, ails->code_size, "inverted list codes"));
                 read_vector_with_known_size(ails->ids[i], f, n);
             }
         }
@@ -396,132 +432,150 @@ InvertedLists* read_InvertedLists(IOReader* f, int io_flags) {
         READ1(code_size);
         std::vector<size_t> sizes(nlist);
         read_ArrayInvertedLists_sizes(f, sizes);
-        return InvertedListsIOHook::lookup(h2)->read_ArrayInvertedLists(
-                f, io_flags, nlist, code_size, sizes);
+        return std::unique_ptr<InvertedLists>(
+                InvertedListsIOHook::lookup(h2)->read_ArrayInvertedLists(
+                        f, io_flags, nlist, code_size, sizes));
     } else {
-        return InvertedListsIOHook::lookup(h)->read(f, io_flags);
+        return std::unique_ptr<InvertedLists>(
+                InvertedListsIOHook::lookup(h)->read(f, io_flags));
     }
 }
 
-void read_InvertedLists(IndexIVF* ivf, IOReader* f, int io_flags) {
-    InvertedLists* ils = read_InvertedLists(f, io_flags);
+InvertedLists* read_InvertedLists(IOReader* f, int io_flags) {
+    return read_InvertedLists_up(f, io_flags).release();
+}
+
+void read_InvertedLists(IndexIVF& ivf, IOReader* f, int io_flags) {
+    auto ils = read_InvertedLists_up(f, io_flags);
     if (ils) {
-        FAISS_THROW_IF_NOT(ils->nlist == ivf->nlist);
+        FAISS_THROW_IF_NOT(ils->nlist == ivf.nlist);
         FAISS_THROW_IF_NOT(
                 ils->code_size == InvertedLists::INVALID_CODE_SIZE ||
-                ils->code_size == ivf->code_size);
+                ils->code_size == ivf.code_size);
     }
-    ivf->invlists = ils;
-    ivf->own_invlists = true;
+    ivf.invlists = ils.release();
+    ivf.own_invlists = true;
 }
 
 void read_ProductQuantizer(ProductQuantizer* pq, IOReader* f) {
     READ1(pq->d);
     READ1(pq->M);
     READ1(pq->nbits);
+    FAISS_THROW_IF_NOT_FMT(
+            pq->M > 0, "invalid ProductQuantizer M=%zd (must be > 0)", pq->M);
     pq->set_derived_values();
     READVECTOR(pq->centroids);
 }
 
-static void read_ResidualQuantizer_old(ResidualQuantizer* rq, IOReader* f) {
-    READ1(rq->d);
-    READ1(rq->M);
-    READVECTOR(rq->nbits);
-    READ1(rq->is_trained);
-    READ1(rq->train_type);
-    READ1(rq->max_beam_size);
-    READVECTOR(rq->codebooks);
-    READ1(rq->search_type);
-    READ1(rq->norm_min);
-    READ1(rq->norm_max);
-    rq->set_derived_values();
+static void read_ResidualQuantizer_old(ResidualQuantizer& rq, IOReader* f) {
+    READ1(rq.d);
+    READ1(rq.M);
+    READVECTOR(rq.nbits);
+    FAISS_THROW_IF_NOT_FMT(
+            rq.nbits.size() == rq.M,
+            "ResidualQuantizer nbits size %zd != M %zd",
+            rq.nbits.size(),
+            rq.M);
+    READ1(rq.is_trained);
+    READ1(rq.train_type);
+    READ1(rq.max_beam_size);
+    READVECTOR(rq.codebooks);
+    READ1(rq.search_type);
+    READ1(rq.norm_min);
+    READ1(rq.norm_max);
+    rq.set_derived_values();
 }
 
-static void read_AdditiveQuantizer(AdditiveQuantizer* aq, IOReader* f) {
-    READ1(aq->d);
-    READ1(aq->M);
-    READVECTOR(aq->nbits);
-    READ1(aq->is_trained);
-    READVECTOR(aq->codebooks);
-    READ1(aq->search_type);
-    READ1(aq->norm_min);
-    READ1(aq->norm_max);
-    if (aq->search_type == AdditiveQuantizer::ST_norm_cqint8 ||
-        aq->search_type == AdditiveQuantizer::ST_norm_cqint4 ||
-        aq->search_type == AdditiveQuantizer::ST_norm_lsq2x4 ||
-        aq->search_type == AdditiveQuantizer::ST_norm_rq2x4) {
-        read_xb_vector(aq->qnorm.codes, f);
-        aq->qnorm.ntotal = aq->qnorm.codes.size() / 4;
-        aq->qnorm.update_permutation();
+static void read_AdditiveQuantizer(AdditiveQuantizer& aq, IOReader* f) {
+    READ1(aq.d);
+    READ1(aq.M);
+    READVECTOR(aq.nbits);
+    READ1(aq.is_trained);
+    READVECTOR(aq.codebooks);
+    FAISS_THROW_IF_NOT_FMT(
+            aq.nbits.size() == aq.M,
+            "AdditiveQuantizer nbits size %zd != M %zd",
+            aq.nbits.size(),
+            aq.M);
+    READ1(aq.search_type);
+    READ1(aq.norm_min);
+    READ1(aq.norm_max);
+    if (aq.search_type == AdditiveQuantizer::ST_norm_cqint8 ||
+        aq.search_type == AdditiveQuantizer::ST_norm_cqint4 ||
+        aq.search_type == AdditiveQuantizer::ST_norm_lsq2x4 ||
+        aq.search_type == AdditiveQuantizer::ST_norm_rq2x4) {
+        read_xb_vector(aq.qnorm.codes, f);
+        aq.qnorm.ntotal = aq.qnorm.codes.size() / 4;
+        aq.qnorm.update_permutation();
     }
 
-    if (aq->search_type == AdditiveQuantizer::ST_norm_lsq2x4 ||
-        aq->search_type == AdditiveQuantizer::ST_norm_rq2x4) {
-        READVECTOR(aq->norm_tabs);
+    if (aq.search_type == AdditiveQuantizer::ST_norm_lsq2x4 ||
+        aq.search_type == AdditiveQuantizer::ST_norm_rq2x4) {
+        READVECTOR(aq.norm_tabs);
     }
 
-    aq->set_derived_values();
+    aq.set_derived_values();
 }
 
 static void read_ResidualQuantizer(
-        ResidualQuantizer* rq,
+        ResidualQuantizer& rq,
         IOReader* f,
         int io_flags) {
     read_AdditiveQuantizer(rq, f);
-    READ1(rq->train_type);
-    READ1(rq->max_beam_size);
-    if ((rq->train_type & ResidualQuantizer::Skip_codebook_tables) ||
+    READ1(rq.train_type);
+    READ1(rq.max_beam_size);
+    if ((rq.train_type & ResidualQuantizer::Skip_codebook_tables) ||
         (io_flags & IO_FLAG_SKIP_PRECOMPUTE_TABLE)) {
         // don't precompute the tables
     } else {
-        rq->compute_codebook_tables();
+        rq.compute_codebook_tables();
     }
 }
 
-static void read_LocalSearchQuantizer(LocalSearchQuantizer* lsq, IOReader* f) {
+static void read_LocalSearchQuantizer(LocalSearchQuantizer& lsq, IOReader* f) {
     read_AdditiveQuantizer(lsq, f);
-    READ1(lsq->K);
-    READ1(lsq->train_iters);
-    READ1(lsq->encode_ils_iters);
-    READ1(lsq->train_ils_iters);
-    READ1(lsq->icm_iters);
-    READ1(lsq->p);
-    READ1(lsq->lambd);
-    READ1(lsq->chunk_size);
-    READ1(lsq->random_seed);
-    READ1(lsq->nperts);
-    READ1(lsq->update_codebooks_with_double);
+    READ1(lsq.K);
+    READ1(lsq.train_iters);
+    READ1(lsq.encode_ils_iters);
+    READ1(lsq.train_ils_iters);
+    READ1(lsq.icm_iters);
+    READ1(lsq.p);
+    READ1(lsq.lambd);
+    READ1(lsq.chunk_size);
+    READ1(lsq.random_seed);
+    READ1(lsq.nperts);
+    READ1(lsq.update_codebooks_with_double);
 }
 
 static void read_ProductAdditiveQuantizer(
-        ProductAdditiveQuantizer* paq,
+        ProductAdditiveQuantizer& paq,
         IOReader* f) {
     read_AdditiveQuantizer(paq, f);
-    READ1(paq->nsplits);
+    READ1(paq.nsplits);
 }
 
 static void read_ProductResidualQuantizer(
-        ProductResidualQuantizer* prq,
+        ProductResidualQuantizer& prq,
         IOReader* f,
         int io_flags) {
     read_ProductAdditiveQuantizer(prq, f);
 
-    for (size_t i = 0; i < prq->nsplits; i++) {
-        auto rq = new ResidualQuantizer();
-        read_ResidualQuantizer(rq, f, io_flags);
-        prq->quantizers.push_back(rq);
+    for (size_t i = 0; i < prq.nsplits; i++) {
+        auto rq = std::make_unique<ResidualQuantizer>();
+        read_ResidualQuantizer(*rq, f, io_flags);
+        prq.quantizers.push_back(rq.release());
     }
 }
 
 static void read_ProductLocalSearchQuantizer(
-        ProductLocalSearchQuantizer* plsq,
+        ProductLocalSearchQuantizer& plsq,
         IOReader* f) {
     read_ProductAdditiveQuantizer(plsq, f);
 
-    for (size_t i = 0; i < plsq->nsplits; i++) {
-        auto lsq = new LocalSearchQuantizer();
-        read_LocalSearchQuantizer(lsq, f);
-        plsq->quantizers.push_back(lsq);
+    for (size_t i = 0; i < plsq.nsplits; i++) {
+        auto lsq = std::make_unique<LocalSearchQuantizer>();
+        read_LocalSearchQuantizer(*lsq, f);
+        plsq.quantizers.push_back(lsq.release());
     }
 }
 
@@ -535,97 +589,237 @@ void read_ScalarQuantizer(ScalarQuantizer* ivsc, IOReader* f) {
     ivsc->set_derived_sizes();
 }
 
-static void read_HNSW(HNSW* hnsw, IOReader* f) {
-    READVECTOR(hnsw->assign_probas);
-    READVECTOR(hnsw->cum_nneighbor_per_level);
-    READVECTOR(hnsw->levels);
-    READVECTOR(hnsw->offsets);
-    read_vector(hnsw->neighbors, f);
+static void validate_HNSW(const HNSW& hnsw) {
+    size_t ntotal = hnsw.levels.size();
+    size_t nb_neighbors_size = hnsw.neighbors.size();
+
+    // cum_nneighbor_per_level must be non-empty and monotonically
+    // non-decreasing, starting at 0
+    if (!hnsw.cum_nneighbor_per_level.empty()) {
+        FAISS_THROW_IF_NOT_FMT(
+                hnsw.cum_nneighbor_per_level[0] == 0,
+                "HNSW cum_nneighbor_per_level[0] = %d, expected 0",
+                hnsw.cum_nneighbor_per_level[0]);
+        for (size_t i = 1; i < hnsw.cum_nneighbor_per_level.size(); i++) {
+            FAISS_THROW_IF_NOT_FMT(
+                    hnsw.cum_nneighbor_per_level[i] >=
+                            hnsw.cum_nneighbor_per_level[i - 1],
+                    "HNSW cum_nneighbor_per_level not monotonic at %zd: "
+                    "%d < %d",
+                    i,
+                    hnsw.cum_nneighbor_per_level[i],
+                    hnsw.cum_nneighbor_per_level[i - 1]);
+        }
+    }
+
+    // every levels[i] must be a valid index into cum_nneighbor_per_level
+    size_t cum_size = hnsw.cum_nneighbor_per_level.size();
+    for (size_t i = 0; i < ntotal; i++) {
+        FAISS_THROW_IF_NOT_FMT(
+                hnsw.levels[i] >= 0 &&
+                        static_cast<size_t>(hnsw.levels[i]) < cum_size,
+                "HNSW levels[%zd] = %d out of range [0, %zd)",
+                i,
+                hnsw.levels[i],
+                cum_size);
+    }
+
+    // offsets must have size ntotal + 1, be monotonically non-decreasing,
+    // and all values must be <= neighbors.size()
+    FAISS_THROW_IF_NOT_FMT(
+            hnsw.offsets.size() == ntotal + 1,
+            "HNSW offsets size %zd != levels size %zd + 1",
+            hnsw.offsets.size(),
+            ntotal);
+    for (size_t i = 0; i < hnsw.offsets.size(); i++) {
+        FAISS_THROW_IF_NOT_FMT(
+                hnsw.offsets[i] <= nb_neighbors_size,
+                "HNSW offsets[%zd] = %zd > neighbors.size() = %zd",
+                i,
+                hnsw.offsets[i],
+                nb_neighbors_size);
+        if (i > 0) {
+            FAISS_THROW_IF_NOT_FMT(
+                    hnsw.offsets[i] ==
+                            hnsw.offsets[i - 1] +
+                                    hnsw.cum_nneighbor_per_level
+                                            [hnsw.levels[i - 1]],
+                    "HNSW offsets not increasing by cum_neighbor_per_level at %zd: %zd + %d != %zd",
+                    i,
+                    hnsw.offsets[i - 1],
+                    hnsw.cum_nneighbor_per_level[hnsw.levels[i - 1]],
+                    hnsw.offsets[i]);
+        }
+    }
+
+    // max_level must be valid
+    FAISS_THROW_IF_NOT_FMT(
+            hnsw.max_level < (int)hnsw.cum_nneighbor_per_level.size(),
+            "HNSW max_level %d >= cum_nneighbor_per_level size %zd",
+            hnsw.max_level,
+            hnsw.cum_nneighbor_per_level.size());
+
+    // entry_point must be -1 (empty) or a valid node id
+    FAISS_THROW_IF_NOT_FMT(
+            hnsw.entry_point >= -1 && hnsw.entry_point < (int)ntotal,
+            "HNSW entry_point %d out of range [-1, %zd)",
+            (int)hnsw.entry_point,
+            ntotal);
+
+    // All neighbor ids must be -1 or in [0, ntotal)
+    for (size_t i = 0; i < nb_neighbors_size; i++) {
+        auto id = hnsw.neighbors[i];
+        FAISS_THROW_IF_NOT_FMT(
+                id >= -1 && id < (int)ntotal,
+                "HNSW neighbors[%zd] = %d out of range [-1, %zd)",
+                i,
+                (int)id,
+                ntotal);
+    }
+
+    // For each node, verify that its level is valid and that
+    // offsets[i] + cum_nneighbor_per_level[levels[i]] <= neighbors.size().
+    // This ensures neighbor_range() can never produce an out-of-bounds offset
+    // into neighbors.
+    int cum_levels = (int)hnsw.cum_nneighbor_per_level.size();
+    for (size_t i = 0; i < ntotal; i++) {
+        int level = hnsw.levels[i];
+        FAISS_CHECK_RANGE(level, 1, cum_levels + 1);
+        size_t end = hnsw.offsets[i] + hnsw.cum_nneighbor_per_level[level];
+        FAISS_THROW_IF_NOT_FMT(
+                end <= nb_neighbors_size,
+                "HNSW neighbor range overflow for node %zd: "
+                "offsets[%zd] (%zd) + cum_nneighbor_per_level[%d] (%d) "
+                "= %zd > neighbors.size() (%zd)",
+                i,
+                i,
+                hnsw.offsets[i],
+                level,
+                hnsw.cum_nneighbor_per_level[level],
+                end,
+                nb_neighbors_size);
+    }
+}
 
-    READ1(hnsw->entry_point);
-    READ1(hnsw->max_level);
-    READ1(hnsw->efConstruction);
-    READ1(hnsw->efSearch);
+static void read_HNSW(HNSW& hnsw, IOReader* f) {
+    READVECTOR(hnsw.assign_probas);
+    READVECTOR(hnsw.cum_nneighbor_per_level);
+    READVECTOR(hnsw.levels);
+    READVECTOR(hnsw.offsets);
+    read_vector(hnsw.neighbors, f);
+
+    READ1(hnsw.entry_point);
+    READ1(hnsw.max_level);
+    READ1(hnsw.efConstruction);
+    READ1(hnsw.efSearch);
 
     // // deprecated field
-    // READ1(hnsw->upper_beam);
+    // READ1(hnsw.upper_beam);
     READ1_DUMMY(int)
+
+    validate_HNSW(hnsw);
 }
 
-static void read_NSG(NSG* nsg, IOReader* f) {
-    READ1(nsg->ntotal);
-    READ1(nsg->R);
-    READ1(nsg->L);
-    READ1(nsg->C);
-    READ1(nsg->search_L);
-    READ1(nsg->enterpoint);
-    READ1(nsg->is_built);
+static void read_NSG(NSG& nsg, IOReader* f) {
+    READ1(nsg.ntotal);
+    READ1(nsg.R);
+    READ1(nsg.L);
+    READ1(nsg.C);
+    READ1(nsg.search_L);
+    READ1(nsg.enterpoint);
+    READ1(nsg.is_built);
 
-    if (!nsg->is_built) {
+    FAISS_THROW_IF_NOT_FMT(
+            nsg.ntotal >= 0, "invalid NSG ntotal %d", nsg.ntotal);
+
+    if (!nsg.is_built) {
         return;
     }
 
     constexpr int EMPTY_ID = -1;
-    int N = nsg->ntotal;
-    int R = nsg->R;
-    auto& graph = nsg->final_graph;
+    int N = nsg.ntotal;
+    int R = nsg.R;
+
+    auto& graph = nsg.final_graph;
     graph = std::make_shared<nsg::Graph<int>>(N, R);
-    std::fill_n(graph->data, N * R, EMPTY_ID);
+    std::fill_n(graph->data, (size_t)N * R, EMPTY_ID);
 
     for (int i = 0; i < N; i++) {
-        for (int j = 0; j < R + 1; j++) {
+        int j;
+        for (j = 0; j < R; j++) {
             int id;
             READ1(id);
             if (id != EMPTY_ID) {
+                FAISS_CHECK_RANGE(id, 0, N);
                 graph->at(i, j) = id;
             } else {
                 break;
             }
         }
+        if (j == R) {
+            // All R neighbor slots were filled; consume the trailing
+            // EMPTY_ID sentinel that write_NSG always appends.
+            int sentinel;
+            READ1(sentinel);
+            FAISS_THROW_IF_NOT(sentinel == EMPTY_ID);
+        }
     }
+
+    // enterpoint must be a valid node id
+    FAISS_CHECK_RANGE(nsg.enterpoint, 0, N);
 }
 
-static void read_NNDescent(NNDescent* nnd, IOReader* f) {
-    READ1(nnd->ntotal);
-    READ1(nnd->d);
-    READ1(nnd->K);
-    READ1(nnd->S);
-    READ1(nnd->R);
-    READ1(nnd->L);
-    READ1(nnd->iter);
-    READ1(nnd->search_L);
-    READ1(nnd->random_seed);
-    READ1(nnd->has_built);
-
-    READVECTOR(nnd->final_graph);
+static void read_NNDescent(NNDescent& nnd, IOReader* f) {
+    READ1(nnd.ntotal);
+    READ1(nnd.d);
+    READ1(nnd.K);
+    READ1(nnd.S);
+    READ1(nnd.R);
+    READ1(nnd.L);
+    READ1(nnd.iter);
+    READ1(nnd.search_L);
+    READ1(nnd.random_seed);
+    READ1(nnd.has_built);
+
+    FAISS_THROW_IF_NOT_FMT(
+            nnd.ntotal >= 0, "invalid NNDescent ntotal %d", nnd.ntotal);
+
+    READVECTOR(nnd.final_graph);
 }
 
-ProductQuantizer* read_ProductQuantizer(const char* fname) {
+std::unique_ptr<ProductQuantizer> read_ProductQuantizer_up(const char* fname) {
     FileIOReader reader(fname);
-    return read_ProductQuantizer(&reader);
+    return read_ProductQuantizer_up(&reader);
 }
 
-ProductQuantizer* read_ProductQuantizer(IOReader* reader) {
-    ProductQuantizer* pq = new ProductQuantizer();
-    std::unique_ptr<ProductQuantizer> del(pq);
+ProductQuantizer* read_ProductQuantizer(const char* fname) {
+    return read_ProductQuantizer_up(fname).release();
+}
 
-    read_ProductQuantizer(pq, reader);
-    del.release();
+std::unique_ptr<ProductQuantizer> read_ProductQuantizer_up(IOReader* reader) {
+    auto pq = std::make_unique<ProductQuantizer>();
+    read_ProductQuantizer(pq.get(), reader);
     return pq;
 }
 
+ProductQuantizer* read_ProductQuantizer(IOReader* reader) {
+    return read_ProductQuantizer_up(reader).release();
+}
+
 static void read_RaBitQuantizer(
-        RaBitQuantizer* rabitq,
+        RaBitQuantizer& rabitq,
         IOReader* f,
         bool multi_bit = true) {
-    READ1(rabitq->d);
-    READ1(rabitq->code_size);
-    READ1(rabitq->metric_type);
+    READ1(rabitq.d);
+    READ1(rabitq.code_size);
+    int metric_type_int;
+    READ1(metric_type_int);
+    rabitq.metric_type = metric_type_from_int(metric_type_int);
 
     if (multi_bit) {
-        READ1(rabitq->nb_bits);
+        READ1(rabitq.nb_bits);
     } else {
-        rabitq->nb_bits = 1;
+        rabitq.nb_bits = 1;
     }
 }
 
@@ -649,7 +843,7 @@ void read_ivf_header(
         IndexIVF* ivf,
         IOReader* f,
         std::vector<std::vector<idx_t>>* ids) {
-    read_index_header(ivf, f);
+    read_index_header(*ivf, f);
     READ1(ivf->nlist);
     READ1(ivf->nprobe);
     ivf->quantizer = read_index(f);
@@ -666,39 +860,46 @@ void read_ivf_header(
 ArrayInvertedLists* set_array_invlist(
         IndexIVF* ivf,
         std::vector<std::vector<idx_t>>& ids) {
-    ArrayInvertedLists* ail =
-            new ArrayInvertedLists(ivf->nlist, ivf->code_size);
+    auto ail = std::make_unique<ArrayInvertedLists>(ivf->nlist, ivf->code_size);
 
     ail->ids.resize(ids.size());
     for (size_t i = 0; i < ids.size(); i++) {
         ail->ids[i] = MaybeOwnedVector<idx_t>(std::move(ids[i]));
     }
 
-    ivf->invlists = ail;
+    ArrayInvertedLists* result = ail.get();
+    ivf->invlists = ail.release();
     ivf->own_invlists = true;
-    return ail;
+    return result;
 }
 
-static IndexIVFPQ* read_ivfpq(IOReader* f, uint32_t h, int io_flags) {
+static std::unique_ptr<IndexIVFPQ> read_ivfpq(
+        IOReader* f,
+        uint32_t h,
+        int io_flags) {
     bool legacy = h == fourcc("IvQR") || h == fourcc("IvPQ");
 
-    IndexIVFPQR* ivfpqr = h == fourcc("IvQR") || h == fourcc("IwQR")
-            ? new IndexIVFPQR()
-            : nullptr;
-    IndexIVFPQ* ivpq = ivfpqr ? ivfpqr : new IndexIVFPQ();
+    IndexIVFPQR* ivfpqr = nullptr;
+    std::unique_ptr<IndexIVFPQ> ivpq;
+    if (h == fourcc("IvQR") || h == fourcc("IwQR")) {
+        ivpq = std::make_unique<IndexIVFPQR>();
+        ivfpqr = static_cast<IndexIVFPQR*>(ivpq.get());
+    } else {
+        ivpq = std::make_unique<IndexIVFPQ>();
+    }
 
     std::vector<std::vector<idx_t>> ids;
-    read_ivf_header(ivpq, f, legacy ? &ids : nullptr);
+    read_ivf_header(ivpq.get(), f, legacy ? &ids : nullptr);
     READ1(ivpq->by_residual);
     READ1(ivpq->code_size);
     read_ProductQuantizer(&ivpq->pq, f);
 
     if (legacy) {
-        ArrayInvertedLists* ail = set_array_invlist(ivpq, ids);
+        ArrayInvertedLists* ail = set_array_invlist(ivpq.get(), ids);
         for (size_t i = 0; i < ail->nlist; i++)
             READVECTOR(ail->codes[i]);
     } else {
-        read_InvertedLists(ivpq, f, io_flags);
+        read_InvertedLists(*ivpq, f, io_flags);
     }
 
     if (ivpq->is_trained) {
@@ -721,47 +922,53 @@ static IndexIVFPQ* read_ivfpq(IOReader* f, uint32_t h, int io_flags) {
 
 int read_old_fmt_hack = 0;
 
-Index* read_index(IOReader* f, int io_flags) {
-    Index* idx = nullptr;
+std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
+    std::unique_ptr<Index> idx;
     uint32_t h;
     READ1(h);
     if (h == fourcc("null")) {
         // denotes a missing index, useful for some cases
-        return nullptr;
-    } else if (h == fourcc("IxFP")) {
+        return idx;
+    } else if (h == fourcc("IxFP") || h == fourcc("IxFp")) {
         int d;
         size_t n_levels, batch_size;
         READ1(d);
         READ1(n_levels);
+        FAISS_THROW_IF_NOT_FMT(n_levels > 0, "invalid n_levels %zd", n_levels);
         READ1(batch_size);
-        IndexFlatL2Panorama* idxp =
-                new IndexFlatL2Panorama(d, n_levels, batch_size);
+        std::unique_ptr<IndexFlatPanorama> idxp;
+        if (h == fourcc("IxFP")) {
+            idxp = std::make_unique<IndexFlatL2Panorama>(
+                    d, n_levels, batch_size);
+        } else {
+            idxp = std::make_unique<IndexFlatIPPanorama>(
+                    d, n_levels, batch_size);
+        }
         READ1(idxp->ntotal);
         READ1(idxp->is_trained);
         READVECTOR(idxp->codes);
         READVECTOR(idxp->cum_sums);
         idxp->verbose = false;
-        idx = idxp;
+        idx = std::move(idxp);
     } else if (
             h == fourcc("IxFI") || h == fourcc("IxF2") || h == fourcc("IxFl")) {
-        IndexFlat* idxf;
+        std::unique_ptr<IndexFlat> idxf;
         if (h == fourcc("IxFI")) {
-            idxf = new IndexFlatIP();
+            idxf = std::make_unique<IndexFlatIP>();
         } else if (h == fourcc("IxF2")) {
-            idxf = new IndexFlatL2();
+            idxf = std::make_unique<IndexFlatL2>();
         } else {
-            idxf = new IndexFlat();
+            idxf = std::make_unique<IndexFlat>();
         }
-        read_index_header(idxf, f);
+        read_index_header(*idxf, f);
         idxf->code_size = idxf->d * sizeof(float);
         read_xb_vector(idxf->codes, f);
         FAISS_THROW_IF_NOT(
                 idxf->codes.size() == idxf->ntotal * idxf->code_size);
-        // leak!
-        idx = idxf;
+        idx = std::move(idxf);
     } else if (h == fourcc("IxHE") || h == fourcc("IxHe")) {
-        IndexLSH* idxl = new IndexLSH();
-        read_index_header(idxl, f);
+        auto idxl = std::make_unique<IndexLSH>();
+        read_index_header(*idxl, f);
         READ1(idxl->nbits);
         READ1(idxl->rotate_data);
         READ1(idxl->train_thresholds);
@@ -779,23 +986,24 @@ Index* read_index(IOReader* f, int io_flags) {
             idxl->code_size *= 8;
         }
         {
-            RandomRotationMatrix* rrot = dynamic_cast<RandomRotationMatrix*>(
-                    read_VectorTransform(f));
+            // Read, dereference, discard.
+            auto sub_vt = read_VectorTransform_up(f);
+            RandomRotationMatrix* rrot =
+                    dynamic_cast<RandomRotationMatrix*>(sub_vt.get());
             FAISS_THROW_IF_NOT_MSG(rrot, "expected a random rotation");
             idxl->rrot = *rrot;
-            delete rrot;
         }
         read_vector(idxl->codes, f);
         FAISS_THROW_IF_NOT(
                 idxl->rrot.d_in == idxl->d && idxl->rrot.d_out == idxl->nbits);
         FAISS_THROW_IF_NOT(
                 idxl->codes.size() == idxl->ntotal * idxl->code_size);
-        idx = idxl;
+        idx = std::move(idxl);
     } else if (
             h == fourcc("IxPQ") || h == fourcc("IxPo") || h == fourcc("IxPq")) {
         // IxPQ and IxPo were merged into the same IndexPQ object
-        IndexPQ* idxp = new IndexPQ();
-        read_index_header(idxp, f);
+        auto idxp = std::make_unique<IndexPQ>();
+        read_index_header(*idxp, f);
         read_ProductQuantizer(&idxp->pq, f);
         idxp->code_size = idxp->pq.code_size;
         read_vector(idxp->codes, f);
@@ -810,43 +1018,43 @@ Index* read_index(IOReader* f, int io_flags) {
         if (h == fourcc("IxPQ") || h == fourcc("IxPo")) {
             idxp->metric_type = METRIC_L2;
         }
-        idx = idxp;
+        idx = std::move(idxp);
     } else if (h == fourcc("IxRQ") || h == fourcc("IxRq")) {
-        IndexResidualQuantizer* idxr = new IndexResidualQuantizer();
-        read_index_header(idxr, f);
+        auto idxr = std::make_unique<IndexResidualQuantizer>();
+        read_index_header(*idxr, f);
         if (h == fourcc("IxRQ")) {
-            read_ResidualQuantizer_old(&idxr->rq, f);
+            read_ResidualQuantizer_old(idxr->rq, f);
         } else {
-            read_ResidualQuantizer(&idxr->rq, f, io_flags);
+            read_ResidualQuantizer(idxr->rq, f, io_flags);
         }
         READ1(idxr->code_size);
         read_vector(idxr->codes, f);
-        idx = idxr;
+        idx = std::move(idxr);
     } else if (h == fourcc("IxLS")) {
-        auto idxr = new IndexLocalSearchQuantizer();
-        read_index_header(idxr, f);
-        read_LocalSearchQuantizer(&idxr->lsq, f);
+        auto idxr = std::make_unique<IndexLocalSearchQuantizer>();
+        read_index_header(*idxr, f);
+        read_LocalSearchQuantizer(idxr->lsq, f);
         READ1(idxr->code_size);
         read_vector(idxr->codes, f);
-        idx = idxr;
+        idx = std::move(idxr);
     } else if (h == fourcc("IxPR")) {
-        auto idxpr = new IndexProductResidualQuantizer();
-        read_index_header(idxpr, f);
-        read_ProductResidualQuantizer(&idxpr->prq, f, io_flags);
+        auto idxpr = std::make_unique<IndexProductResidualQuantizer>();
+        read_index_header(*idxpr, f);
+        read_ProductResidualQuantizer(idxpr->prq, f, io_flags);
         READ1(idxpr->code_size);
         read_vector(idxpr->codes, f);
-        idx = idxpr;
+        idx = std::move(idxpr);
     } else if (h == fourcc("IxPL")) {
-        auto idxpl = new IndexProductLocalSearchQuantizer();
-        read_index_header(idxpl, f);
-        read_ProductLocalSearchQuantizer(&idxpl->plsq, f);
+        auto idxpl = std::make_unique<IndexProductLocalSearchQuantizer>();
+        read_index_header(*idxpl, f);
+        read_ProductLocalSearchQuantizer(idxpl->plsq, f);
         READ1(idxpl->code_size);
         read_vector(idxpl->codes, f);
-        idx = idxpl;
+        idx = std::move(idxpl);
     } else if (h == fourcc("ImRQ")) {
-        ResidualCoarseQuantizer* idxr = new ResidualCoarseQuantizer();
-        read_index_header(idxr, f);
-        read_ResidualQuantizer(&idxr->rq, f, io_flags);
+        auto idxr = std::make_unique<ResidualCoarseQuantizer>();
+        read_index_header(*idxr, f);
+        read_ResidualQuantizer(idxr->rq, f, io_flags);
         READ1(idxr->beam_factor);
         if (io_flags & IO_FLAG_SKIP_PRECOMPUTE_TABLE) {
             // then we force the beam factor to -1
@@ -854,7 +1062,7 @@ Index* read_index(IOReader* f, int io_flags) {
             idxr->beam_factor = -1;
         }
         idxr->set_beam_factor(idxr->beam_factor);
-        idx = idxr;
+        idx = std::move(idxr);
     } else if (
             h == fourcc("ILfs") || h == fourcc("IRfs") || h == fourcc("IPRf") ||
             h == fourcc("IPLf")) {
@@ -862,29 +1070,30 @@ Index* read_index(IOReader* f, int io_flags) {
         bool is_RQ = h == fourcc("IRfs");
         bool is_PLSQ = h == fourcc("IPLf");
 
-        IndexAdditiveQuantizerFastScan* idxaqfs;
+        std::unique_ptr<IndexAdditiveQuantizerFastScan> idxaqfs;
         if (is_LSQ) {
-            idxaqfs = new IndexLocalSearchQuantizerFastScan();
+            idxaqfs = std::make_unique<IndexLocalSearchQuantizerFastScan>();
         } else if (is_RQ) {
-            idxaqfs = new IndexResidualQuantizerFastScan();
+            idxaqfs = std::make_unique<IndexResidualQuantizerFastScan>();
         } else if (is_PLSQ) {
-            idxaqfs = new IndexProductLocalSearchQuantizerFastScan();
+            idxaqfs = std::make_unique<
+                    IndexProductLocalSearchQuantizerFastScan>();
         } else {
-            idxaqfs = new IndexProductResidualQuantizerFastScan();
+            idxaqfs = std::make_unique<IndexProductResidualQuantizerFastScan>();
         }
-        read_index_header(idxaqfs, f);
+        read_index_header(*idxaqfs, f);
 
         if (is_LSQ) {
-            read_LocalSearchQuantizer((LocalSearchQuantizer*)idxaqfs->aq, f);
+            read_LocalSearchQuantizer(*(LocalSearchQuantizer*)idxaqfs->aq, f);
         } else if (is_RQ) {
             read_ResidualQuantizer(
-                    (ResidualQuantizer*)idxaqfs->aq, f, io_flags);
+                    *(ResidualQuantizer*)idxaqfs->aq, f, io_flags);
         } else if (is_PLSQ) {
             read_ProductLocalSearchQuantizer(
-                    (ProductLocalSearchQuantizer*)idxaqfs->aq, f);
+                    *(ProductLocalSearchQuantizer*)idxaqfs->aq, f);
         } else {
             read_ProductResidualQuantizer(
-                    (ProductResidualQuantizer*)idxaqfs->aq, f, io_flags);
+                    *(ProductResidualQuantizer*)idxaqfs->aq, f, io_flags);
         }
 
         READ1(idxaqfs->implem);
@@ -903,7 +1112,7 @@ Index* read_index(IOReader* f, int io_flags) {
         READ1(idxaqfs->max_train_points);
 
         READVECTOR(idxaqfs->codes);
-        idx = idxaqfs;
+        idx = std::move(idxaqfs);
     } else if (
             h == fourcc("IVLf") || h == fourcc("IVRf") || h == fourcc("NPLf") ||
             h == fourcc("NPRf")) {
@@ -911,28 +1120,31 @@ Index* read_index(IOReader* f, int io_flags) {
         bool is_RQ = h == fourcc("IVRf");
         bool is_PLSQ = h == fourcc("NPLf");
 
-        IndexIVFAdditiveQuantizerFastScan* ivaqfs;
+        std::unique_ptr<IndexIVFAdditiveQuantizerFastScan> ivaqfs;
         if (is_LSQ) {
-            ivaqfs = new IndexIVFLocalSearchQuantizerFastScan();
+            ivaqfs = std::make_unique<IndexIVFLocalSearchQuantizerFastScan>();
         } else if (is_RQ) {
-            ivaqfs = new IndexIVFResidualQuantizerFastScan();
+            ivaqfs = std::make_unique<IndexIVFResidualQuantizerFastScan>();
         } else if (is_PLSQ) {
-            ivaqfs = new IndexIVFProductLocalSearchQuantizerFastScan();
+            ivaqfs = std::make_unique<
+                    IndexIVFProductLocalSearchQuantizerFastScan>();
         } else {
-            ivaqfs = new IndexIVFProductResidualQuantizerFastScan();
+            ivaqfs = std::make_unique<
+                    IndexIVFProductResidualQuantizerFastScan>();
         }
-        read_ivf_header(ivaqfs, f);
+        read_ivf_header(ivaqfs.get(), f);
 
         if (is_LSQ) {
-            read_LocalSearchQuantizer((LocalSearchQuantizer*)ivaqfs->aq, f);
+            read_LocalSearchQuantizer(*(LocalSearchQuantizer*)ivaqfs->aq, f);
         } else if (is_RQ) {
-            read_ResidualQuantizer((ResidualQuantizer*)ivaqfs->aq, f, io_flags);
+            read_ResidualQuantizer(
+                    *(ResidualQuantizer*)ivaqfs->aq, f, io_flags);
         } else if (is_PLSQ) {
             read_ProductLocalSearchQuantizer(
-                    (ProductLocalSearchQuantizer*)ivaqfs->aq, f);
+                    *(ProductLocalSearchQuantizer*)ivaqfs->aq, f);
         } else {
             read_ProductResidualQuantizer(
-                    (ProductResidualQuantizer*)ivaqfs->aq, f, io_flags);
+                    *(ProductResidualQuantizer*)ivaqfs->aq, f, io_flags);
         }
 
         READ1(ivaqfs->by_residual);
@@ -951,15 +1163,15 @@ Index* read_index(IOReader* f, int io_flags) {
         READ1(ivaqfs->norm_scale);
         READ1(ivaqfs->max_train_points);
 
-        read_InvertedLists(ivaqfs, f, io_flags);
+        read_InvertedLists(*ivaqfs, f, io_flags);
         ivaqfs->init_code_packer();
-        idx = ivaqfs;
+        idx = std::move(ivaqfs);
     } else if (h == fourcc("IvFl") || h == fourcc("IvFL")) { // legacy
-        IndexIVFFlat* ivfl = new IndexIVFFlat();
+        auto ivfl = std::make_unique<IndexIVFFlat>();
         std::vector<std::vector<idx_t>> ids;
-        read_ivf_header(ivfl, f, &ids);
+        read_ivf_header(ivfl.get(), f, &ids);
         ivfl->code_size = ivfl->d * sizeof(float);
-        ArrayInvertedLists* ail = set_array_invlist(ivfl, ids);
+        ArrayInvertedLists* ail = set_array_invlist(ivfl.get(), ids);
 
         if (h == fourcc("IvFL")) {
             for (size_t i = 0; i < ivfl->nlist; i++) {
@@ -973,64 +1185,85 @@ Index* read_index(IOReader* f, int io_flags) {
                 memcpy(ail->codes[i].data(), vec.data(), ail->codes[i].size());
             }
         }
-        idx = ivfl;
+        idx = std::move(ivfl);
     } else if (h == fourcc("IwFd")) {
-        IndexIVFFlatDedup* ivfl = new IndexIVFFlatDedup();
-        read_ivf_header(ivfl, f);
+        auto ivfl = std::make_unique<IndexIVFFlatDedup>();
+        read_ivf_header(ivfl.get(), f);
         ivfl->code_size = ivfl->d * sizeof(float);
         {
             std::vector<idx_t> tab;
             READVECTOR(tab);
+            FAISS_THROW_IF_NOT_FMT(
+                    tab.size() % 2 == 0,
+                    "invalid IVFFlatDedup instances table size: %zd "
+                    "(must be even)",
+                    tab.size());
             for (long i = 0; i < tab.size(); i += 2) {
                 std::pair<idx_t, idx_t> pair(tab[i], tab[i + 1]);
                 ivfl->instances.insert(pair);
             }
         }
-        read_InvertedLists(ivfl, f, io_flags);
-        idx = ivfl;
+        read_InvertedLists(*ivfl, f, io_flags);
+        idx = std::move(ivfl);
     } else if (h == fourcc("IwPn")) {
-        IndexIVFFlatPanorama* ivfp = new IndexIVFFlatPanorama();
-        read_ivf_header(ivfp, f);
+        auto ivfp = std::make_unique<IndexIVFFlatPanorama>();
+        read_ivf_header(ivfp.get(), f);
         ivfp->code_size = ivfp->d * sizeof(float);
         READ1(ivfp->n_levels);
-        read_InvertedLists(ivfp, f, io_flags);
-        idx = ivfp;
+        read_InvertedLists(*ivfp, f, io_flags);
+        idx = std::move(ivfp);
     } else if (h == fourcc("IwFl")) {
-        IndexIVFFlat* ivfl = new IndexIVFFlat();
-        read_ivf_header(ivfl, f);
+        auto ivfl = std::make_unique<IndexIVFFlat>();
+        read_ivf_header(ivfl.get(), f);
         ivfl->code_size = ivfl->d * sizeof(float);
-        read_InvertedLists(ivfl, f, io_flags);
-        idx = ivfl;
+        read_InvertedLists(*ivfl, f, io_flags);
+        idx = std::move(ivfl);
     } else if (h == fourcc("IxSQ")) {
-        IndexScalarQuantizer* idxs = new IndexScalarQuantizer();
-        read_index_header(idxs, f);
+        auto idxs = std::make_unique<IndexScalarQuantizer>();
+        read_index_header(*idxs, f);
         read_ScalarQuantizer(&idxs->sq, f);
         read_vector(idxs->codes, f);
         idxs->code_size = idxs->sq.code_size;
-        idx = idxs;
+        idx = std::move(idxs);
     } else if (h == fourcc("IxLa")) {
         int d, nsq, scale_nbit, r2;
         READ1(d);
         READ1(nsq);
         READ1(scale_nbit);
         READ1(r2);
-        IndexLattice* idxl = new IndexLattice(d, nsq, scale_nbit, r2);
-        read_index_header(idxl, f);
+        FAISS_THROW_IF_NOT_FMT(
+                nsq > 0, "invalid IndexLattice nsq %d (must be > 0)", nsq);
+        FAISS_THROW_IF_NOT_FMT(
+                d > 0 && d % nsq == 0,
+                "invalid IndexLattice d=%d, nsq=%d (d must be > 0 and divisible by nsq)",
+                d,
+                nsq);
+        FAISS_THROW_IF_NOT_FMT(
+                r2 > 0, "invalid IndexLattice r2 %d (must be > 0)", r2);
+        int dsq = d / nsq;
+        FAISS_THROW_IF_NOT_FMT(
+                dsq >= 2 && (dsq & (dsq - 1)) == 0,
+                "invalid IndexLattice d=%d, nsq=%d: d/nsq=%d must be a power of 2 >= 2",
+                d,
+                nsq,
+                dsq);
+        auto idxl = std::make_unique<IndexLattice>(d, nsq, scale_nbit, r2);
+        read_index_header(*idxl, f);
         READVECTOR(idxl->trained);
-        idx = idxl;
+        idx = std::move(idxl);
     } else if (h == fourcc("IvSQ")) { // legacy
-        IndexIVFScalarQuantizer* ivsc = new IndexIVFScalarQuantizer();
+        auto ivsc = std::make_unique<IndexIVFScalarQuantizer>();
         std::vector<std::vector<idx_t>> ids;
-        read_ivf_header(ivsc, f, &ids);
+        read_ivf_header(ivsc.get(), f, &ids);
         read_ScalarQuantizer(&ivsc->sq, f);
         READ1(ivsc->code_size);
-        ArrayInvertedLists* ail = set_array_invlist(ivsc, ids);
+        ArrayInvertedLists* ail = set_array_invlist(ivsc.get(), ids);
         for (int i = 0; i < ivsc->nlist; i++)
             READVECTOR(ail->codes[i]);
-        idx = ivsc;
+        idx = std::move(ivsc);
     } else if (h == fourcc("IwSQ") || h == fourcc("IwSq")) {
-        IndexIVFScalarQuantizer* ivsc = new IndexIVFScalarQuantizer();
-        read_ivf_header(ivsc, f);
+        auto ivsc = std::make_unique<IndexIVFScalarQuantizer>();
+        read_ivf_header(ivsc.get(), f);
         read_ScalarQuantizer(&ivsc->sq, f);
         READ1(ivsc->code_size);
         if (h == fourcc("IwSQ")) {
@@ -1038,44 +1271,44 @@ Index* read_index(IOReader* f, int io_flags) {
         } else {
             READ1(ivsc->by_residual);
         }
-        read_InvertedLists(ivsc, f, io_flags);
-        idx = ivsc;
+        read_InvertedLists(*ivsc, f, io_flags);
+        idx = std::move(ivsc);
     } else if (
             h == fourcc("IwLS") || h == fourcc("IwRQ") || h == fourcc("IwPL") ||
             h == fourcc("IwPR")) {
         bool is_LSQ = h == fourcc("IwLS");
         bool is_RQ = h == fourcc("IwRQ");
         bool is_PLSQ = h == fourcc("IwPL");
-        IndexIVFAdditiveQuantizer* iva;
+        std::unique_ptr<IndexIVFAdditiveQuantizer> iva;
         if (is_LSQ) {
-            iva = new IndexIVFLocalSearchQuantizer();
+            iva = std::make_unique<IndexIVFLocalSearchQuantizer>();
         } else if (is_RQ) {
-            iva = new IndexIVFResidualQuantizer();
+            iva = std::make_unique<IndexIVFResidualQuantizer>();
         } else if (is_PLSQ) {
-            iva = new IndexIVFProductLocalSearchQuantizer();
+            iva = std::make_unique<IndexIVFProductLocalSearchQuantizer>();
         } else {
-            iva = new IndexIVFProductResidualQuantizer();
+            iva = std::make_unique<IndexIVFProductResidualQuantizer>();
         }
-        read_ivf_header(iva, f);
+        read_ivf_header(iva.get(), f);
         READ1(iva->code_size);
         if (is_LSQ) {
-            read_LocalSearchQuantizer((LocalSearchQuantizer*)iva->aq, f);
+            read_LocalSearchQuantizer(*(LocalSearchQuantizer*)iva->aq, f);
         } else if (is_RQ) {
-            read_ResidualQuantizer((ResidualQuantizer*)iva->aq, f, io_flags);
+            read_ResidualQuantizer(*(ResidualQuantizer*)iva->aq, f, io_flags);
         } else if (is_PLSQ) {
             read_ProductLocalSearchQuantizer(
-                    (ProductLocalSearchQuantizer*)iva->aq, f);
+                    *(ProductLocalSearchQuantizer*)iva->aq, f);
         } else {
             read_ProductResidualQuantizer(
-                    (ProductResidualQuantizer*)iva->aq, f, io_flags);
+                    *(ProductResidualQuantizer*)iva->aq, f, io_flags);
         }
         READ1(iva->by_residual);
         READ1(iva->use_precomputed_table);
-        read_InvertedLists(iva, f, io_flags);
-        idx = iva;
+        read_InvertedLists(*iva, f, io_flags);
+        idx = std::move(iva);
     } else if (h == fourcc("IwSh")) {
-        IndexIVFSpectralHash* ivsp = new IndexIVFSpectralHash();
-        read_ivf_header(ivsp, f);
+        auto ivsp = std::make_unique<IndexIVFSpectralHash>();
+        read_ivf_header(ivsp.get(), f);
         ivsp->vt = read_VectorTransform(f);
         ivsp->own_fields = true;
         READ1(ivsp->nbit);
@@ -1084,32 +1317,34 @@ Index* read_index(IOReader* f, int io_flags) {
         READ1(ivsp->period);
         READ1(ivsp->threshold_type);
         READVECTOR(ivsp->trained);
-        read_InvertedLists(ivsp, f, io_flags);
-        idx = ivsp;
+        read_InvertedLists(*ivsp, f, io_flags);
+        idx = std::move(ivsp);
     } else if (
             h == fourcc("IvPQ") || h == fourcc("IvQR") || h == fourcc("IwPQ") ||
             h == fourcc("IwQR")) {
         idx = read_ivfpq(f, h, io_flags);
     } else if (h == fourcc("IwIQ")) {
-        auto* indep = new IndexIVFIndependentQuantizer();
+        auto indep = std::make_unique<IndexIVFIndependentQuantizer>();
         indep->own_fields = true;
-        read_index_header(indep, f);
+        read_index_header(*indep, f);
         indep->quantizer = read_index(f, io_flags);
         bool has_vt;
         READ1(has_vt);
         if (has_vt) {
             indep->vt = read_VectorTransform(f);
         }
-        indep->index_ivf = dynamic_cast<IndexIVF*>(read_index(f, io_flags));
+        auto ivf_idx = read_index_up(f, io_flags);
+        indep->index_ivf = dynamic_cast<IndexIVF*>(ivf_idx.get());
         FAISS_THROW_IF_NOT(indep->index_ivf);
+        ivf_idx.release();
         if (auto index_ivfpq = dynamic_cast<IndexIVFPQ*>(indep->index_ivf)) {
             READ1(index_ivfpq->use_precomputed_table);
         }
-        idx = indep;
+        idx = std::move(indep);
     } else if (h == fourcc("IxPT")) {
-        IndexPreTransform* ixpt = new IndexPreTransform();
+        auto ixpt = std::make_unique<IndexPreTransform>();
         ixpt->own_fields = true;
-        read_index_header(ixpt, f);
+        read_index_header(*ixpt, f);
         int nt;
         if (read_old_fmt_hack == 2) {
             nt = 1;
@@ -1120,48 +1355,50 @@ Index* read_index(IOReader* f, int io_flags) {
             ixpt->chain.push_back(read_VectorTransform(f));
         }
         ixpt->index = read_index(f, io_flags);
-        idx = ixpt;
+        idx = std::move(ixpt);
     } else if (h == fourcc("Imiq")) {
-        MultiIndexQuantizer* imiq = new MultiIndexQuantizer();
-        read_index_header(imiq, f);
+        auto imiq = std::make_unique<MultiIndexQuantizer>();
+        read_index_header(*imiq, f);
         read_ProductQuantizer(&imiq->pq, f);
-        idx = imiq;
+        idx = std::move(imiq);
     } else if (h == fourcc("IxRF") || h == fourcc("IxRP")) {
-        IndexRefine* idxrf = new IndexRefine();
-        read_index_header(idxrf, f);
-        idxrf->base_index = read_index(f, io_flags);
-        idxrf->refine_index = read_index(f, io_flags);
+        auto idxrf = std::make_unique<IndexRefine>();
+        read_index_header(*idxrf, f);
+        auto base = read_index_up(f, io_flags);
+        auto refine = read_index_up(f, io_flags);
         READ1(idxrf->k_factor);
         if (h == fourcc("IxRP")) {
             // then make a RefineFlatPanorama with it
-            IndexRefine* idxrf_old = idxrf;
-            idxrf = new IndexRefinePanorama();
-            *idxrf = *idxrf_old;
-            delete idxrf_old;
-        } else if (dynamic_cast<IndexFlat*>(idxrf->refine_index)) {
+            auto idxrf_new = std::make_unique<IndexRefinePanorama>();
+            static_cast<IndexRefine&>(*idxrf_new) = *idxrf;
+            idxrf = std::move(idxrf_new);
+        } else if (dynamic_cast<IndexFlat*>(refine.get())) {
             // then make a RefineFlat with it
-            IndexRefine* idxrf_old = idxrf;
-            idxrf = new IndexRefineFlat();
-            *idxrf = *idxrf_old;
-            delete idxrf_old;
+            auto idxrf_new = std::make_unique<IndexRefineFlat>();
+            static_cast<IndexRefine&>(*idxrf_new) = *idxrf;
+            idxrf = std::move(idxrf_new);
         }
+        idxrf->base_index = base.release();
+        idxrf->refine_index = refine.release();
         idxrf->own_fields = true;
         idxrf->own_refine_index = true;
-        idx = idxrf;
+        idx = std::move(idxrf);
     } else if (h == fourcc("IxMp") || h == fourcc("IxM2")) {
         bool is_map2 = h == fourcc("IxM2");
-        IndexIDMap* idxmap = is_map2 ? new IndexIDMap2() : new IndexIDMap();
-        read_index_header(idxmap, f);
+        std::unique_ptr<IndexIDMap> idxmap = is_map2
+                ? std::make_unique<IndexIDMap2>()
+                : std::make_unique<IndexIDMap>();
+        read_index_header(*idxmap, f);
         idxmap->index = read_index(f, io_flags);
         idxmap->own_fields = true;
         READVECTOR(idxmap->id_map);
         if (is_map2) {
-            static_cast<IndexIDMap2*>(idxmap)->construct_rev_map();
+            static_cast<IndexIDMap2*>(idxmap.get())->construct_rev_map();
         }
-        idx = idxmap;
+        idx = std::move(idxmap);
     } else if (h == fourcc("Ix2L")) {
-        Index2Layer* idxp = new Index2Layer();
-        read_index_header(idxp, f);
+        auto idxp = std::make_unique<Index2Layer>();
+        read_index_header(*idxp, f);
         idxp->q1.quantizer = read_index(f, io_flags);
         READ1(idxp->q1.nlist);
         READ1(idxp->q1.quantizer_trains_alone);
@@ -1170,46 +1407,47 @@ Index* read_index(IOReader* f, int io_flags) {
         READ1(idxp->code_size_2);
         READ1(idxp->code_size);
         read_vector(idxp->codes, f);
-        idx = idxp;
+        idx = std::move(idxp);
     } else if (
             h == fourcc("IHNf") || h == fourcc("IHNp") || h == fourcc("IHNs") ||
             h == fourcc("IHN2") || h == fourcc("IHNc") || h == fourcc("IHc2") ||
             h == fourcc("IHfP")) {
-        IndexHNSW* idxhnsw = nullptr;
+        std::unique_ptr<IndexHNSW> idxhnsw;
         if (h == fourcc("IHNf")) {
-            idxhnsw = new IndexHNSWFlat();
+            idxhnsw = std::make_unique<IndexHNSWFlat>();
         }
         if (h == fourcc("IHfP")) {
-            idxhnsw = new IndexHNSWFlatPanorama();
+            idxhnsw = std::make_unique<IndexHNSWFlatPanorama>();
         }
         if (h == fourcc("IHNp")) {
-            idxhnsw = new IndexHNSWPQ();
+            idxhnsw = std::make_unique<IndexHNSWPQ>();
         }
         if (h == fourcc("IHNs")) {
-            idxhnsw = new IndexHNSWSQ();
+            idxhnsw = std::make_unique<IndexHNSWSQ>();
         }
         if (h == fourcc("IHN2")) {
-            idxhnsw = new IndexHNSW2Level();
+            idxhnsw = std::make_unique<IndexHNSW2Level>();
         }
         if (h == fourcc("IHNc")) {
-            idxhnsw = new IndexHNSWCagra();
+            idxhnsw = std::make_unique<IndexHNSWCagra>();
         }
         if (h == fourcc("IHc2")) {
-            idxhnsw = new IndexHNSWCagra();
+            idxhnsw = std::make_unique<IndexHNSWCagra>();
         }
-        read_index_header(idxhnsw, f);
+        read_index_header(*idxhnsw, f);
         if (h == fourcc("IHfP")) {
-            auto idx_panorama = dynamic_cast<IndexHNSWFlatPanorama*>(idxhnsw);
+            auto idx_panorama =
+                    dynamic_cast<IndexHNSWFlatPanorama*>(idxhnsw.get());
             size_t nlevels;
             READ1(nlevels);
             const_cast<size_t&>(idx_panorama->num_panorama_levels) = nlevels;
-            const_cast<size_t&>(idx_panorama->panorama_level_width) =
-                    (idx_panorama->d + nlevels - 1) / nlevels;
+            const_cast<Panorama&>(idx_panorama->pano) =
+                    Panorama(idx_panorama->d * sizeof(float), nlevels, 1);
             READVECTOR(idx_panorama->cum_sums);
         }
         if (h == fourcc("IHNc") || h == fourcc("IHc2")) {
             READ1(idxhnsw->keep_max_size_level0);
-            auto idx_hnsw_cagra = dynamic_cast<IndexHNSWCagra*>(idxhnsw);
+            auto idx_hnsw_cagra = dynamic_cast<IndexHNSWCagra*>(idxhnsw.get());
             READ1(idx_hnsw_cagra->base_level_only);
             READ1(idx_hnsw_cagra->num_base_level_search_entrypoints);
             if (h == fourcc("IHc2")) {
@@ -1218,47 +1456,47 @@ Index* read_index(IOReader* f, int io_flags) {
                 idx_hnsw_cagra->set_numeric_type(faiss::Float32);
             }
         }
-        read_HNSW(&idxhnsw->hnsw, f);
+        read_HNSW(idxhnsw->hnsw, f);
         idxhnsw->hnsw.is_panorama = (h == fourcc("IHfP"));
         idxhnsw->storage = read_index(f, io_flags);
         idxhnsw->own_fields = idxhnsw->storage != nullptr;
         if (h == fourcc("IHNp") && !(io_flags & IO_FLAG_PQ_SKIP_SDC_TABLE)) {
             dynamic_cast<IndexPQ*>(idxhnsw->storage)->pq.compute_sdc_table();
         }
-        idx = idxhnsw;
+        idx = std::move(idxhnsw);
     } else if (
             h == fourcc("INSf") || h == fourcc("INSp") || h == fourcc("INSs")) {
-        IndexNSG* idxnsg;
+        std::unique_ptr<IndexNSG> idxnsg;
         if (h == fourcc("INSf")) {
-            idxnsg = new IndexNSGFlat();
+            idxnsg = std::make_unique<IndexNSGFlat>();
         }
         if (h == fourcc("INSp")) {
-            idxnsg = new IndexNSGPQ();
+            idxnsg = std::make_unique<IndexNSGPQ>();
         }
         if (h == fourcc("INSs")) {
-            idxnsg = new IndexNSGSQ();
+            idxnsg = std::make_unique<IndexNSGSQ>();
         }
-        read_index_header(idxnsg, f);
+        read_index_header(*idxnsg, f);
         READ1(idxnsg->GK);
         READ1(idxnsg->build_type);
         READ1(idxnsg->nndescent_S);
         READ1(idxnsg->nndescent_R);
         READ1(idxnsg->nndescent_L);
         READ1(idxnsg->nndescent_iter);
-        read_NSG(&idxnsg->nsg, f);
+        read_NSG(idxnsg->nsg, f);
         idxnsg->storage = read_index(f, io_flags);
         idxnsg->own_fields = true;
-        idx = idxnsg;
+        idx = std::move(idxnsg);
     } else if (h == fourcc("INNf")) {
-        IndexNNDescent* idxnnd = new IndexNNDescentFlat();
-        read_index_header(idxnnd, f);
-        read_NNDescent(&idxnnd->nndescent, f);
+        auto idxnnd = std::make_unique<IndexNNDescentFlat>();
+        read_index_header(*idxnnd, f);
+        read_NNDescent(idxnnd->nndescent, f);
         idxnnd->storage = read_index(f, io_flags);
         idxnnd->own_fields = true;
-        idx = idxnnd;
+        idx = std::move(idxnnd);
     } else if (h == fourcc("IPfs")) {
-        IndexPQFastScan* idxpqfs = new IndexPQFastScan();
-        read_index_header(idxpqfs, f);
+        auto idxpqfs = std::make_unique<IndexPQFastScan>();
+        read_index_header(*idxpqfs, f);
         read_ProductQuantizer(&idxpqfs->pq, f);
         READ1(idxpqfs->implem);
         READ1(idxpqfs->bbs);
@@ -1273,11 +1511,11 @@ Index* read_index(IOReader* f, int io_flags) {
         idxpqfs->ksub = (1 << pq.nbits);
         idxpqfs->code_size = pq.code_size;
 
-        idx = idxpqfs;
+        idx = std::move(idxpqfs);
 
     } else if (h == fourcc("IwPf")) {
-        IndexIVFPQFastScan* ivpq = new IndexIVFPQFastScan();
-        read_ivf_header(ivpq, f);
+        auto ivpq = std::make_unique<IndexIVFPQFastScan>();
+        read_ivf_header(ivpq.get(), f);
         READ1(ivpq->by_residual);
         READ1(ivpq->code_size);
         READ1(ivpq->bbs);
@@ -1285,7 +1523,7 @@ Index* read_index(IOReader* f, int io_flags) {
         READ1(ivpq->implem);
         READ1(ivpq->qbs2);
         read_ProductQuantizer(&ivpq->pq, f);
-        read_InvertedLists(ivpq, f, io_flags);
+        read_InvertedLists(*ivpq, f, io_flags);
         ivpq->precompute_table();
 
         const auto& pq = ivpq->pq;
@@ -1295,30 +1533,38 @@ Index* read_index(IOReader* f, int io_flags) {
         ivpq->code_size = pq.code_size;
         ivpq->init_code_packer();
 
-        idx = ivpq;
+        idx = std::move(ivpq);
     } else if (h == fourcc("IRMf")) {
-        IndexRowwiseMinMax* imm = new IndexRowwiseMinMax();
-        read_index_header(imm, f);
+        auto imm = std::make_unique<IndexRowwiseMinMax>();
+        read_index_header(*imm, f);
 
         imm->index = read_index(f, io_flags);
         imm->own_fields = true;
 
-        idx = imm;
+        idx = std::move(imm);
     } else if (h == fourcc("IRMh")) {
-        IndexRowwiseMinMaxFP16* imm = new IndexRowwiseMinMaxFP16();
-        read_index_header(imm, f);
+        auto imm = std::make_unique<IndexRowwiseMinMaxFP16>();
+        read_index_header(*imm, f);
 
         imm->index = read_index(f, io_flags);
         imm->own_fields = true;
 
-        idx = imm;
-    } else if (h == fourcc("Irfs")) {
-        IndexRaBitQFastScan* idxqfs = new IndexRaBitQFastScan();
-        read_index_header(idxqfs, f);
-        read_RaBitQuantizer(&idxqfs->rabitq, f, true);
+        idx = std::move(imm);
+    } else if (h == fourcc("Irfn") || h == fourcc("Irfs")) {
+        // Irfn = new format (aux data embedded in SIMD blocks)
+        // Irfs = legacy format (flat_storage separate, needs migration)
+        const bool is_legacy = (h == fourcc("Irfs"));
+
+        auto idxqfs = std::make_unique<IndexRaBitQFastScan>();
+        read_index_header(*idxqfs, f);
+        read_RaBitQuantizer(idxqfs->rabitq, f, true);
         READVECTOR(idxqfs->center);
         READ1(idxqfs->qb);
-        READVECTOR(idxqfs->flat_storage);
+
+        std::vector<uint8_t> legacy_flat_storage;
+        if (is_legacy) {
+            READVECTOR(legacy_flat_storage);
+        }
 
         READ1(idxqfs->bbs);
         READ1(idxqfs->ntotal2);
@@ -1332,33 +1578,55 @@ Index* read_index(IOReader* f, int io_flags) {
         idxqfs->ksub = (1 << nbits_fastscan);
 
         READVECTOR(idxqfs->codes);
-        idx = idxqfs;
+
+        if (is_legacy) {
+            const size_t storage_size =
+                    rabitq_utils::compute_per_vector_storage_size(
+                            idxqfs->rabitq.nb_bits, idxqfs->d);
+
+            FAISS_THROW_IF_NOT_MSG(
+                    legacy_flat_storage.size() ==
+                            static_cast<size_t>(idxqfs->ntotal) * storage_size,
+                    "legacy flat_storage size mismatch during migration");
+
+            rabitq_utils::populate_block_aux_from_flat_storage(
+                    legacy_flat_storage,
+                    idxqfs->codes,
+                    static_cast<size_t>(idxqfs->ntotal),
+                    idxqfs->bbs,
+                    idxqfs->M2,
+                    ((idxqfs->M2 + 1) / 2) * idxqfs->bbs,
+                    idxqfs->get_block_stride(),
+                    storage_size);
+        }
+
+        idx = std::move(idxqfs);
     } else if (h == fourcc("Ixrq")) {
-        IndexRaBitQ* idxq = new IndexRaBitQ();
-        read_index_header(idxq, f);
-        read_RaBitQuantizer(&idxq->rabitq, f, false);
+        auto idxq = std::make_unique<IndexRaBitQ>();
+        read_index_header(*idxq, f);
+        read_RaBitQuantizer(idxq->rabitq, f, false);
         READVECTOR(idxq->codes);
         READVECTOR(idxq->center);
         READ1(idxq->qb);
 
         // rabitq.nb_bits is already set to 1 by read_RaBitQuantizer
         idxq->code_size = idxq->rabitq.code_size;
-        idx = idxq;
+        idx = std::move(idxq);
     } else if (h == fourcc("Ixrr")) {
         // Ixrr = multi-bit format (new)
-        IndexRaBitQ* idxq = new IndexRaBitQ();
-        read_index_header(idxq, f);
-        read_RaBitQuantizer(&idxq->rabitq, f, true); // Reads nb_bits from file
+        auto idxq = std::make_unique<IndexRaBitQ>();
+        read_index_header(*idxq, f);
+        read_RaBitQuantizer(idxq->rabitq, f, true); // Reads nb_bits from file
         READVECTOR(idxq->codes);
         READVECTOR(idxq->center);
         READ1(idxq->qb);
 
         idxq->code_size = idxq->rabitq.code_size;
-        idx = idxq;
+        idx = std::move(idxq);
     } else if (h == fourcc("Iwrq")) {
-        IndexIVFRaBitQ* ivrq = new IndexIVFRaBitQ();
-        read_ivf_header(ivrq, f);
-        read_RaBitQuantizer(&ivrq->rabitq, f, false);
+        auto ivrq = std::make_unique<IndexIVFRaBitQ>();
+        read_ivf_header(ivrq.get(), f);
+        read_RaBitQuantizer(ivrq->rabitq, f, false);
         READ1(ivrq->code_size);
         READ1(ivrq->by_residual);
         READ1(ivrq->qb);
@@ -1368,13 +1636,13 @@ Index* read_index(IOReader* f, int io_flags) {
         ivrq->rabitq.code_size =
                 ivrq->rabitq.compute_code_size(ivrq->d, ivrq->rabitq.nb_bits);
         ivrq->code_size = ivrq->rabitq.code_size;
-        read_InvertedLists(ivrq, f, io_flags);
-        idx = ivrq;
+        read_InvertedLists(*ivrq, f, io_flags);
+        idx = std::move(ivrq);
     } else if (h == fourcc("Iwrr")) {
         // Iwrr = multi-bit format (new)
-        IndexIVFRaBitQ* ivrq = new IndexIVFRaBitQ();
-        read_ivf_header(ivrq, f);
-        read_RaBitQuantizer(&ivrq->rabitq, f, true); // Reads nb_bits from file
+        auto ivrq = std::make_unique<IndexIVFRaBitQ>();
+        read_ivf_header(ivrq.get(), f);
+        read_RaBitQuantizer(ivrq->rabitq, f, true); // Reads nb_bits from file
         READ1(ivrq->code_size);
         READ1(ivrq->by_residual);
         READ1(ivrq->qb);
@@ -1383,22 +1651,22 @@ Index* read_index(IOReader* f, int io_flags) {
         ivrq->rabitq.code_size =
                 ivrq->rabitq.compute_code_size(ivrq->d, ivrq->rabitq.nb_bits);
         ivrq->code_size = ivrq->rabitq.code_size;
-        read_InvertedLists(ivrq, f, io_flags);
-        idx = ivrq;
+        read_InvertedLists(*ivrq, f, io_flags);
+        idx = std::move(ivrq);
     }
 #ifdef FAISS_ENABLE_SVS
     else if (
             h == fourcc("ILVQ") || h == fourcc("ISVL") || h == fourcc("ISVD")) {
-        IndexSVSVamana* svs;
+        std::unique_ptr<IndexSVSVamana> svs;
         if (h == fourcc("ILVQ")) {
-            svs = new IndexSVSVamanaLVQ();
+            svs = std::make_unique<IndexSVSVamanaLVQ>();
         } else if (h == fourcc("ISVL")) {
-            svs = new IndexSVSVamanaLeanVec();
+            svs = std::make_unique<IndexSVSVamanaLeanVec>();
         } else if (h == fourcc("ISVD")) {
-            svs = new IndexSVSVamana();
+            svs = std::make_unique<IndexSVSVamana>();
         }
 
-        read_index_header(svs, f);
+        read_index_header(*svs, f);
         READ1(svs->graph_max_degree);
         READ1(svs->alpha);
         READ1(svs->search_window_size);
@@ -1409,7 +1677,7 @@ Index* read_index(IOReader* f, int io_flags) {
         READ1(svs->use_full_search_history);
         READ1(svs->storage_kind);
         if (h == fourcc("ISVL")) {
-            READ1(dynamic_cast<IndexSVSVamanaLeanVec*>(svs)->leanvec_d);
+            READ1(dynamic_cast<IndexSVSVamanaLeanVec*>(svs.get())->leanvec_d);
         }
 
         bool initialized;
@@ -1425,14 +1693,14 @@ Index* read_index(IOReader* f, int io_flags) {
             if (trained) {
                 faiss::svs_io::ReaderStreambuf rbuf(f);
                 std::istream is(&rbuf);
-                dynamic_cast<IndexSVSVamanaLeanVec*>(svs)
+                dynamic_cast<IndexSVSVamanaLeanVec*>(svs.get())
                         ->deserialize_training_data(is);
             }
         }
-        idx = svs;
+        idx = std::move(svs);
     } else if (h == fourcc("ISVF")) {
-        IndexSVSFlat* svs = new IndexSVSFlat();
-        read_index_header(svs, f);
+        auto svs = std::make_unique<IndexSVSFlat>();
+        read_index_header(*svs, f);
 
         bool initialized;
         READ1(initialized);
@@ -1440,14 +1708,18 @@ Index* read_index(IOReader* f, int io_flags) {
             faiss::svs_io::ReaderStreambuf rbuf(f);
             std::istream is(&rbuf);
             svs->deserialize_impl(is);
-            idx = svs;
         }
+        idx = std::move(svs);
     }
 #endif // FAISS_ENABLE_SVS
-    else if (h == fourcc("Iwrf")) {
-        IndexIVFRaBitQFastScan* ivrqfs = new IndexIVFRaBitQFastScan();
-        read_ivf_header(ivrqfs, f);
-        read_RaBitQuantizer(&ivrqfs->rabitq, f);
+    else if (h == fourcc("Iwrn") || h == fourcc("Iwrf")) {
+        // Iwrn = new format (aux data embedded in SIMD blocks)
+        // Iwrf = legacy format (flat_storage separate, needs migration)
+        const bool is_legacy = (h == fourcc("Iwrf"));
+
+        auto ivrqfs = std::make_unique<IndexIVFRaBitQFastScan>();
+        read_ivf_header(ivrqfs.get(), f);
+        read_RaBitQuantizer(ivrqfs->rabitq, f);
         READ1(ivrqfs->by_residual);
         READ1(ivrqfs->code_size);
         READ1(ivrqfs->bbs);
@@ -1456,7 +1728,11 @@ Index* read_index(IOReader* f, int io_flags) {
         READ1(ivrqfs->implem);
         READ1(ivrqfs->qb);
         READ1(ivrqfs->centered);
-        READVECTOR(ivrqfs->flat_storage);
+
+        std::vector<uint8_t> legacy_flat_storage;
+        if (is_legacy) {
+            READVECTOR(legacy_flat_storage);
+        }
 
         // Initialize FastScan base class fields
         const size_t M_fastscan = (ivrqfs->d + 3) / 4;
@@ -1465,87 +1741,140 @@ Index* read_index(IOReader* f, int io_flags) {
         ivrqfs->nbits = nbits_fastscan;
         ivrqfs->ksub = (1 << nbits_fastscan);
 
-        read_InvertedLists(ivrqfs, f, io_flags);
+        read_InvertedLists(*ivrqfs, f, io_flags);
         ivrqfs->init_code_packer();
-        idx = ivrqfs;
+
+        if (is_legacy) {
+            auto* bil = dynamic_cast<BlockInvertedLists*>(ivrqfs->invlists);
+            FAISS_THROW_IF_NOT(bil);
+
+            const size_t storage_size =
+                    rabitq_utils::compute_per_vector_storage_size(
+                            ivrqfs->rabitq.nb_bits, ivrqfs->d);
+            const size_t new_block_stride = ivrqfs->get_block_stride();
+
+            for (size_t list_no = 0; list_no < ivrqfs->nlist; list_no++) {
+                if (bil->list_size(list_no) == 0) {
+                    continue;
+                }
+                rabitq_utils::populate_block_aux_from_flat_storage(
+                        legacy_flat_storage,
+                        bil->codes[list_no],
+                        bil->list_size(list_no),
+                        ivrqfs->bbs,
+                        ivrqfs->M2,
+                        bil->block_size,
+                        new_block_stride,
+                        storage_size,
+                        bil->ids[list_no].data());
+            }
+
+            if (bil->block_size < new_block_stride) {
+                bil->block_size = new_block_stride;
+            }
+        }
+
+        idx = std::move(ivrqfs);
     } else {
         FAISS_THROW_FMT(
                 "Index type 0x%08x (\"%s\") not recognized",
                 h,
                 fourcc_inv_printable(h).c_str());
-        idx = nullptr;
+        idx.reset();
     }
     return idx;
 }
 
-Index* read_index(FILE* f, int io_flags) {
+Index* read_index(IOReader* f, int io_flags) {
+    return read_index_up(f, io_flags).release();
+}
+
+std::unique_ptr<Index> read_index_up(FILE* f, int io_flags) {
     if ((io_flags & IO_FLAG_MMAP_IFC) == IO_FLAG_MMAP_IFC) {
         // enable mmap-supporting IOReader
         auto owner = std::make_shared<MmappedFileMappingOwner>(f);
         MappedFileIOReader reader(owner);
-        return read_index(&reader, io_flags);
+        return read_index_up(&reader, io_flags);
     } else {
         FileIOReader reader(f);
-        return read_index(&reader, io_flags);
+        return read_index_up(&reader, io_flags);
     }
 }
 
-Index* read_index(const char* fname, int io_flags) {
+Index* read_index(FILE* f, int io_flags) {
+    return read_index_up(f, io_flags).release();
+}
+
+std::unique_ptr<Index> read_index_up(const char* fname, int io_flags) {
     if ((io_flags & IO_FLAG_MMAP_IFC) == IO_FLAG_MMAP_IFC) {
         // enable mmap-supporting IOReader
         auto owner = std::make_shared<MmappedFileMappingOwner>(fname);
         MappedFileIOReader reader(owner);
-        return read_index(&reader, io_flags);
+        return read_index_up(&reader, io_flags);
     } else {
         FileIOReader reader(fname);
-        Index* idx = read_index(&reader, io_flags);
-        return idx;
+        return read_index_up(&reader, io_flags);
     }
 }
 
-VectorTransform* read_VectorTransform(const char* fname) {
+Index* read_index(const char* fname, int io_flags) {
+    return read_index_up(fname, io_flags).release();
+}
+
+std::unique_ptr<VectorTransform> read_VectorTransform_up(const char* fname) {
     FileIOReader reader(fname);
-    VectorTransform* vt = read_VectorTransform(&reader);
-    return vt;
+    return read_VectorTransform_up(&reader);
+}
+
+VectorTransform* read_VectorTransform(const char* fname) {
+    return read_VectorTransform_up(fname).release();
 }
 
 /*************************************************************
  * Read binary indexes
  **************************************************************/
 
-static void read_InvertedLists(IndexBinaryIVF* ivf, IOReader* f, int io_flags) {
-    InvertedLists* ils = read_InvertedLists(f, io_flags);
+static void read_InvertedLists(IndexBinaryIVF& ivf, IOReader* f, int io_flags) {
+    auto ils = read_InvertedLists_up(f, io_flags);
     FAISS_THROW_IF_NOT(
             !ils ||
-            (ils->nlist == ivf->nlist && ils->code_size == ivf->code_size));
-    ivf->invlists = ils;
-    ivf->own_invlists = true;
+            (ils->nlist == ivf.nlist && ils->code_size == ivf.code_size));
+    ivf.invlists = ils.release();
+    ivf.own_invlists = true;
 }
 
-static void read_index_binary_header(IndexBinary* idx, IOReader* f) {
-    READ1(idx->d);
-    READ1(idx->code_size);
-    READ1(idx->ntotal);
-    READ1(idx->is_trained);
-    READ1(idx->metric_type);
-    idx->verbose = false;
+static void read_index_binary_header(IndexBinary& idx, IOReader* f) {
+    READ1(idx.d);
+    READ1(idx.code_size);
+    READ1(idx.ntotal);
+    READ1(idx.is_trained);
+    int metric_type_int;
+    READ1(metric_type_int);
+    idx.metric_type = metric_type_from_int(metric_type_int);
+    FAISS_THROW_IF_NOT_FMT(
+            idx.d >= 0, "invalid binary index dimension %d", idx.d);
+    FAISS_THROW_IF_NOT_FMT(
+            idx.ntotal >= 0,
+            "invalid binary index ntotal %" PRId64,
+            (int64_t)idx.ntotal);
+    idx.verbose = false;
 }
 
 static void read_binary_ivf_header(
-        IndexBinaryIVF* ivf,
+        IndexBinaryIVF& ivf,
         IOReader* f,
         std::vector<std::vector<idx_t>>* ids = nullptr) {
     read_index_binary_header(ivf, f);
-    READ1(ivf->nlist);
-    READ1(ivf->nprobe);
-    ivf->quantizer = read_index_binary(f);
-    ivf->own_fields = true;
+    READ1(ivf.nlist);
+    READ1(ivf.nprobe);
+    ivf.quantizer = read_index_binary(f);
+    ivf.own_fields = true;
     if (ids) { // used in legacy "Iv" formats
-        ids->resize(ivf->nlist);
-        for (size_t i = 0; i < ivf->nlist; i++)
+        ids->resize(ivf.nlist);
+        for (size_t i = 0; i < ivf.nlist; i++)
             READVECTOR((*ids)[i]);
     }
-    read_direct_map(&ivf->direct_map, f);
+    read_direct_map(&ivf.direct_map, f);
 }
 
 static void read_binary_hash_invlists(
@@ -1557,8 +1886,20 @@ static void read_binary_hash_invlists(
     int il_nbit = 0;
     READ1(il_nbit);
     // buffer for bitstrings
-    std::vector<uint8_t> buf((b + il_nbit) * sz);
+    size_t bits_per_entry = (size_t)b + (size_t)il_nbit;
+    size_t total_bits =
+            mul_no_overflow(bits_per_entry, sz, "binary hash invlists");
+    size_t needed_bytes = (total_bits + 7) / 8;
+    std::vector<uint8_t> buf;
     READVECTOR(buf);
+    FAISS_THROW_IF_NOT_FMT(
+            buf.size() >= needed_bytes,
+            "binary hash invlists: buffer size %zd < needed %zd bytes "
+            "for %zd entries of %zd bits each",
+            buf.size(),
+            needed_bytes,
+            sz,
+            bits_per_entry);
     BitstringReader rd(buf.data(), buf.size());
     invlists.reserve(sz);
     for (size_t i = 0; i < sz; i++) {
@@ -1582,7 +1923,10 @@ static void read_binary_multi_hash_map(
     READ1(sz);
     std::vector<uint8_t> buf;
     READVECTOR(buf);
-    size_t nbit = (b + id_bits) * sz + ntotal * id_bits;
+    size_t nbit = add_no_overflow(
+            mul_no_overflow((size_t)(b + id_bits), sz, "multi hash map"),
+            mul_no_overflow(ntotal, (size_t)id_bits, "multi hash map"),
+            "multi hash map total bits");
     FAISS_THROW_IF_NOT(buf.size() == (nbit + 7) / 8);
     BitstringReader rd(buf.data(), buf.size());
     map.reserve(sz);
@@ -1596,72 +1940,74 @@ static void read_binary_multi_hash_map(
     }
 }
 
-IndexBinary* read_index_binary(IOReader* f, int io_flags) {
-    IndexBinary* idx = nullptr;
+std::unique_ptr<IndexBinary> read_index_binary_up(IOReader* f, int io_flags) {
+    std::unique_ptr<IndexBinary> idx;
     uint32_t h;
     READ1(h);
     if (h == fourcc("IBxF")) {
-        IndexBinaryFlat* idxf = new IndexBinaryFlat();
-        read_index_binary_header(idxf, f);
+        auto idxf = std::make_unique<IndexBinaryFlat>();
+        read_index_binary_header(*idxf, f);
         read_vector(idxf->xb, f);
         FAISS_THROW_IF_NOT(idxf->xb.size() == idxf->ntotal * idxf->code_size);
-        // leak!
-        idx = idxf;
+        idx = std::move(idxf);
     } else if (h == fourcc("IBwF")) {
-        IndexBinaryIVF* ivf = new IndexBinaryIVF();
-        read_binary_ivf_header(ivf, f);
-        read_InvertedLists(ivf, f, io_flags);
-        idx = ivf;
+        auto ivf = std::make_unique<IndexBinaryIVF>();
+        read_binary_ivf_header(*ivf, f);
+        read_InvertedLists(*ivf, f, io_flags);
+        idx = std::move(ivf);
     } else if (h == fourcc("IBFf")) {
-        IndexBinaryFromFloat* idxff = new IndexBinaryFromFloat();
-        read_index_binary_header(idxff, f);
+        auto idxff = std::make_unique<IndexBinaryFromFloat>();
+        read_index_binary_header(*idxff, f);
         idxff->own_fields = true;
         idxff->index = read_index(f, io_flags);
-        idx = idxff;
+        idx = std::move(idxff);
     } else if (h == fourcc("IBHf")) {
-        IndexBinaryHNSW* idxhnsw = new IndexBinaryHNSW();
-        read_index_binary_header(idxhnsw, f);
-        read_HNSW(&idxhnsw->hnsw, f);
+        auto idxhnsw = std::make_unique<IndexBinaryHNSW>();
+        read_index_binary_header(*idxhnsw, f);
+        read_HNSW(idxhnsw->hnsw, f);
         idxhnsw->hnsw.is_panorama = false;
         idxhnsw->storage = read_index_binary(f, io_flags);
         idxhnsw->own_fields = true;
-        idx = idxhnsw;
+        idx = std::move(idxhnsw);
     } else if (h == fourcc("IBHc")) {
-        IndexBinaryHNSWCagra* idxhnsw = new IndexBinaryHNSWCagra();
-        read_index_binary_header(idxhnsw, f);
+        auto idxhnsw = std::make_unique<IndexBinaryHNSWCagra>();
+        read_index_binary_header(*idxhnsw, f);
         READ1(idxhnsw->keep_max_size_level0);
         READ1(idxhnsw->base_level_only);
         READ1(idxhnsw->num_base_level_search_entrypoints);
-        read_HNSW(&idxhnsw->hnsw, f);
+        read_HNSW(idxhnsw->hnsw, f);
         idxhnsw->hnsw.is_panorama = false;
         idxhnsw->storage = read_index_binary(f, io_flags);
         idxhnsw->own_fields = true;
-        idx = idxhnsw;
+        idx = std::move(idxhnsw);
     } else if (h == fourcc("IBMp") || h == fourcc("IBM2")) {
         bool is_map2 = h == fourcc("IBM2");
-        IndexBinaryIDMap* idxmap =
-                is_map2 ? new IndexBinaryIDMap2() : new IndexBinaryIDMap();
-        read_index_binary_header(idxmap, f);
+        std::unique_ptr<IndexBinaryIDMap> idxmap = is_map2
+                ? std::make_unique<IndexBinaryIDMap2>()
+                : std::make_unique<IndexBinaryIDMap>();
+        read_index_binary_header(*idxmap, f);
         idxmap->index = read_index_binary(f, io_flags);
         idxmap->own_fields = true;
         READVECTOR(idxmap->id_map);
         if (is_map2) {
-            static_cast<IndexBinaryIDMap2*>(idxmap)->construct_rev_map();
+            static_cast<IndexBinaryIDMap2*>(idxmap.get())->construct_rev_map();
         }
-        idx = idxmap;
+        idx = std::move(idxmap);
     } else if (h == fourcc("IBHh")) {
-        IndexBinaryHash* idxh = new IndexBinaryHash();
-        read_index_binary_header(idxh, f);
+        auto idxh = std::make_unique<IndexBinaryHash>();
+        read_index_binary_header(*idxh, f);
         READ1(idxh->b);
         READ1(idxh->nflip);
         read_binary_hash_invlists(idxh->invlists, idxh->b, f);
-        idx = idxh;
+        idx = std::move(idxh);
     } else if (h == fourcc("IBHm")) {
-        IndexBinaryMultiHash* idxmh = new IndexBinaryMultiHash();
-        read_index_binary_header(idxmh, f);
-        idxmh->storage = dynamic_cast<IndexBinaryFlat*>(read_index_binary(f));
-        FAISS_THROW_IF_NOT(
-                idxmh->storage && idxmh->storage->ntotal == idxmh->ntotal);
+        auto idxmh = std::make_unique<IndexBinaryMultiHash>();
+        read_index_binary_header(*idxmh, f);
+        auto storage_idx = read_index_binary_up(f);
+        auto* flat_ptr = dynamic_cast<IndexBinaryFlat*>(storage_idx.get());
+        FAISS_THROW_IF_NOT(flat_ptr && flat_ptr->ntotal == idxmh->ntotal);
+        idxmh->storage = flat_ptr;
+        storage_idx.release();
         idxmh->own_fields = true;
         READ1(idxmh->b);
         READ1(idxmh->nhash);
@@ -1671,40 +2017,52 @@ IndexBinary* read_index_binary(IOReader* f, int io_flags) {
             read_binary_multi_hash_map(
                     idxmh->maps[i], idxmh->b, idxmh->ntotal, f);
         }
-        idx = idxmh;
+        idx = std::move(idxmh);
     } else {
         FAISS_THROW_FMT(
                 "Index type %08x (\"%s\") not recognized",
                 h,
                 fourcc_inv_printable(h).c_str());
-        idx = nullptr;
     }
     return idx;
 }
 
-IndexBinary* read_index_binary(FILE* f, int io_flags) {
+IndexBinary* read_index_binary(IOReader* f, int io_flags) {
+    return read_index_binary_up(f, io_flags).release();
+}
+
+std::unique_ptr<IndexBinary> read_index_binary_up(FILE* f, int io_flags) {
     if ((io_flags & IO_FLAG_MMAP_IFC) == IO_FLAG_MMAP_IFC) {
         // enable mmap-supporting IOReader
         auto owner = std::make_shared<MmappedFileMappingOwner>(f);
         MappedFileIOReader reader(owner);
-        return read_index_binary(&reader, io_flags);
+        return read_index_binary_up(&reader, io_flags);
     } else {
         FileIOReader reader(f);
-        return read_index_binary(&reader, io_flags);
+        return read_index_binary_up(&reader, io_flags);
     }
 }
 
-IndexBinary* read_index_binary(const char* fname, int io_flags) {
+IndexBinary* read_index_binary(FILE* f, int io_flags) {
+    return read_index_binary_up(f, io_flags).release();
+}
+
+std::unique_ptr<IndexBinary> read_index_binary_up(
+        const char* fname,
+        int io_flags) {
     if ((io_flags & IO_FLAG_MMAP_IFC) == IO_FLAG_MMAP_IFC) {
         // enable mmap-supporting IOReader
         auto owner = std::make_shared<MmappedFileMappingOwner>(fname);
         MappedFileIOReader reader(owner);
-        return read_index_binary(&reader, io_flags);
+        return read_index_binary_up(&reader, io_flags);
     } else {
         FileIOReader reader(fname);
-        IndexBinary* idx = read_index_binary(&reader, io_flags);
-        return idx;
+        return read_index_binary_up(&reader, io_flags);
     }
 }
 
+IndexBinary* read_index_binary(const char* fname, int io_flags) {
+    return read_index_binary_up(fname, io_flags).release();
+}
+
 } // namespace faiss