faiss 0.3.0 → 0.3.2

data/vendor/faiss/faiss/impl/index_read.cpp

@@ -5,8 +5,7 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-// -*- c++ -*-
-
+#include <faiss/impl/index_read_utils.h>
 #include <faiss/index_io.h>
 
 #include <faiss/impl/io_macros.h>
@@ -33,6 +32,7 @@
 #include <faiss/IndexIVFAdditiveQuantizer.h>
 #include <faiss/IndexIVFAdditiveQuantizerFastScan.h>
 #include <faiss/IndexIVFFlat.h>
+#include <faiss/IndexIVFIndependentQuantizer.h>
 #include <faiss/IndexIVFPQ.h>
 #include <faiss/IndexIVFPQFastScan.h>
 #include <faiss/IndexIVFPQR.h>
@@ -62,7 +62,7 @@ namespace faiss {
  * Read
  **************************************************************/
 
-static void read_index_header(Index* idx, IOReader* f) {
+void read_index_header(Index* idx, IOReader* f) {
     READ1(idx->d);
     READ1(idx->ntotal);
     idx_t dummy;
@@ -231,7 +231,7 @@ InvertedLists* read_InvertedLists(IOReader* f, int io_flags) {
     }
 }
 
-static void read_InvertedLists(IndexIVF* ivf, IOReader* f, int io_flags) {
+void read_InvertedLists(IndexIVF* ivf, IOReader* f, int io_flags) {
     InvertedLists* ils = read_InvertedLists(f, io_flags);
     if (ils) {
         FAISS_THROW_IF_NOT(ils->nlist == ivf->nlist);
@@ -243,7 +243,7 @@ static void read_InvertedLists(IndexIVF* ivf, IOReader* f, int io_flags) {
     ivf->own_invlists = true;
 }
 
-static void read_ProductQuantizer(ProductQuantizer* pq, IOReader* f) {
+void read_ProductQuantizer(ProductQuantizer* pq, IOReader* f) {
     READ1(pq->d);
     READ1(pq->M);
     READ1(pq->nbits);
@@ -291,11 +291,17 @@ static void read_AdditiveQuantizer(AdditiveQuantizer* aq, IOReader* f) {
     aq->set_derived_values();
 }
 
-static void read_ResidualQuantizer(ResidualQuantizer* rq, IOReader* f) {
+static void read_ResidualQuantizer(
+        ResidualQuantizer* rq,
+        IOReader* f,
+        int io_flags) {
     read_AdditiveQuantizer(rq, f);
     READ1(rq->train_type);
     READ1(rq->max_beam_size);
-    if (!(rq->train_type & ResidualQuantizer::Skip_codebook_tables)) {
+    if ((rq->train_type & ResidualQuantizer::Skip_codebook_tables) ||
+        (io_flags & IO_FLAG_SKIP_PRECOMPUTE_TABLE)) {
+        // don't precompute the tables
+    } else {
         rq->compute_codebook_tables();
     }
 }
@@ -324,12 +330,13 @@ static void read_ProductAdditiveQuantizer(
 
 static void read_ProductResidualQuantizer(
         ProductResidualQuantizer* prq,
-        IOReader* f) {
+        IOReader* f,
+        int io_flags) {
     read_ProductAdditiveQuantizer(prq, f);
 
     for (size_t i = 0; i < prq->nsplits; i++) {
         auto rq = new ResidualQuantizer();
-        read_ResidualQuantizer(rq, f);
+        read_ResidualQuantizer(rq, f, io_flags);
         prq->quantizers.push_back(rq);
     }
 }
@@ -346,7 +353,7 @@ static void read_ProductLocalSearchQuantizer(
     }
 }
 
-static void read_ScalarQuantizer(ScalarQuantizer* ivsc, IOReader* f) {
+void read_ScalarQuantizer(ScalarQuantizer* ivsc, IOReader* f) {
     READ1(ivsc->qtype);
     READ1(ivsc->rangestat);
     READ1(ivsc->rangestat_arg);
@@ -367,7 +374,10 @@ static void read_HNSW(HNSW* hnsw, IOReader* f) {
     READ1(hnsw->max_level);
     READ1(hnsw->efConstruction);
     READ1(hnsw->efSearch);
-    READ1(hnsw->upper_beam);
+
+    // // deprecated field
+    // READ1(hnsw->upper_beam);
+    READ1_DUMMY(int)
 }
 
 static void read_NSG(NSG* nsg, IOReader* f) {
@@ -390,15 +400,12 @@ static void read_NSG(NSG* nsg, IOReader* f) {
     graph = std::make_shared<nsg::Graph<int>>(N, R);
     std::fill_n(graph->data, N * R, EMPTY_ID);
 
-    int size = 0;
-
     for (int i = 0; i < N; i++) {
         for (int j = 0; j < R + 1; j++) {
             int id;
             READ1(id);
             if (id != EMPTY_ID) {
                 graph->at(i, j) = id;
-                size += 1;
             } else {
                 break;
             }
@@ -428,14 +435,14 @@ ProductQuantizer* read_ProductQuantizer(const char* fname) {
 
 ProductQuantizer* read_ProductQuantizer(IOReader* reader) {
     ProductQuantizer* pq = new ProductQuantizer();
-    ScopeDeleter1<ProductQuantizer> del(pq);
+    std::unique_ptr<ProductQuantizer> del(pq);
 
     read_ProductQuantizer(pq, reader);
     del.release();
     return pq;
 }
 
-static void read_direct_map(DirectMap* dm, IOReader* f) {
+void read_direct_map(DirectMap* dm, IOReader* f) {
     char maintain_direct_map;
     READ1(maintain_direct_map);
     dm->type = (DirectMap::Type)maintain_direct_map;
@@ -451,10 +458,10 @@ static void read_direct_map(DirectMap* dm, IOReader* f) {
     }
 }
 
-static void read_ivf_header(
+void read_ivf_header(
         IndexIVF* ivf,
         IOReader* f,
-        std::vector<std::vector<idx_t>>* ids = nullptr) {
+        std::vector<std::vector<idx_t>>* ids) {
     read_index_header(ivf, f);
     READ1(ivf->nlist);
     READ1(ivf->nprobe);
@@ -469,7 +476,7 @@ static void read_ivf_header(
 }
 
 // used for legacy formats
-static ArrayInvertedLists* set_array_invlist(
+ArrayInvertedLists* set_array_invlist(
         IndexIVF* ivf,
         std::vector<std::vector<idx_t>>& ids) {
     ArrayInvertedLists* ail =
@@ -526,7 +533,11 @@ Index* read_index(IOReader* f, int io_flags) {
     Index* idx = nullptr;
     uint32_t h;
     READ1(h);
-    if (h == fourcc("IxFI") || h == fourcc("IxF2") || h == fourcc("IxFl")) {
+    if (h == fourcc("null")) {
+        // denotes a missing index, useful for some cases
+        return nullptr;
+    } else if (
+            h == fourcc("IxFI") || h == fourcc("IxF2") || h == fourcc("IxFl")) {
         IndexFlat* idxf;
         if (h == fourcc("IxFI")) {
             idxf = new IndexFlatIP();
@@ -587,7 +598,7 @@ Index* read_index(IOReader* f, int io_flags) {
             READ1(idxp->encode_signs);
             READ1(idxp->polysemous_ht);
         }
-        // Old versoins of PQ all had metric_type set to INNER_PRODUCT
+        // Old versions of PQ all had metric_type set to INNER_PRODUCT
         // when they were in fact using L2. Therefore, we force metric type
         // to L2 when the old format is detected
         if (h == fourcc("IxPQ") || h == fourcc("IxPo")) {
@@ -600,7 +611,7 @@ Index* read_index(IOReader* f, int io_flags) {
         if (h == fourcc("IxRQ")) {
             read_ResidualQuantizer_old(&idxr->rq, f);
         } else {
-            read_ResidualQuantizer(&idxr->rq, f);
+            read_ResidualQuantizer(&idxr->rq, f, io_flags);
         }
         READ1(idxr->code_size);
         READVECTOR(idxr->codes);
@@ -615,7 +626,7 @@ Index* read_index(IOReader* f, int io_flags) {
     } else if (h == fourcc("IxPR")) {
         auto idxpr = new IndexProductResidualQuantizer();
         read_index_header(idxpr, f);
-        read_ProductResidualQuantizer(&idxpr->prq, f);
+        read_ProductResidualQuantizer(&idxpr->prq, f, io_flags);
         READ1(idxpr->code_size);
         READVECTOR(idxpr->codes);
         idx = idxpr;
@@ -629,8 +640,13 @@ Index* read_index(IOReader* f, int io_flags) {
     } else if (h == fourcc("ImRQ")) {
         ResidualCoarseQuantizer* idxr = new ResidualCoarseQuantizer();
         read_index_header(idxr, f);
-        read_ResidualQuantizer(&idxr->rq, f);
+        read_ResidualQuantizer(&idxr->rq, f, io_flags);
         READ1(idxr->beam_factor);
+        if (io_flags & IO_FLAG_SKIP_PRECOMPUTE_TABLE) {
+            // then we force the beam factor to -1
+            // which skips the table precomputation.
+            idxr->beam_factor = -1;
+        }
         idxr->set_beam_factor(idxr->beam_factor);
         idx = idxr;
     } else if (
@@ -655,13 +671,14 @@ Index* read_index(IOReader* f, int io_flags) {
         if (is_LSQ) {
             read_LocalSearchQuantizer((LocalSearchQuantizer*)idxaqfs->aq, f);
         } else if (is_RQ) {
-            read_ResidualQuantizer((ResidualQuantizer*)idxaqfs->aq, f);
+            read_ResidualQuantizer(
+                    (ResidualQuantizer*)idxaqfs->aq, f, io_flags);
         } else if (is_PLSQ) {
             read_ProductLocalSearchQuantizer(
                     (ProductLocalSearchQuantizer*)idxaqfs->aq, f);
         } else {
             read_ProductResidualQuantizer(
-                    (ProductResidualQuantizer*)idxaqfs->aq, f);
+                    (ProductResidualQuantizer*)idxaqfs->aq, f, io_flags);
         }
 
         READ1(idxaqfs->implem);
@@ -703,13 +720,13 @@ Index* read_index(IOReader* f, int io_flags) {
         if (is_LSQ) {
             read_LocalSearchQuantizer((LocalSearchQuantizer*)ivaqfs->aq, f);
         } else if (is_RQ) {
-            read_ResidualQuantizer((ResidualQuantizer*)ivaqfs->aq, f);
+            read_ResidualQuantizer((ResidualQuantizer*)ivaqfs->aq, f, io_flags);
         } else if (is_PLSQ) {
             read_ProductLocalSearchQuantizer(
                     (ProductLocalSearchQuantizer*)ivaqfs->aq, f);
         } else {
             read_ProductResidualQuantizer(
-                    (ProductResidualQuantizer*)ivaqfs->aq, f);
+                    (ProductResidualQuantizer*)ivaqfs->aq, f, io_flags);
         }
 
         READ1(ivaqfs->by_residual);
@@ -831,13 +848,13 @@ Index* read_index(IOReader* f, int io_flags) {
         if (is_LSQ) {
             read_LocalSearchQuantizer((LocalSearchQuantizer*)iva->aq, f);
         } else if (is_RQ) {
-            read_ResidualQuantizer((ResidualQuantizer*)iva->aq, f);
+            read_ResidualQuantizer((ResidualQuantizer*)iva->aq, f, io_flags);
         } else if (is_PLSQ) {
             read_ProductLocalSearchQuantizer(
                     (ProductLocalSearchQuantizer*)iva->aq, f);
         } else {
             read_ProductResidualQuantizer(
-                    (ProductResidualQuantizer*)iva->aq, f);
+                    (ProductResidualQuantizer*)iva->aq, f, io_flags);
         }
         READ1(iva->by_residual);
         READ1(iva->use_precomputed_table);
@@ -860,7 +877,22 @@ Index* read_index(IOReader* f, int io_flags) {
             h == fourcc("IvPQ") || h == fourcc("IvQR") || h == fourcc("IwPQ") ||
             h == fourcc("IwQR")) {
         idx = read_ivfpq(f, h, io_flags);
-
+    } else if (h == fourcc("IwIQ")) {
+        auto* indep = new IndexIVFIndependentQuantizer();
+        indep->own_fields = true;
+        read_index_header(indep, f);
+        indep->quantizer = read_index(f, io_flags);
+        bool has_vt;
+        READ1(has_vt);
+        if (has_vt) {
+            indep->vt = read_VectorTransform(f);
+        }
+        indep->index_ivf = dynamic_cast<IndexIVF*>(read_index(f, io_flags));
+        FAISS_THROW_IF_NOT(indep->index_ivf);
+        if (auto index_ivfpq = dynamic_cast<IndexIVFPQ*>(indep->index_ivf)) {
+            READ1(index_ivfpq->use_precomputed_table);
+        }
+        idx = indep;
     } else if (h == fourcc("IxPT")) {
         IndexPreTransform* ixpt = new IndexPreTransform();
         ixpt->own_fields = true;
@@ -922,7 +954,7 @@ Index* read_index(IOReader* f, int io_flags) {
         idx = idxp;
     } else if (
             h == fourcc("IHNf") || h == fourcc("IHNp") || h == fourcc("IHNs") ||
-            h == fourcc("IHN2")) {
+            h == fourcc("IHN2") || h == fourcc("IHNc")) {
         IndexHNSW* idxhnsw = nullptr;
         if (h == fourcc("IHNf"))
             idxhnsw = new IndexHNSWFlat();
@@ -932,11 +964,19 @@ Index* read_index(IOReader* f, int io_flags) {
             idxhnsw = new IndexHNSWSQ();
         if (h == fourcc("IHN2"))
             idxhnsw = new IndexHNSW2Level();
+        if (h == fourcc("IHNc"))
+            idxhnsw = new IndexHNSWCagra();
         read_index_header(idxhnsw, f);
+        if (h == fourcc("IHNc")) {
+            READ1(idxhnsw->keep_max_size_level0);
+            auto idx_hnsw_cagra = dynamic_cast<IndexHNSWCagra*>(idxhnsw);
+            READ1(idx_hnsw_cagra->base_level_only);
+            READ1(idx_hnsw_cagra->num_base_level_search_entrypoints);
+        }
         read_HNSW(&idxhnsw->hnsw, f);
         idxhnsw->storage = read_index(f, io_flags);
-        idxhnsw->own_fields = true;
-        if (h == fourcc("IHNp")) {
+        idxhnsw->own_fields = idxhnsw->storage != nullptr;
+        if (h == fourcc("IHNp") && !(io_flags & IO_FLAG_PQ_SKIP_SDC_TABLE)) {
             dynamic_cast<IndexPQ*>(idxhnsw->storage)->pq.compute_sdc_table();
         }
         idx = idxhnsw;

data/vendor/faiss/faiss/impl/index_read_utils.h

@@ -0,0 +1,37 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Utils for index_read
+
+#ifndef FAISS_INDEX_READ_UTILS_H
+#define FAISS_INDEX_READ_UTILS_H
+
+#include <faiss/IndexIVF.h>
+#include <faiss/impl/io.h>
+
+#pragma once
+
+namespace faiss {
+struct ProductQuantizer;
+struct ScalarQuantizer;
+
+void read_index_header(Index* idx, IOReader* f);
+void read_direct_map(DirectMap* dm, IOReader* f);
+void read_ivf_header(
+        IndexIVF* ivf,
+        IOReader* f,
+        std::vector<std::vector<idx_t>>* ids = nullptr);
+void read_InvertedLists(IndexIVF* ivf, IOReader* f, int io_flags);
+ArrayInvertedLists* set_array_invlist(
+        IndexIVF* ivf,
+        std::vector<std::vector<idx_t>>& ids);
+void read_ProductQuantizer(ProductQuantizer* pq, IOReader* f);
+void read_ScalarQuantizer(ScalarQuantizer* ivsc, IOReader* f);
+
+} // namespace faiss
+
+#endif

data/vendor/faiss/faiss/impl/index_write.cpp

@@ -5,8 +5,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-// -*- c++ -*-
-
 #include <faiss/index_io.h>
 
 #include <faiss/impl/io.h>
@@ -34,6 +32,7 @@
 #include <faiss/IndexIVFAdditiveQuantizer.h>
 #include <faiss/IndexIVFAdditiveQuantizerFastScan.h>
 #include <faiss/IndexIVFFlat.h>
+#include <faiss/IndexIVFIndependentQuantizer.h>
 #include <faiss/IndexIVFPQ.h>
 #include <faiss/IndexIVFPQFastScan.h>
 #include <faiss/IndexIVFPQR.h>
@@ -313,7 +312,11 @@ static void write_HNSW(const HNSW* hnsw, IOWriter* f) {
     WRITE1(hnsw->max_level);
     WRITE1(hnsw->efConstruction);
     WRITE1(hnsw->efSearch);
-    WRITE1(hnsw->upper_beam);
+
+    // // deprecated field
+    // WRITE1(hnsw->upper_beam);
+    constexpr int tmp_upper_beam = 1;
+    WRITE1(tmp_upper_beam);
 }
 
 static void write_NSG(const NSG* nsg, IOWriter* f) {
@@ -337,13 +340,11 @@ static void write_NSG(const NSG* nsg, IOWriter* f) {
     FAISS_THROW_IF_NOT(K == nsg->R);
     FAISS_THROW_IF_NOT(true == graph->own_fields);
 
-    int size = 0;
     for (int i = 0; i < N; i++) {
         for (int j = 0; j < K; j++) {
             int id = graph->at(i, j);
             if (id != EMPTY_ID) {
                 WRITE1(id);
-                size += 1;
             } else {
                 break;
             }
@@ -385,12 +386,18 @@ static void write_ivf_header(const IndexIVF* ivf, IOWriter* f) {
     write_index_header(ivf, f);
     WRITE1(ivf->nlist);
     WRITE1(ivf->nprobe);
+    // subclasses write by_residual (some of them support only one setting of
+    // by_residual).
     write_index(ivf->quantizer, f);
     write_direct_map(&ivf->direct_map, f);
 }
 
-void write_index(const Index* idx, IOWriter* f) {
-    if (const IndexFlat* idxf = dynamic_cast<const IndexFlat*>(idx)) {
+void write_index(const Index* idx, IOWriter* f, int io_flags) {
+    if (idx == nullptr) {
+        // eg. for a storage component of HNSW that is set to nullptr
+        uint32_t h = fourcc("null");
+        WRITE1(h);
+    } else if (const IndexFlat* idxf = dynamic_cast<const IndexFlat*>(idx)) {
         uint32_t h =
                 fourcc(idxf->metric_type == METRIC_INNER_PRODUCT ? "IxFI"
                                : idxf->metric_type == METRIC_L2  ? "IxF2"
@@ -430,13 +437,14 @@ void write_index(const Index* idx, IOWriter* f) {
         WRITE1(idxr->code_size);
         WRITEVECTOR(idxr->codes);
     } else if (
-            auto* idxr = dynamic_cast<const IndexLocalSearchQuantizer*>(idx)) {
+            auto* idxr_2 =
+                    dynamic_cast<const IndexLocalSearchQuantizer*>(idx)) {
         uint32_t h = fourcc("IxLS");
         WRITE1(h);
         write_index_header(idx, f);
-        write_LocalSearchQuantizer(&idxr->lsq, f);
-        WRITE1(idxr->code_size);
-        WRITEVECTOR(idxr->codes);
+        write_LocalSearchQuantizer(&idxr_2->lsq, f);
+        WRITE1(idxr_2->code_size);
+        WRITEVECTOR(idxr_2->codes);
     } else if (
             const IndexProductResidualQuantizer* idxpr =
                     dynamic_cast<const IndexProductResidualQuantizer*>(idx)) {
@@ -569,26 +577,26 @@ void write_index(const Index* idx, IOWriter* f) {
 
         write_InvertedLists(ivaqfs->invlists, f);
     } else if (
-            const ResidualCoarseQuantizer* idxr =
+            const ResidualCoarseQuantizer* idxr_2 =
                     dynamic_cast<const ResidualCoarseQuantizer*>(idx)) {
         uint32_t h = fourcc("ImRQ");
         WRITE1(h);
         write_index_header(idx, f);
-        write_ResidualQuantizer(&idxr->rq, f);
-        WRITE1(idxr->beam_factor);
+        write_ResidualQuantizer(&idxr_2->rq, f);
+        WRITE1(idxr_2->beam_factor);
     } else if (
-            const Index2Layer* idxp = dynamic_cast<const Index2Layer*>(idx)) {
+            const Index2Layer* idxp_2 = dynamic_cast<const Index2Layer*>(idx)) {
         uint32_t h = fourcc("Ix2L");
         WRITE1(h);
         write_index_header(idx, f);
-        write_index(idxp->q1.quantizer, f);
-        WRITE1(idxp->q1.nlist);
-        WRITE1(idxp->q1.quantizer_trains_alone);
-        write_ProductQuantizer(&idxp->pq, f);
-        WRITE1(idxp->code_size_1);
-        WRITE1(idxp->code_size_2);
-        WRITE1(idxp->code_size);
-        WRITEVECTOR(idxp->codes);
+        write_index(idxp_2->q1.quantizer, f);
+        WRITE1(idxp_2->q1.nlist);
+        WRITE1(idxp_2->q1.quantizer_trains_alone);
+        write_ProductQuantizer(&idxp_2->pq, f);
+        WRITE1(idxp_2->code_size_1);
+        WRITE1(idxp_2->code_size_2);
+        WRITE1(idxp_2->code_size);
+        WRITEVECTOR(idxp_2->codes);
     } else if (
             const IndexScalarQuantizer* idxs =
                     dynamic_cast<const IndexScalarQuantizer*>(idx)) {
@@ -598,15 +606,16 @@ void write_index(const Index* idx, IOWriter* f) {
         write_ScalarQuantizer(&idxs->sq, f);
         WRITEVECTOR(idxs->codes);
     } else if (
-            const IndexLattice* idxl = dynamic_cast<const IndexLattice*>(idx)) {
+            const IndexLattice* idxl_2 =
+                    dynamic_cast<const IndexLattice*>(idx)) {
         uint32_t h = fourcc("IxLa");
         WRITE1(h);
-        WRITE1(idxl->d);
-        WRITE1(idxl->nsq);
-        WRITE1(idxl->scale_nbit);
-        WRITE1(idxl->zn_sphere_codec.r2);
+        WRITE1(idxl_2->d);
+        WRITE1(idxl_2->nsq);
+        WRITE1(idxl_2->scale_nbit);
+        WRITE1(idxl_2->zn_sphere_codec.r2);
         write_index_header(idx, f);
-        WRITEVECTOR(idxl->trained);
+        WRITEVECTOR(idxl_2->trained);
     } else if (
             const IndexIVFFlatDedup* ivfl =
                     dynamic_cast<const IndexIVFFlatDedup*>(idx)) {
@@ -625,11 +634,12 @@ void write_index(const Index* idx, IOWriter* f) {
         }
         write_InvertedLists(ivfl->invlists, f);
     } else if (
-            const IndexIVFFlat* ivfl = dynamic_cast<const IndexIVFFlat*>(idx)) {
+            const IndexIVFFlat* ivfl_2 =
+                    dynamic_cast<const IndexIVFFlat*>(idx)) {
         uint32_t h = fourcc("IwFl");
         WRITE1(h);
-        write_ivf_header(ivfl, f);
-        write_InvertedLists(ivfl->invlists, f);
+        write_ivf_header(ivfl_2, f);
+        write_InvertedLists(ivfl_2->invlists, f);
     } else if (
             const IndexIVFScalarQuantizer* ivsc =
                     dynamic_cast<const IndexIVFScalarQuantizer*>(idx)) {
@@ -700,7 +710,22 @@ void write_index(const Index* idx, IOWriter* f) {
             WRITEVECTOR(ivfpqr->refine_codes);
             WRITE1(ivfpqr->k_factor);
         }
-
+    } else if (
+            auto* indep =
+                    dynamic_cast<const IndexIVFIndependentQuantizer*>(idx)) {
+        uint32_t h = fourcc("IwIQ");
+        WRITE1(h);
+        write_index_header(indep, f);
+        write_index(indep->quantizer, f);
+        bool has_vt = indep->vt != nullptr;
+        WRITE1(has_vt);
+        if (has_vt) {
+            write_VectorTransform(indep->vt, f);
+        }
+        write_index(indep->index_ivf, f);
+        if (auto index_ivfpq = dynamic_cast<IndexIVFPQ*>(indep->index_ivf)) {
+            WRITE1(index_ivfpq->use_precomputed_table);
+        }
     } else if (
             const IndexPreTransform* ixpt =
                     dynamic_cast<const IndexPreTransform*>(idx)) {
@@ -741,12 +766,24 @@ void write_index(const Index* idx, IOWriter* f) {
                 : dynamic_cast<const IndexHNSWPQ*>(idx)      ? fourcc("IHNp")
                 : dynamic_cast<const IndexHNSWSQ*>(idx)      ? fourcc("IHNs")
                 : dynamic_cast<const IndexHNSW2Level*>(idx)  ? fourcc("IHN2")
+                : dynamic_cast<const IndexHNSWCagra*>(idx)   ? fourcc("IHNc")
                                                              : 0;
         FAISS_THROW_IF_NOT(h != 0);
         WRITE1(h);
         write_index_header(idxhnsw, f);
+        if (h == fourcc("IHNc")) {
+            WRITE1(idxhnsw->keep_max_size_level0);
+            auto idx_hnsw_cagra = dynamic_cast<const IndexHNSWCagra*>(idxhnsw);
+            WRITE1(idx_hnsw_cagra->base_level_only);
+            WRITE1(idx_hnsw_cagra->num_base_level_search_entrypoints);
+        }
         write_HNSW(&idxhnsw->hnsw, f);
-        write_index(idxhnsw->storage, f);
+        if (io_flags & IO_FLAG_SKIP_STORAGE) {
+            uint32_t n4 = fourcc("null");
+            WRITE1(n4);
+        } else {
+            write_index(idxhnsw->storage, f);
+        }
     } else if (const IndexNSG* idxnsg = dynamic_cast<const IndexNSG*>(idx)) {
         uint32_t h = dynamic_cast<const IndexNSGFlat*>(idx) ? fourcc("INSf")
                 : dynamic_cast<const IndexNSGPQ*>(idx)      ? fourcc("INSp")
@@ -788,19 +825,19 @@ void write_index(const Index* idx, IOWriter* f) {
         WRITE1(idxpqfs->M2);
         WRITEVECTOR(idxpqfs->codes);
     } else if (
-            const IndexIVFPQFastScan* ivpq =
+            const IndexIVFPQFastScan* ivpq_2 =
                     dynamic_cast<const IndexIVFPQFastScan*>(idx)) {
         uint32_t h = fourcc("IwPf");
         WRITE1(h);
-        write_ivf_header(ivpq, f);
-        WRITE1(ivpq->by_residual);
-        WRITE1(ivpq->code_size);
-        WRITE1(ivpq->bbs);
-        WRITE1(ivpq->M2);
-        WRITE1(ivpq->implem);
-        WRITE1(ivpq->qbs2);
-        write_ProductQuantizer(&ivpq->pq, f);
-        write_InvertedLists(ivpq->invlists, f);
+        write_ivf_header(ivpq_2, f);
+        WRITE1(ivpq_2->by_residual);
+        WRITE1(ivpq_2->code_size);
+        WRITE1(ivpq_2->bbs);
+        WRITE1(ivpq_2->M2);
+        WRITE1(ivpq_2->implem);
+        WRITE1(ivpq_2->qbs2);
+        write_ProductQuantizer(&ivpq_2->pq, f);
+        write_InvertedLists(ivpq_2->invlists, f);
     } else if (
             const IndexRowwiseMinMax* imm =
                     dynamic_cast<const IndexRowwiseMinMax*>(idx)) {
@@ -810,26 +847,26 @@ void write_index(const Index* idx, IOWriter* f) {
         write_index_header(imm, f);
         write_index(imm->index, f);
     } else if (
-            const IndexRowwiseMinMaxFP16* imm =
+            const IndexRowwiseMinMaxFP16* imm_2 =
                     dynamic_cast<const IndexRowwiseMinMaxFP16*>(idx)) {
         // IndexRowwiseMinmaxHalf
         uint32_t h = fourcc("IRMh");
         WRITE1(h);
-        write_index_header(imm, f);
-        write_index(imm->index, f);
+        write_index_header(imm_2, f);
+        write_index(imm_2->index, f);
     } else {
         FAISS_THROW_MSG("don't know how to serialize this type of index");
     }
 }
 
-void write_index(const Index* idx, FILE* f) {
+void write_index(const Index* idx, FILE* f, int io_flags) {
     FileIOWriter writer(f);
-    write_index(idx, &writer);
+    write_index(idx, &writer, io_flags);
 }
 
-void write_index(const Index* idx, const char* fname) {
+void write_index(const Index* idx, const char* fname, int io_flags) {
     FileIOWriter writer(fname);
-    write_index(idx, &writer);
+    write_index(idx, &writer, io_flags);
 }
 
 void write_VectorTransform(const VectorTransform* vt, const char* fname) {