faiss 0.4.3 → 0.5.1

data/vendor/faiss/faiss/impl/pq4_fast_scan_search_qbs.cpp

@@ -53,8 +53,8 @@ void kernel_accumulate_block(
 
     // _mm_prefetch(codes + 768, 0);
     for (int sq = 0; sq < nsq - scaler.nscale; sq += 2) {
-        // prefetch
-        simd32uint8 c(codes);
+        simd32uint8 c;
+        c.loadu(codes);
         codes += 32;
 
         simd32uint8 mask(0xf);
@@ -79,8 +79,8 @@ void kernel_accumulate_block(
     }
 
     for (int sq = 0; sq < scaler.nscale; sq += 2) {
-        // prefetch
-        simd32uint8 c(codes);
+        simd32uint8 c;
+        c.loadu(codes);
         codes += 32;
 
         simd32uint8 mask(0xf);
@@ -623,7 +623,6 @@ void accumulate(
         ResultHandler& res,
         const Scaler& scaler) {
     assert(nsq % 2 == 0);
-    assert(is_aligned_pointer(codes));
     assert(is_aligned_pointer(LUT));
 
 #define DISPATCH(NQ)                                     \
@@ -782,7 +781,7 @@ void accumulate_to_mem(
 }
 
 int pq4_preferred_qbs(int n) {
-    // from timmings in P141901742, P141902828
+    // from timings in P141901742, P141902828
     static int map[12] = {
             0, 1, 2, 3, 0x13, 0x23, 0x33, 0x223, 0x233, 0x333, 0x2233, 0x2333};
     if (n <= 11) {

data/vendor/faiss/faiss/impl/simd_result_handlers.h

@@ -46,7 +46,7 @@ struct SIMDResultHandler {
     virtual ~SIMDResultHandler() {}
 };
 
-/* Result handler that will return float resutls eventually */
+/* Result handler that will return float results eventually */
 struct SIMDResultHandlerToFloat : SIMDResultHandler {
     size_t nq;     // number of queries
     size_t ntotal; // ignore excess elements after ntotal
@@ -70,13 +70,35 @@ struct SIMDResultHandlerToFloat : SIMDResultHandler {
     virtual void end() {
         normalizers = nullptr;
     }
+
+    // Number of updates made to the underlying data structure.
+    // For example: number of heap updates.
+    virtual size_t num_updates() {
+        return 0;
+    }
+
+    /** Set context information for handlers that need additional data
+     *
+     * This method can be overridden by handlers that need list numbers
+     * and probe mappings (e.g., RaBitQ handlers). Base implementation
+     * does nothing since most handlers don't need this context.
+     *
+     * @param list_no      current inverted list number being processed
+     * @param probe_map    mapping from local query index to probe index
+     */
+    virtual void set_list_context(
+            size_t /* list_no */,
+            const std::vector<int>& /* probe_map */) {
+        // Default implementation does nothing
+        // Derived handlers can override if they need this context
+    }
 };
 
 FAISS_API extern bool simd_result_handlers_accept_virtual;
 
 namespace simd_result_handlers {
 
-/** Dummy structure that just computes a chqecksum on results
+/** Dummy structure that just computes a checksum on results
  * (to avoid the computation to be optimized away) */
 struct DummyResultHandler : SIMDResultHandler {
     size_t cs = 0;
@@ -318,8 +340,8 @@ struct HeapHandler : ResultHandlerCompare<C, with_id_map> {
     std::vector<TI> iids;
     float* dis;
     int64_t* ids;
-
-    int64_t k; // number of results to keep
+    size_t k;       // number of results to keep
+    size_t nup = 0; // number of heap updates
 
     HeapHandler(
             size_t nq,
@@ -327,14 +349,30 @@ struct HeapHandler : ResultHandlerCompare<C, with_id_map> {
             int64_t k,
             float* dis,
             int64_t* ids,
-            const IDSelector* sel_in)
+            const IDSelector* sel_in,
+            const float* normalizers = nullptr)
             : RHC(nq, ntotal, sel_in),
-              idis(nq * k),
-              iids(nq * k),
+              idis(nq * k, threshold_idis(dis, normalizers)),
+              iids(nq * k, -1),
               dis(dis),
               ids(ids),
-              k(k) {
-        heap_heapify<C>(k * nq, idis.data(), iids.data());
+              k(k) {}
+
+    static uint16_t threshold_idis(float* dis_in, const float* normalizers) {
+        if (dis_in[0] == std::numeric_limits<float>::max()) {
+            return std::numeric_limits<uint16_t>::max();
+        }
+        if (dis_in[0] == std::numeric_limits<float>::lowest()) {
+            return 0;
+        }
+        if (normalizers) {
+            float one_a = 1 / normalizers[0], b = normalizers[1];
+            float f = (dis_in[0] - b) / one_a;
+            f = C::is_max ? std::ceil(f) : std::floor(f);
+            return std::clamp<float>(
+                    f, 0, std::numeric_limits<uint16_t>::max());
+        }
+        return C::neutral();
     }
 
     void handle(size_t q, size_t b, simd16uint16 d0, simd16uint16 d1) final {
@@ -372,6 +410,7 @@ struct HeapHandler : ResultHandlerCompare<C, with_id_map> {
                     if (C::cmp(heap_dis[0], dis_2)) {
                         heap_replace_top<C>(
                                 k, heap_dis, heap_ids, dis_2, real_idx);
+                        nup++;
                     }
                 }
             }
@@ -384,6 +423,7 @@ struct HeapHandler : ResultHandlerCompare<C, with_id_map> {
                 if (C::cmp(heap_dis[0], dis_2)) {
                     int64_t idx = this->adjust_id(b, j);
                     heap_replace_top<C>(k, heap_dis, heap_ids, dis_2, idx);
+                    nup++;
                 }
             }
         }
@@ -408,6 +448,10 @@ struct HeapHandler : ResultHandlerCompare<C, with_id_map> {
             }
         }
     }
+
+    size_t num_updates() override {
+        return nup;
+    }
 };
 
 /** Simple top-N implementation using a reservoir.
@@ -726,7 +770,7 @@ void dispatch_SIMDResultHandler_fixedCW(
     } else { // generic handler -- will not be inlined
         FAISS_THROW_IF_NOT_FMT(
                 simd_result_handlers_accept_virtual,
-                "Running vitrual handler for %s",
+                "Running virtual handler for %s",
                 typeid(res).name());
         consumer.template f<SIMDResultHandler>(res, args...);
     }
@@ -757,7 +801,7 @@ void dispatch_SIMDResultHandler(
         } else { // generic path
             FAISS_THROW_IF_NOT_FMT(
                     simd_result_handlers_accept_virtual,
-                    "Running vitrual handler for %s",
+                    "Running virtual handler for %s",
                     typeid(res).name());
             consumer.template f<SIMDResultHandler>(res, args...);
         }

data/vendor/faiss/faiss/impl/svs_io.cpp

@@ -0,0 +1,86 @@
+/*
+ * Portions Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/*
+ * Portions Copyright 2025 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <faiss/impl/svs_io.h>
+
+#include <algorithm>
+#include <cstdlib>
+
+#include <faiss/impl/FaissAssert.h>
+
+namespace faiss {
+namespace svs_io {
+
+WriterStreambuf::WriterStreambuf(IOWriter* w_) : w(w_) {}
+
+WriterStreambuf::~WriterStreambuf() = default;
+
+std::streamsize WriterStreambuf::xsputn(const char* s, std::streamsize n) {
+    if (n <= 0)
+        return 0;
+    size_t wrote = (*w)(s, 1, static_cast<size_t>(n));
+    return static_cast<std::streamsize>(wrote);
+}
+
+int WriterStreambuf::overflow(int ch) {
+    if (ch == traits_type::eof())
+        return 0;
+    char c = static_cast<char>(ch);
+    size_t wrote = (*w)(&c, 1, 1);
+    return wrote == 1 ? ch : traits_type::eof();
+}
+
+ReaderStreambuf::ReaderStreambuf(IOReader* rr) : r(rr), single_char_buffer(0) {
+    // Initialize with empty get area
+    setg(nullptr, nullptr, nullptr);
+}
+
+ReaderStreambuf::~ReaderStreambuf() = default;
+
+std::streambuf::int_type ReaderStreambuf::underflow() {
+    // Called by std::istream for single-character operations (get, peek, etc.)
+    // when the get area is exhausted. Reads one byte from IOReader.
+    size_t got = (*r)(&single_char_buffer, 1, 1);
+    if (got == 0) {
+        return traits_type::eof();
+    }
+
+    // Configure get area to expose the single buffered character
+    setg(&single_char_buffer, &single_char_buffer, &single_char_buffer + 1);
+    return traits_type::to_int_type(single_char_buffer);
+}
+
+std::streamsize ReaderStreambuf::xsgetn(char* s, std::streamsize n) {
+    // Called by std::istream for bulk reads (read, readsome, etc.).
+    // Forwards directly to IOReader without intermediate buffering to avoid
+    // advancing IOReader beyond what the stream consumer requested.
+    if (n <= 0) {
+        return 0;
+    }
+
+    size_t got = (*r)(s, 1, n);
+    return static_cast<std::streamsize>(got);
+}
+
+} // namespace svs_io
+} // namespace faiss

data/vendor/faiss/faiss/impl/svs_io.h

@@ -0,0 +1,67 @@
+/*
+ * Portions Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/*
+ * Portions Copyright 2025 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <iostream>
+#include <streambuf>
+#include <vector>
+
+#include <faiss/impl/io.h>
+
+namespace faiss {
+namespace svs_io {
+
+// Bridges IOWriter to std::ostream for streaming serialization.
+// No buffering concerns since consumer is expected to write everything
+// he receives.
+struct WriterStreambuf : std::streambuf {
+    IOWriter* w;
+    explicit WriterStreambuf(IOWriter* w_);
+    ~WriterStreambuf() override;
+
+   protected:
+    std::streamsize xsputn(const char* s, std::streamsize n) override;
+    int overflow(int ch) override;
+};
+
+// Bridges IOReader to std::istream for streaming deserialization.
+// Uses minimal buffering (single byte) to avoid over-reading from IOReader,
+// which would advance its position beyond what the stream consumer actually
+// read. This ensures subsequent direct reads from IOReader continue at the
+// correct position. Bulk reads via xsgetn() forward directly to IOReader
+// without intermediate buffering.
+struct ReaderStreambuf : std::streambuf {
+    IOReader* r;
+    char single_char_buffer; // Single-byte buffer for underflow() operations
+
+    explicit ReaderStreambuf(IOReader* rr);
+    ~ReaderStreambuf() override;
+
+   protected:
+    int_type underflow() override;
+    std::streamsize xsgetn(char* s, std::streamsize n) override;
+};
+
+} // namespace svs_io
+} // namespace faiss

data/vendor/faiss/faiss/impl/zerocopy_io.h

@@ -20,7 +20,7 @@ struct ZeroCopyIOReader : public faiss::IOReader {
     size_t total_ = 0;
 
     ZeroCopyIOReader(const uint8_t* data, size_t size);
-    ~ZeroCopyIOReader();
+    ~ZeroCopyIOReader() override;
 
     void reset();
     size_t get_data_view(void** ptr, size_t size, size_t nitems);

data/vendor/faiss/faiss/index_factory.cpp

@@ -27,10 +27,12 @@
 #include <faiss/IndexIVFAdditiveQuantizer.h>
 #include <faiss/IndexIVFAdditiveQuantizerFastScan.h>
 #include <faiss/IndexIVFFlat.h>
+#include <faiss/IndexIVFFlatPanorama.h>
 #include <faiss/IndexIVFPQ.h>
 #include <faiss/IndexIVFPQFastScan.h>
 #include <faiss/IndexIVFPQR.h>
 #include <faiss/IndexIVFRaBitQ.h>
+#include <faiss/IndexIVFRaBitQFastScan.h>
 #include <faiss/IndexIVFSpectralHash.h>
 #include <faiss/IndexLSH.h>
 #include <faiss/IndexLattice.h>
@@ -39,6 +41,7 @@
 #include <faiss/IndexPQFastScan.h>
 #include <faiss/IndexPreTransform.h>
 #include <faiss/IndexRaBitQ.h>
+#include <faiss/IndexRaBitQFastScan.h>
 #include <faiss/IndexRefine.h>
 #include <faiss/IndexRowwiseMinMax.h>
 #include <faiss/IndexScalarQuantizer.h>
@@ -49,6 +52,16 @@
 #include <faiss/IndexBinaryHNSW.h>
 #include <faiss/IndexBinaryHash.h>
 #include <faiss/IndexBinaryIVF.h>
+
+#ifdef FAISS_ENABLE_SVS
+#include <faiss/svs/IndexSVSFlat.h>
+#include <faiss/svs/IndexSVSVamana.h>
+#include <faiss/svs/IndexSVSVamanaLVQ.h>
+#include <faiss/svs/IndexSVSVamanaLeanVec.h>
+#endif
+#include <faiss/IndexIDMap.h>
+#include <algorithm>
+#include <cctype>
 #include <string>
 
 namespace faiss {
@@ -187,8 +200,6 @@ std::vector<size_t> aq_parse_nbits(std::string stok) {
     return nbits;
 }
 
-const std::string rabitq_pattern = "(RaBitQ)";
-
 /***************************************************************
  * Parse VectorTransform
  */
@@ -326,6 +337,10 @@ IndexIVF* parse_IndexIVF(
     if (match("FlatDedup")) {
         return new IndexIVFFlatDedup(get_q(), d, nlist, mt, own_il);
     }
+    if (match("FlatPanorama([0-9]+)?")) {
+        int nlevels = mres_to_int(sm[1], 8); // default to 8 levels
+        return new IndexIVFFlatPanorama(get_q(), d, nlist, nlevels, mt, own_il);
+    }
     if (match(sq_pattern)) {
         return new IndexIVFScalarQuantizer(
                 get_q(),
@@ -447,8 +462,21 @@ IndexIVF* parse_IndexIVF(
         }
         return index_ivf;
     }
-    if (match(rabitq_pattern)) {
-        return new IndexIVFRaBitQ(get_q(), d, nlist, mt, own_il);
+    // IndexIVFRaBitQ with optional nb_bits (1-9)
+    // Accepts: "RaBitQ" (default 1-bit) or "RaBitQ{nb_bits}" (e.g., "RaBitQ4")
+    if (match("RaBitQ([1-9])?")) {
+        uint8_t nb_bits = sm[1].length() > 0 ? std::stoi(sm[1].str()) : 1;
+        return new IndexIVFRaBitQ(get_q(), d, nlist, mt, own_il, nb_bits);
+    }
+    // Accepts: "RaBitQfs" (default 1-bit, batch size 32)
+    //          "RaBitQfs{nb_bits}" (e.g., "RaBitQfs4")
+    //          "RaBitQfs_64" (1-bit, batch size 64)
+    //          "RaBitQfs{nb_bits}_{bbs}" (e.g., "RaBitQfs4_64")
+    if (match("RaBitQfs([1-9])?(_[0-9]+)?")) {
+        uint8_t nb_bits = sm[1].length() > 0 ? std::stoi(sm[1].str()) : 1;
+        int bbs = mres_to_int(sm[2], 32, 1);
+        return new IndexIVFRaBitQFastScan(
+                get_q(), d, nlist, mt, bbs, own_il, nb_bits);
     }
     return nullptr;
 }
@@ -471,6 +499,11 @@ IndexHNSW* parse_IndexHNSW(
         return new IndexHNSWFlat(d, hnsw_M, mt);
     }
 
+    if (match("FlatPanorama([0-9]+)?")) {
+        int nlevels = mres_to_int(sm[1], 8); // default to 8 levels
+        return new IndexHNSWFlatPanorama(d, hnsw_M, nlevels, mt);
+    }
+
     if (match("PQ([0-9]+)(x[0-9]+)?(np)?")) {
         int M = std::stoi(sm[1].str());
         int nbit = mres_to_int(sm[2], 8, 1);
@@ -537,6 +570,109 @@ IndexNSG* parse_IndexNSG(
     return nullptr;
 }
 
+#ifdef FAISS_ENABLE_SVS
+/***************************************************************
+ * Parse IndexSVS
+ */
+
+SVSStorageKind parse_lvq(const std::string& lvq_string) {
+    if (lvq_string == "LVQ4x0") {
+        return SVSStorageKind::SVS_LVQ4x0;
+    }
+    if (lvq_string == "LVQ4x4") {
+        return SVSStorageKind::SVS_LVQ4x4;
+    }
+    if (lvq_string == "LVQ4x8") {
+        return SVSStorageKind::SVS_LVQ4x8;
+    }
+    FAISS_ASSERT(!"not supported SVS LVQ level");
+}
+
+SVSStorageKind parse_leanvec(const std::string& leanvec_string) {
+    if (leanvec_string == "LeanVec4x4") {
+        return SVSStorageKind::SVS_LeanVec4x4;
+    }
+    if (leanvec_string == "LeanVec4x8") {
+        return SVSStorageKind::SVS_LeanVec4x8;
+    }
+    if (leanvec_string == "LeanVec8x8") {
+        return SVSStorageKind::SVS_LeanVec8x8;
+    }
+    FAISS_ASSERT(!"not supported SVS Leanvec level");
+}
+
+Index* parse_svs_datatype(
+        const std::string& index_type,
+        const std::string& arg_string,
+        const std::string& datatype_string,
+        int d,
+        MetricType mt) {
+    std::smatch sm;
+
+    if (datatype_string.empty()) {
+        if (index_type == "Vamana")
+            return new IndexSVSVamana(d, std::stoul(arg_string), mt);
+        if (index_type == "Flat")
+            return new IndexSVSFlat(d, mt);
+        FAISS_ASSERT(!"Unspported SVS index type");
+    }
+    if (re_match(datatype_string, "FP16", sm)) {
+        if (index_type == "Vamana")
+            return new IndexSVSVamana(
+                    d, std::stoul(arg_string), mt, SVSStorageKind::SVS_FP16);
+        FAISS_ASSERT(!"Unspported SVS index type for Float16");
+    }
+    if (re_match(datatype_string, "SQI8", sm)) {
+        if (index_type == "Vamana")
+            return new IndexSVSVamana(
+                    d, std::stoul(arg_string), mt, SVSStorageKind::SVS_SQI8);
+        FAISS_ASSERT(!"Unspported SVS index type for SQI8");
+    }
+    if (re_match(datatype_string, "(LVQ[0-9]+x[0-9]+)", sm)) {
+        if (index_type == "Vamana")
+            return new IndexSVSVamanaLVQ(
+                    d, std::stoul(arg_string), mt, parse_lvq(sm[0].str()));
+        FAISS_ASSERT(!"Unspported SVS index type for LVQ");
+    }
+    if (re_match(datatype_string, "(LeanVec[0-9]+x[0-9]+)(_[0-9]+)?", sm)) {
+        std::string leanvec_d_string =
+                sm[2].length() > 0 ? sm[2].str().substr(1) : "0";
+        int leanvec_d = std::stoul(leanvec_d_string);
+
+        if (index_type == "Vamana")
+            return new IndexSVSVamanaLeanVec(
+                    d,
+                    std::stoul(arg_string),
+                    mt,
+                    leanvec_d,
+                    parse_leanvec(sm[1].str()));
+        FAISS_ASSERT(!"Unspported SVS index type for LeanVec");
+    }
+    return nullptr;
+}
+
+Index* parse_IndexSVS(const std::string& code_string, int d, MetricType mt) {
+    std::smatch sm;
+    if (re_match(code_string, "Flat(,.+)?", sm)) {
+        std::string datatype_string =
+                sm[1].length() > 0 ? sm[1].str().substr(1) : "";
+        return parse_svs_datatype("Flat", "", datatype_string, d, mt);
+    }
+    if (re_match(code_string, "Vamana([0-9]+)(,.+)?", sm)) {
+        Index* index{nullptr};
+        std::string degree_string = sm[1].str();
+        std::string datatype_string =
+                sm[2].length() > 0 ? sm[2].str().substr(1) : "";
+        return parse_svs_datatype(
+                "Vamana", degree_string, datatype_string, d, mt);
+    }
+    if (re_match(code_string, "IVF([0-9]+)(,.+)?", sm)) {
+        FAISS_ASSERT(!"Unspported SVS index type");
+    }
+    return nullptr;
+}
+#endif // FAISS_ENABLE_SVS
+
 /***************************************************************
  * Parse basic indexes
  */
@@ -555,6 +691,18 @@ Index* parse_other_indexes(
         return new IndexFlat(d, metric);
     }
 
+    // IndexFlatL2Panorama
+    if (match("FlatL2Panorama([0-9]+)(_[0-9]+)?")) {
+        FAISS_THROW_IF_NOT(metric == METRIC_L2);
+        int nlevels = std::stoi(sm[1].str());
+        if (sm[2].length() > 0) {
+            int batch_size = std::stoi(sm[2].str().substr(1));
+            return new IndexFlatL2Panorama(d, nlevels, (size_t)batch_size);
+        } else {
+            return new IndexFlatL2Panorama(d, nlevels);
+        }
+    }
+
     // IndexLSH
     if (match("LSH([0-9]*)(r?)(t?)")) {
         int nbits = sm[1].length() > 0 ? std::stoi(sm[1].str()) : d;
@@ -671,9 +819,17 @@ Index* parse_other_indexes(
         }
     }
 
-    // IndexRaBitQ
-    if (match(rabitq_pattern)) {
-        return new IndexRaBitQ(d, metric);
+    // IndexRaBitQ with optional nb_bits (1-9)
+    // Accepts: "RaBitQ" (default 1-bit) or "RaBitQ{nb_bits}" (e.g., "RaBitQ4")
+    if (match("RaBitQ([1-9])?")) {
+        uint8_t nb_bits = sm[1].length() > 0 ? std::stoi(sm[1].str()) : 1;
+        return new IndexRaBitQ(d, metric, nb_bits);
+    }
+
+    if (match("RaBitQfs([1-9])?(_[0-9]+)?")) {
+        uint8_t nb_bits = sm[1].length() > 0 ? std::stoi(sm[1].str()) : 1;
+        int bbs = mres_to_int(sm[2], 32, 1);
+        return new IndexRaBitQFastScan(d, metric, bbs, nb_bits);
     }
 
     return nullptr;
@@ -716,6 +872,18 @@ std::unique_ptr<Index> index_factory_sub(
         return std::unique_ptr<Index>(idmap);
     }
 
+    // handle refine Panorama
+    // TODO(aknayar): Add tests to test_factory.py
+    if (re_match(description, "(.+),RefinePanorama\\((.+)\\)", sm)) {
+        std::unique_ptr<Index> filter_index =
+                index_factory_sub(d, sm[1].str(), metric);
+        std::unique_ptr<Index> refine_index =
+                index_factory_sub(d, sm[2].str(), metric);
+        auto* index_rf = new IndexRefinePanorama(
+                filter_index.release(), refine_index.release());
+        return std::unique_ptr<Index>(index_rf);
+    }
+
     // handle refines
     if (re_match(description, "(.+),RFlat", sm) ||
         re_match(description, "(.+),Refine\\((.+)\\)", sm)) {
@@ -822,6 +990,25 @@ std::unique_ptr<Index> index_factory_sub(
         return std::unique_ptr<Index>(index);
     }
 
+#ifdef FAISS_ENABLE_SVS
+    if (re_match(description, "SVS((?:Flat|Vamana|IVF).*)", sm)) {
+        std::string code_string = sm[1].str();
+        if (verbose) {
+            printf("parsing SVS string %s code_string=%s",
+                   description.c_str(),
+                   code_string.c_str());
+        }
+
+        Index* index = parse_IndexSVS(code_string, d, metric);
+        FAISS_THROW_IF_NOT_FMT(
+                index,
+                "could not parse SVS code description %s in %s",
+                code_string.c_str(),
+                description.c_str());
+        return std::unique_ptr<Index>(index);
+    }
+#endif // FAISS_ENABLE_SVS
+
     // NSG variants (it was unclear in the old version that the separator was a
     // "," so we support both "_" and ",")
     if (re_match(description, "NSG([0-9]*)([,_].*)?", sm)) {
@@ -934,6 +1121,28 @@ IndexBinary* index_binary_factory(
         bool own_invlists) {
     IndexBinary* index = nullptr;
 
+    std::smatch sm;
+    std::string desc_str(description);
+
+    // Handle IDMap2 and IDMap wrappers (prefix or suffix)
+    if (re_match(desc_str, "(.+),IDMap2", sm) ||
+        re_match(desc_str, "IDMap2,(.+)", sm)) {
+        IndexBinary* sub_index =
+                index_binary_factory(d, sm[1].str().c_str(), own_invlists);
+        IndexBinaryIDMap2* idmap2 = new IndexBinaryIDMap2(sub_index);
+        idmap2->own_fields = true;
+        return idmap2;
+    }
+
+    if (re_match(desc_str, "(.+),IDMap", sm) ||
+        re_match(desc_str, "IDMap,(.+)", sm)) {
+        IndexBinary* sub_index =
+                index_binary_factory(d, sm[1].str().c_str(), own_invlists);
+        IndexBinaryIDMap* idmap = new IndexBinaryIDMap(sub_index);
+        idmap->own_fields = true;
+        return idmap;
+    }
+
     int ncentroids = -1;
     int M, nhash, b;
 
@@ -959,7 +1168,7 @@ IndexBinary* index_binary_factory(
     } else if (sscanf(description, "BHash%d", &b) == 1) {
         index = new IndexBinaryHash(d, b);
 
-    } else if (std::string(description) == "BFlat") {
+    } else if (desc_str == "BFlat") {
         index = new IndexBinaryFlat(d);
 
     } else {

data/vendor/faiss/faiss/index_factory.h

@@ -12,7 +12,7 @@
 
 namespace faiss {
 
-/** Build and index with the sequence of processing steps described in
+/** Build an index with the sequence of processing steps described in
  *  the string. */
 Index* index_factory(
         int d,

data/vendor/faiss/faiss/index_io.h

@@ -16,7 +16,7 @@
  * object that abstracts the medium.
  *
  * The read functions return objects that should be deallocated with
- * delete. All references within these objectes are owned by the
+ * delete. All references within these objects are owned by the
  * object.
  */
 

data/vendor/faiss/faiss/invlists/BlockInvertedLists.h

@@ -20,7 +20,7 @@ struct IDSelector;
 /** Inverted Lists that are organized by blocks.
  *
  * Different from the regular inverted lists, the codes are organized by blocks
- * of size block_size bytes that reprsent a set of n_per_block. Therefore, code
+ * of size block_size bytes that represent a set of n_per_block. Therefore, code
  * allocations are always rounded up to block_size bytes. The codes are also
  * aligned on 32-byte boundaries for use with SIMD.
  *