faiss 0.4.1 → 0.4.2

data/vendor/faiss/faiss/IndexIVFPQFastScan.h

@@ -47,7 +47,8 @@ struct IndexIVFPQFastScan : IndexIVFFastScan {
             size_t M,
             size_t nbits,
             MetricType metric = METRIC_L2,
-            int bbs = 32);
+            int bbs = 32,
+            bool own_invlists = true);
 
     IndexIVFPQFastScan();
 

data/vendor/faiss/faiss/IndexIVFPQR.cpp

@@ -30,8 +30,16 @@ IndexIVFPQR::IndexIVFPQR(
         size_t M,
         size_t nbits_per_idx,
         size_t M_refine,
-        size_t nbits_per_idx_refine)
-        : IndexIVFPQ(quantizer, d, nlist, M, nbits_per_idx),
+        size_t nbits_per_idx_refine,
+        bool own_invlists)
+        : IndexIVFPQ(
+                  quantizer,
+                  d,
+                  nlist,
+                  M,
+                  nbits_per_idx,
+                  METRIC_L2,
+                  own_invlists),
           refine_pq(d, M_refine, nbits_per_idx_refine),
           k_factor(4) {
     by_residual = true;
@@ -160,8 +168,9 @@ void IndexIVFPQR::search_preassigned(
             for (int j = 0; j < k_coarse; j++) {
                 idx_t sl = shortlist[j];
 
-                if (sl == -1)
+                if (sl == -1) {
                     continue;
+                }
 
                 int list_no = lo_listno(sl);
                 int ofs = lo_offset(sl);
@@ -176,8 +185,9 @@ void IndexIVFPQR::search_preassigned(
                 const uint8_t* l2code = invlists->get_single_code(list_no, ofs);
 
                 pq.decode(l2code, residual_2);
-                for (int l = 0; l < d; l++)
+                for (int l = 0; l < d; l++) {
                     residual_2[l] = residual_1[l] - residual_2[l];
+                }
 
                 // 3rd level residual's approximation
                 idx_t id = invlists->get_single_id(list_no, ofs);

data/vendor/faiss/faiss/IndexIVFPQR.h

@@ -30,7 +30,8 @@ struct IndexIVFPQR : IndexIVFPQ {
             size_t M,
             size_t nbits_per_idx,
             size_t M_refine,
-            size_t nbits_per_idx_refine);
+            size_t nbits_per_idx_refine,
+            bool own_invlists = true);
 
     void reset() override;
 

data/vendor/faiss/faiss/IndexIVFRaBitQ.cpp

@@ -23,10 +23,14 @@ IndexIVFRaBitQ::IndexIVFRaBitQ(
         Index* quantizer,
         const size_t d,
         const size_t nlist,
-        MetricType metric)
-        : IndexIVF(quantizer, d, nlist, 0, metric), rabitq(d, metric) {
+        MetricType metric,
+        bool own_invlists)
+        : IndexIVF(quantizer, d, nlist, 0, metric, own_invlists),
+          rabitq(d, metric) {
     code_size = rabitq.code_size;
-    invlists->code_size = code_size;
+    if (own_invlists) {
+        invlists->code_size = code_size;
+    }
     is_trained = false;
 
     by_residual = true;
@@ -76,6 +80,27 @@ void IndexIVFRaBitQ::encode_vectors(
     }
 }
 
+void IndexIVFRaBitQ::decode_vectors(
+        idx_t n,
+        const uint8_t* codes,
+        const idx_t* listnos,
+        float* x) const {
+#pragma omp parallel
+    {
+        std::vector<float> centroid(d);
+
+#pragma omp for
+        for (idx_t i = 0; i < n; i++) {
+            const uint8_t* code = codes + i * code_size;
+            int64_t list_no = listnos[i];
+            float* xi = x + i * d;
+
+            quantizer->reconstruct(list_no, centroid.data());
+            rabitq.decode_core(code, xi, 1, centroid.data());
+        }
+    }
+}
+
 void IndexIVFRaBitQ::add_core(
         idx_t n,
         const float* x,

data/vendor/faiss/faiss/IndexIVFRaBitQ.h

@@ -33,7 +33,8 @@ struct IndexIVFRaBitQ : IndexIVF {
             Index* quantizer,
             const size_t d,
             const size_t nlist,
-            MetricType metric = METRIC_L2);
+            MetricType metric = METRIC_L2,
+            bool own_invlists = true);
 
     IndexIVFRaBitQ();
 
@@ -46,6 +47,12 @@ struct IndexIVFRaBitQ : IndexIVF {
             uint8_t* codes,
             bool include_listnos = false) const override;
 
+    void decode_vectors(
+            idx_t n,
+            const uint8_t* codes,
+            const idx_t* list_nos,
+            float* x) const override;
+
     void add_core(
             idx_t n,
             const float* x,

data/vendor/faiss/faiss/IndexIVFSpectralHash.cpp

@@ -27,8 +27,15 @@ IndexIVFSpectralHash::IndexIVFSpectralHash(
         size_t d,
         size_t nlist,
         int nbit,
-        float period)
-        : IndexIVF(quantizer, d, nlist, (nbit + 7) / 8, METRIC_L2),
+        float period,
+        bool own_invlists)
+        : IndexIVF(
+                  quantizer,
+                  d,
+                  nlist,
+                  (nbit + 7) / 8,
+                  METRIC_L2,
+                  own_invlists),
           nbit(nbit),
           period(period) {
     RandomRotationMatrix* rr = new RandomRotationMatrix(d, nbit);

data/vendor/faiss/faiss/IndexIVFSpectralHash.h

@@ -56,7 +56,8 @@ struct IndexIVFSpectralHash : IndexIVF {
             size_t d,
             size_t nlist,
             int nbit,
-            float period);
+            float period,
+            bool own_invlists = true);
 
     IndexIVFSpectralHash();
 

data/vendor/faiss/faiss/IndexLattice.cpp

@@ -47,10 +47,12 @@ void IndexLattice::train(idx_t n, const float* x) {
     for (idx_t i = 0; i < n; i++) {
         for (int sq = 0; sq < nsq; sq++) {
             float norm2 = fvec_norm_L2sqr(x + i * d + sq * dsq, dsq);
-            if (norm2 > maxs[sq])
+            if (norm2 > maxs[sq]) {
                 maxs[sq] = norm2;
-            if (norm2 < mins[sq])
+            }
+            if (norm2 < mins[sq]) {
                 mins[sq] = norm2;
+            }
         }
     }
 
@@ -79,10 +81,12 @@ void IndexLattice::sa_encode(idx_t n, const float* x, uint8_t* codes) const {
         for (int j = 0; j < nsq; j++) {
             float nj = (sqrtf(fvec_norm_L2sqr(xi, dsq)) - mins[j]) * sc /
                     (maxs[j] - mins[j]);
-            if (nj < 0)
+            if (nj < 0) {
                 nj = 0;
-            if (nj >= sc)
+            }
+            if (nj >= sc) {
                 nj = sc - 1;
+            }
             wr.write((int64_t)nj, scale_nbit);
             wr.write(zn_sphere_codec.encode(xi), lattice_nbit);
             xi += dsq;

data/vendor/faiss/faiss/IndexNNDescent.cpp

@@ -9,24 +9,17 @@
 
 #include <faiss/IndexNNDescent.h>
 
-#include <omp.h>
-
 #include <cinttypes>
 #include <cstdio>
 #include <cstdlib>
 
-#include <queue>
-#include <unordered_set>
-
 #ifdef __SSE__
 #endif
 
 #include <faiss/IndexFlat.h>
 #include <faiss/impl/AuxIndexStructures.h>
 #include <faiss/impl/FaissAssert.h>
-#include <faiss/utils/Heap.h>
 #include <faiss/utils/distances.h>
-#include <faiss/utils/random.h>
 
 extern "C" {
 

data/vendor/faiss/faiss/IndexNSG.cpp

@@ -101,7 +101,7 @@ void IndexNSG::search(
     }
 }
 
-void IndexNSG::build(idx_t n, const float* x, idx_t* knn_graph, int GK_2) {
+void IndexNSG::build(idx_t n, const float* x, idx_t* knn_graph, int gk) {
     FAISS_THROW_IF_NOT_MSG(
             storage,
             "Please use IndexNSGFlat (or variants) instead of IndexNSG directly");
@@ -112,9 +112,9 @@ void IndexNSG::build(idx_t n, const float* x, idx_t* knn_graph, int GK_2) {
     ntotal = storage->ntotal;
 
     // check the knn graph
-    check_knn_graph(knn_graph, n, GK_2);
+    check_knn_graph(knn_graph, n, gk);
 
-    const nsg::Graph<idx_t> knng(knn_graph, n, GK_2);
+    const nsg::Graph<idx_t> knng(knn_graph, n, gk);
     nsg.build(storage, n, knng, verbose);
     is_built = true;
 }

data/vendor/faiss/faiss/IndexPQ.cpp

@@ -8,7 +8,6 @@
 #include <faiss/IndexPQ.h>
 
 #include <cinttypes>
-#include <cmath>
 #include <cstddef>
 #include <cstdio>
 #include <cstring>

data/vendor/faiss/faiss/IndexPQ.h

@@ -153,6 +153,7 @@ struct MultiIndexQuantizer : Index {
 
     /// add and reset will crash at runtime
     void add(idx_t n, const float* x) override;
+
     void reset() override;
 
     MultiIndexQuantizer() {}

data/vendor/faiss/faiss/IndexPQFastScan.cpp

@@ -7,10 +7,8 @@
 
 #include <faiss/IndexPQFastScan.h>
 
-#include <cassert>
 #include <memory>
 
-#include <faiss/impl/FaissAssert.h>
 #include <faiss/impl/pq4_fast_scan.h>
 #include <faiss/utils/utils.h>
 

data/vendor/faiss/faiss/IndexPreTransform.cpp

@@ -48,8 +48,9 @@ void IndexPreTransform::prepend_transform(VectorTransform* ltrans) {
 
 IndexPreTransform::~IndexPreTransform() {
     if (own_fields) {
-        for (int i = 0; i < chain.size(); i++)
+        for (int i = 0; i < chain.size(); i++) {
             delete chain[i];
+        }
         delete index;
     }
 }
@@ -94,8 +95,9 @@ void IndexPreTransform::train(idx_t n, const float* x) {
             }
             index->train(n, prev_x);
         }
-        if (i == last_untrained)
+        if (i == last_untrained) {
             break;
+        }
         if (verbose) {
             printf("   Applying transform %d/%zd\n", i, chain.size());
         }

data/vendor/faiss/faiss/IndexRefine.cpp

@@ -129,10 +129,11 @@ void IndexRefine::search(
     base_index->search(
             n, x, k_base, base_distances, base_labels, base_index_params);
 
-    for (int i = 0; i < n * k_base; i++)
+    for (int i = 0; i < n * k_base; i++) {
         assert(base_labels[i] >= -1 && base_labels[i] < ntotal);
+    }
 
-        // parallelize over queries
+    // parallelize over queries
 #pragma omp parallel if (n > 1)
     {
         std::unique_ptr<DistanceComputer> dc(
@@ -143,8 +144,9 @@ void IndexRefine::search(
             idx_t ij = i * k_base;
             for (idx_t j = 0; j < k_base; j++) {
                 idx_t idx = base_labels[ij];
-                if (idx < 0)
+                if (idx < 0) {
                     break;
+                }
                 base_distances[ij] = (*dc)(idx);
                 ij++;
             }
@@ -238,10 +240,12 @@ void IndexRefine::sa_decode(idx_t n, const uint8_t* bytes, float* x) const {
 }
 
 IndexRefine::~IndexRefine() {
-    if (own_fields)
+    if (own_fields) {
         delete base_index;
-    if (own_refine_index)
+    }
+    if (own_refine_index) {
         delete refine_index;
+    }
 }
 
 /***************************************************
@@ -312,8 +316,9 @@ void IndexRefineFlat::search(
     base_index->search(
             n, x, k_base, base_distances, base_labels, base_index_params);
 
-    for (int i = 0; i < n * k_base; i++)
+    for (int i = 0; i < n * k_base; i++) {
         assert(base_labels[i] >= -1 && base_labels[i] < ntotal);
+    }
 
     // compute refined distances
     auto rf = dynamic_cast<const IndexFlat*>(refine_index);

data/vendor/faiss/faiss/IndexScalarQuantizer.cpp

@@ -122,12 +122,15 @@ IndexIVFScalarQuantizer::IndexIVFScalarQuantizer(
         size_t nlist,
         ScalarQuantizer::QuantizerType qtype,
         MetricType metric,
-        bool by_residual)
-        : IndexIVF(quantizer, d, nlist, 0, metric), sq(d, qtype) {
+        bool by_residual,
+        bool own_invlists)
+        : IndexIVF(quantizer, d, nlist, 0, metric, own_invlists), sq(d, qtype) {
     code_size = sq.code_size;
     this->by_residual = by_residual;
-    // was not known at construction time
-    invlists->code_size = code_size;
+    if (invlists) {
+        // was not known at construction time
+        invlists->code_size = code_size;
+    }
     is_trained = false;
 }
 
@@ -179,6 +182,15 @@ void IndexIVFScalarQuantizer::encode_vectors(
     }
 }
 
+void IndexIVFScalarQuantizer::decode_vectors(
+        idx_t n,
+        const uint8_t* codes,
+        const idx_t*,
+        float* x) const {
+    FAISS_THROW_IF_NOT(is_trained);
+    return sq.decode(codes, x, n);
+}
+
 void IndexIVFScalarQuantizer::sa_decode(idx_t n, const uint8_t* codes, float* x)
         const {
     std::unique_ptr<ScalarQuantizer::SQuantizer> squant(sq.select_quantizer());

data/vendor/faiss/faiss/IndexScalarQuantizer.h

@@ -29,8 +29,8 @@ struct IndexScalarQuantizer : IndexFlatCodes {
     /** Constructor.
      *
      * @param d      dimensionality of the input vectors
-     * @param M      number of subquantizers
-     * @param nbits  number of bit per subvector index
+     * @param qtype  type of scalar quantizer (e.g., QT_4bit)
+     * @param metric distance metric used for search (default: METRIC_L2)
      */
     IndexScalarQuantizer(
             int d,
@@ -72,7 +72,8 @@ struct IndexIVFScalarQuantizer : IndexIVF {
             size_t nlist,
             ScalarQuantizer::QuantizerType qtype,
             MetricType metric = METRIC_L2,
-            bool by_residual = true);
+            bool by_residual = true,
+            bool own_invlists = true);
 
     IndexIVFScalarQuantizer();
 
@@ -87,6 +88,12 @@ struct IndexIVFScalarQuantizer : IndexIVF {
             uint8_t* codes,
             bool include_listnos = false) const override;
 
+    void decode_vectors(
+            idx_t n,
+            const uint8_t* codes,
+            const idx_t* list_nos,
+            float* x) const override;
+
     void add_core(
             idx_t n,
             const float* x,

data/vendor/faiss/faiss/IndexShards.cpp

@@ -31,11 +31,13 @@ void sync_d(IndexBinary* index) {
 
 // add translation to all valid labels
 void translate_labels(int64_t n, idx_t* labels, int64_t translation) {
-    if (translation == 0)
+    if (translation == 0) {
         return;
+    }
     for (int64_t i = 0; i < n; i++) {
-        if (labels[i] < 0)
+        if (labels[i] < 0) {
             continue;
+        }
         labels[i] += translation;
     }
 }
@@ -199,8 +201,6 @@ void IndexShardsTemplate<IndexT>::search(
         distance_t* distances,
         idx_t* labels,
         const SearchParameters* params) const {
-    FAISS_THROW_IF_NOT_MSG(
-            !params, "search params not supported for this index");
     FAISS_THROW_IF_NOT(k > 0);
 
     int64_t nshard = this->count();
@@ -219,7 +219,7 @@ void IndexShardsTemplate<IndexT>::search(
         }
     }
 
-    auto fn = [n, k, x, &all_distances, &all_labels, &translations](
+    auto fn = [n, k, x, params, &all_distances, &all_labels, &translations](
                       int no, const IndexT* index) {
         if (index->verbose) {
             printf("begin query shard %d on %" PRId64 " points\n", no, n);
@@ -230,7 +230,8 @@ void IndexShardsTemplate<IndexT>::search(
                 x,
                 k,
                 all_distances.data() + no * k * n,
-                all_labels.data() + no * k * n);
+                all_labels.data() + no * k * n,
+                params);
 
         translate_labels(
                 n * k, all_labels.data() + no * k * n, translations[no]);

data/vendor/faiss/faiss/MatrixStats.cpp

@@ -32,12 +32,15 @@ void MatrixStats::PerDimStats::add(float x) {
         n_inf++;
         return;
     }
-    if (x == 0)
+    if (x == 0) {
         n0++;
-    if (x < min)
+    }
+    if (x < min) {
         min = x;
-    if (x > max)
+    }
+    if (x > max) {
         max = x;
+    }
     sum += x;
     sum2 += (double)x * (double)x;
 }
@@ -46,8 +49,9 @@ void MatrixStats::PerDimStats::compute_mean_std() {
     n_valid = n - n_nan - n_inf;
     mean = sum / n_valid;
     double var = sum2 / n_valid - mean * mean;
-    if (var < 0)
+    if (var < 0) {
         var = 0;
+    }
     stddev = sqrt(var);
 }
 
@@ -95,10 +99,12 @@ MatrixStats::MatrixStats(size_t n, size_t d, const float* x) : n(n), d(d) {
             if (sum2 == 0) {
                 n0++;
             } else {
-                if (sum2 < min_norm2)
+                if (sum2 < min_norm2) {
                     min_norm2 = sum2;
-                if (sum2 > max_norm2)
+                }
+                if (sum2 > max_norm2) {
                     max_norm2 = sum2;
+                }
             }
         }
 
@@ -194,10 +200,12 @@ MatrixStats::MatrixStats(size_t n, size_t d, const float* x) : n(n), d(d) {
                 n_dangerous_range++;
             }
 
-            if (st.stddev > max_std)
+            if (st.stddev > max_std) {
                 max_std = st.stddev;
-            if (st.stddev < min_std)
+            }
+            if (st.stddev < min_std) {
                 min_std = st.stddev;
+            }
         }
 
         if (n0_2 == 0) {

data/vendor/faiss/faiss/MetaIndexes.cpp

@@ -36,8 +36,9 @@ void IndexSplitVectors::add_sub_index(Index* index) {
 }
 
 void IndexSplitVectors::sync_with_sub_indexes() {
-    if (sub_indexes.empty())
+    if (sub_indexes.empty()) {
         return;
+    }
     Index* index0 = sub_indexes[0];
     sum_d = index0->d;
     metric_type = index0->metric_type;
@@ -81,24 +82,28 @@ void IndexSplitVectors::search(
                         no == 0 ? distances : all_distances.get() + no * k * n;
                 idx_t* labels1 =
                         no == 0 ? labels : all_labels.get() + no * k * n;
-                if (index->verbose)
+                if (index->verbose) {
                     printf("begin query shard %d on %" PRId64 " points\n",
                            no,
                            n);
+                }
                 const Index* sub_index = index->sub_indexes[no];
                 int64_t sub_d = sub_index->d, d = index->d;
                 idx_t ofs = 0;
-                for (int i = 0; i < no; i++)
+                for (int i = 0; i < no; i++) {
                     ofs += index->sub_indexes[i]->d;
+                }
 
                 std::unique_ptr<float[]> sub_x(new float[sub_d * n]);
-                for (idx_t i = 0; i < n; i++)
+                for (idx_t i = 0; i < n; i++) {
                     memcpy(sub_x.get() + i * sub_d,
                            x + ofs + i * d,
                            sub_d * sizeof(float));
+                }
                 sub_index->search(n, sub_x.get(), k, distances1, labels1);
-                if (index->verbose)
+                if (index->verbose) {
                     printf("end query shard %d\n", no);
+                }
             };
 
     if (!threaded) {
@@ -150,8 +155,9 @@ void IndexSplitVectors::reset() {
 
 IndexSplitVectors::~IndexSplitVectors() {
     if (own_fields) {
-        for (int s = 0; s < sub_indexes.size(); s++)
+        for (int s = 0; s < sub_indexes.size(); s++) {
             delete sub_indexes[s];
+        }
     }
 }
 

data/vendor/faiss/faiss/MetricType.h

@@ -10,7 +10,8 @@
 #ifndef FAISS_METRIC_TYPE_H
 #define FAISS_METRIC_TYPE_H
 
-#include <faiss/impl/platform_macros.h>
+#include <cstdint>
+#include <cstdio>
 
 namespace faiss {
 
@@ -36,8 +37,9 @@ enum MetricType {
     METRIC_Jaccard,
     /// Squared Eucliden distance, ignoring NaNs
     METRIC_NaNEuclidean,
-    /// abs(x | y): the distance to a hyperplane
-    METRIC_ABS_INNER_PRODUCT,
+    /// Gower's distance - numeric dimensions are in [0,1] and categorical
+    /// dimensions are negative integers
+    METRIC_GOWER,
 };
 
 /// all vector indices are this type

data/vendor/faiss/faiss/clone_index.cpp

@@ -9,9 +9,6 @@
 
 #include <faiss/clone_index.h>
 
-#include <cstdio>
-#include <cstdlib>
-
 #include <faiss/impl/FaissAssert.h>
 
 #include <faiss/Index2Layer.h>
@@ -315,8 +312,9 @@ Index* Cloner::clone_Index(const Index* index) {
         res->metric_arg = ipt->metric_arg;
 
         res->index = clone_Index(ipt->index);
-        for (int i = 0; i < ipt->chain.size(); i++)
+        for (int i = 0; i < ipt->chain.size(); i++) {
             res->chain.push_back(clone_VectorTransform(ipt->chain[i]));
+        }
         res->own_fields = true;
         return res;
     } else if (

data/vendor/faiss/faiss/cppcontrib/factory_tools.cpp

@@ -18,10 +18,12 @@
 #include <faiss/IndexIDMap.h>
 #include <faiss/IndexIVFFlat.h>
 #include <faiss/IndexIVFPQFastScan.h>
+#include <faiss/IndexIVFRaBitQ.h>
 #include <faiss/IndexLSH.h>
 #include <faiss/IndexNSG.h>
 #include <faiss/IndexPQFastScan.h>
 #include <faiss/IndexPreTransform.h>
+#include <faiss/IndexRaBitQ.h>
 #include <faiss/IndexRefine.h>
 
 namespace faiss {
@@ -103,6 +105,8 @@ std::string reverse_index_factory(const faiss::Index* index) {
                                 ivf_index)) {
             return prefix + ",PQ" + std::to_string(ivfpqfs_index->pq.M) + "x" +
                     std::to_string(ivfpqfs_index->pq.nbits) + "fs";
+        } else if (dynamic_cast<const faiss::IndexIVFRaBitQ*>(ivf_index)) {
+            return prefix + ",RaBitQ";
         }
     } else if (
             const faiss::IndexPreTransform* pretransform_index =
@@ -175,6 +179,8 @@ std::string reverse_index_factory(const faiss::Index* index) {
             const faiss::IndexIDMap* idmap =
                     dynamic_cast<const faiss::IndexIDMap*>(index)) {
         return std::string("IDMap,") + reverse_index_factory(idmap->index);
+    } else if (dynamic_cast<const faiss::IndexRaBitQ*>(index)) {
+        return "RaBitQ";
     }
     // Avoid runtime error, just return empty string for logging.
     return "";