faiss 0.6.2 → 0.6.4

data/vendor/faiss/faiss/impl/HNSW.h

@@ -7,8 +7,10 @@
 
 #pragma once
 
+#include <functional>
 #include <optional>
 #include <queue>
+#include <utility>
 #include <vector>
 
 #include <omp.h>
@@ -152,8 +154,14 @@ struct HNSW {
     /// use bounded queue during exploration
     bool search_bounded_queue = true;
 
-    /// use Panorama progressive pruning in search
-    bool is_panorama = false;
+    /// Specialized level-0 search implementations. This state is derived from
+    /// the owning IndexHNSW subtype after construction or deserialization.
+    enum Search_method_t : uint8_t {
+        SM_DEFAULT,
+        SM_PANORAMA,
+        SM_RABITQ,
+    };
+    Search_method_t search_method = SM_DEFAULT;
 
     /// distance comparison semantics: when true, distances are treated as
     /// similarity scores (larger is better). Default false matches the
@@ -204,14 +212,27 @@ struct HNSW {
             VisitedTable& vt,
             bool keep_max_size_level0 = false);
 
-    /** add point pt_id on all levels <= pt_level and build the link
-     * structure for them. */
-    void add_with_locks(
+    /** Deterministic build, phase A: write pt_id's forward links against an
+     * immutable snapshot, touching only pt_id's own slots. Reciprocal edges
+     * are collected in `pt_reverse_edges` for phase B, not applied. Requires
+     * entry_point set. */
+    void compute_forward_links_deterministic(
             DistanceComputer& ptdis,
             int pt_level,
-            int pt_id,
-            LockVector& locks,
+            storage_idx_t pt_id,
             VisitedTable& vt,
+            std::vector<std::pair<storage_idx_t, int>>& pt_reverse_edges,
+            bool keep_max_size_level0 = false);
+
+    /** Deterministic build, phase B: merge `incoming` into `node` and
+     * re-prune in a total order (distance, ties by id) so the result is
+     * order-independent. Touches only `node`'s slots; `incoming` is sorted
+     * and deduplicated in place. */
+    void merge_reverse_links_deterministic(
+            DistanceComputer& dis,
+            storage_idx_t node,
+            int level,
+            std::vector<storage_idx_t>& incoming,
             bool keep_max_size_level0 = false);
 
     /// Search interface for 1 point, single thread
@@ -257,6 +278,24 @@ struct HNSW {
     void permute_entries(const idx_t* map);
 };
 
+/** Deterministic, lock-free HNSW graph build. This is the only graph build
+ * that add() uses, and it is shared by IndexHNSW and
+ * IndexBinaryHNSW. The callbacks let both share the algorithm:
+ * `make_distance_computer()` returns a fresh DistanceComputer per thread
+ * (caller-owned) and `set_query(dc, pt_id)` points it at pt_id's vector. */
+void hnsw_add_vertices_deterministic(
+        HNSW& hnsw,
+        size_t n0,
+        size_t n,
+        int d,
+        bool init_level0,
+        bool keep_max_size_level0,
+        bool preset_levels,
+        bool verbose,
+        const std::function<DistanceComputer*()>& make_distance_computer,
+        const std::function<void(DistanceComputer&, HNSW::storage_idx_t)>&
+                set_query);
+
 struct HNSWStats {
     size_t n1 = 0; /// number of vectors searched
     size_t n2 =

data/vendor/faiss/faiss/impl/IDSelector.h

@@ -23,6 +23,50 @@ struct IDSelector {
     virtual ~IDSelector() {}
 };
 
+/** Scan context handed to IDSelectorWithContext::is_member_with_context: the
+ * contiguous id block being scanned and the position of the id under test. */
+struct IDScanContext {
+    /// the contiguous block of ids being scanned
+    const idx_t* ids;
+    /// number of entries in `ids`
+    size_t list_size;
+    /// index of the tested id within `ids` (i.e. ids[j] == id)
+    size_t j;
+};
+
+/** IDSelector that also receives the surrounding scan context on each
+ * membership test, letting an implementation exploit locality across a scan
+ * (for example, prefetching data for an entry it will be asked about soon).
+ * The policy is entirely up to the implementation. */
+struct IDSelectorWithContext : IDSelector {
+    virtual bool is_member_with_context(idx_t id, const IDScanContext& ctx)
+            const = 0;
+};
+
+/** Routes each per-candidate membership test to is_member_with_context() when
+ * the selector implements IDSelectorWithContext, else to plain is_member().
+ * Construct one per inverted-list scan: the dynamic_cast is the only RTTI cost
+ * and the per-candidate cost is a single predicted branch (cf. the
+ * IDSelectorRange dynamic_cast in IndexIVF.cpp). The scan context is only
+ * meaningful when the scan exposes a real id array (i.e. !store_pairs); when
+ * store_pairs the context path is disabled and every test falls back to
+ * is_member. */
+struct IDSelectorContextDispatch {
+    const IDSelector* sel;
+    const IDSelectorWithContext* ctx_sel;
+
+    IDSelectorContextDispatch(const IDSelector* sel, bool store_pairs)
+            : sel(sel),
+              ctx_sel((sel != nullptr && !store_pairs)
+                              ? dynamic_cast<const IDSelectorWithContext*>(sel)
+                              : nullptr) {}
+
+    bool is_member(idx_t id, const IDScanContext& ctx) const {
+        return ctx_sel ? ctx_sel->is_member_with_context(id, ctx)
+                       : sel->is_member(id);
+    }
+};
+
 /** ids between [imin, imax) */
 struct IDSelectorRange : IDSelector {
     idx_t imin, imax;

data/vendor/faiss/faiss/impl/LocalSearchQuantizer.cpp

@@ -597,8 +597,8 @@ void LocalSearchQuantizer::icm_encode_step(
         const float* binaries,
         size_t n,
         size_t n_iters) const {
-    FAISS_THROW_IF_NOT(M != 0 && K != 0);
-    FAISS_THROW_IF_NOT(binaries != nullptr);
+    FAISS_THROW_IF_MSG(M == 0 || K == 0, "M and K must be nonzero");
+    FAISS_THROW_IF_NOT(binaries);
 
     // Resolve SIMD level once, not per iteration of the n × n_iters × M loop.
     with_simd_level_256bit([&]<SIMDLevel SL>() {

data/vendor/faiss/faiss/impl/NNDescent.cpp

@@ -264,6 +264,8 @@ void NNDescent::update() {
             auto& nn_new = node.nn_new;
             auto& nn_old = node.nn_old;
 
+            // other.pool.back() below is read without other.lock: in this
+            // phase no thread reorders or resizes a pool. Lock guards rnn_*.
             for (int l = 0; l < node.M; ++l) {
                 auto& nn = node.pool[l];
                 auto& other = graph[nn.id]; // the other side of the edge
@@ -283,7 +285,6 @@ void NNDescent::update() {
                         }
                     }
                     nn.flag = false;
-
                 } else { // the node is old
                     // push the neighbor into nn_old
                     nn_old.push_back(nn.id);
@@ -300,11 +301,17 @@ void NNDescent::update() {
                     }
                 }
             }
-            // make heap to join later (in join() function)
-            std::make_heap(node.pool.begin(), node.pool.end());
         }
     }
 
+    // Restore the max-heap invariant for the next join()'s insert(). Must
+    // stay a separate region: re-heapifying during Step 3 would race with
+    // its unlocked other.pool.back() reads.
+#pragma omp parallel for
+    for (int n = 0; n < ntotal; ++n) {
+        std::make_heap(graph[n].pool.begin(), graph[n].pool.end());
+    }
+
     // Step 4.
     // Combine the forward and the reverse links
     // R = 0 means no reverse links are used.

data/vendor/faiss/faiss/impl/NSG.cpp

@@ -142,7 +142,9 @@ void NSG::build(
         idx_t n,
         const nsg::Graph<idx_t>& knn_graph,
         bool verbose) {
-    FAISS_THROW_IF_NOT(!is_built && ntotal == 0);
+    FAISS_THROW_IF_MSG(
+            is_built || ntotal != 0,
+            "NSG graph must be empty and not yet built");
 
     if (verbose) {
         printf("NSG::build R=%d, L=%d, C=%d\n", R, L, C);

data/vendor/faiss/faiss/impl/Panorama.h

@@ -49,18 +49,20 @@ static inline void compute_level_dot_kernel(
         const uint32_t* active_indices,
         const size_t num_active,
         const size_t level_width_dims,
-        float* FAISS_RESTRICT dot_products) {
+        float* FAISS_RESTRICT dot_products,
+        size_t stride = 0) {
     const size_t width = LevelWidth > 0 ? LevelWidth : level_width_dims;
+    const size_t row_stride = stride == 0 ? width : stride;
     size_t i = 0;
     for (; i + 4 <= num_active; i += 4) {
         const float* y0 = level_storage +
-                (AllActive ? (i + 0) : active_indices[i + 0]) * width;
+                (AllActive ? (i + 0) : active_indices[i + 0]) * row_stride;
         const float* y1 = level_storage +
-                (AllActive ? (i + 1) : active_indices[i + 1]) * width;
+                (AllActive ? (i + 1) : active_indices[i + 1]) * row_stride;
         const float* y2 = level_storage +
-                (AllActive ? (i + 2) : active_indices[i + 2]) * width;
+                (AllActive ? (i + 2) : active_indices[i + 2]) * row_stride;
         const float* y3 = level_storage +
-                (AllActive ? (i + 3) : active_indices[i + 3]) * width;
+                (AllActive ? (i + 3) : active_indices[i + 3]) * row_stride;
 
         float dp0 = 0, dp1 = 0, dp2 = 0, dp3 = 0;
         FAISS_PRAGMA_IMPRECISE_LOOP
@@ -78,8 +80,8 @@ static inline void compute_level_dot_kernel(
         dot_products[i + 3] = dp3;
     }
     for (; i < num_active; i++) {
-        const float* yj =
-                level_storage + (AllActive ? i : active_indices[i]) * width;
+        const float* yj = level_storage +
+                (AllActive ? i : active_indices[i]) * row_stride;
         float dp = 0;
         FAISS_PRAGMA_IMPRECISE_LOOP
         for (size_t j = 0; j < width; j++) {
@@ -233,7 +235,8 @@ inline auto with_bool(bool value, Lambda&& fn) {
  * accelerating the refinement stage.
  */
 struct Panorama {
-    static constexpr size_t kDefaultBatchSize = 128;
+    static constexpr size_t kDefaultBatchSize = 1024;
+    static constexpr size_t kLegacyBatchSize = 128;
 
     size_t d = 0;
     size_t code_size = 0;
@@ -320,12 +323,20 @@ struct Panorama {
         size_t batch_offset = batch_no * batch_size * code_size;
         const uint8_t* storage_base = codes_base + batch_offset;
 
+        // Honor IDSelectorWithContext: the scan-order position within the whole
+        // list is global_idx, so a lookahead crosses batch boundaries correctly
+        // (ids is the full list, length list_size).
+        IDSelectorContextDispatch sel_dispatch(
+                sel, /*store_pairs=*/ids == nullptr);
+
         // Initialize active set with ID-filtered vectors.
         size_t num_active = 0;
         for (size_t i = 0; i < curr_batch_size; i++) {
             size_t global_idx = batch_start + i;
             idx_t id = (ids == nullptr) ? global_idx : ids[global_idx];
-            bool include = !use_sel || sel->is_member(id);
+            bool include = !use_sel ||
+                    sel_dispatch.is_member(
+                            id, IDScanContext{ids, list_size, global_idx});
 
             active_indices[num_active] = i;
             float cum_sum = batch_cum_sums[i];

data/vendor/faiss/faiss/impl/PolysemousTraining.cpp

@@ -26,6 +26,8 @@
 
 #include <faiss/impl/FaissAssert.h>
 
+#include <faiss/impl/polysemous_training/dispatch.h>
+
 /*****************************************
  * Mixed PQ / Hamming
  ******************************************/
@@ -168,6 +170,94 @@ static inline int hamming_dis(uint64_t a, uint64_t b) {
     return popcount64(a ^ b);
 }
 
+static inline double sqr(double x) {
+    return x * x;
+}
+
+// Scalar (NONE) kernels for the two objectives. The dispatch boundary lives in
+// the objective methods below, which route to these or to the AVX-512
+// specializations (in polysemous_training/avx512.cpp) via
+// with_selected_simd_levels.
+namespace polysemous_training {
+
+template <>
+double hamming_compute_cost<SIMDLevel::NONE>(
+        int n,
+        const int* perm,
+        const double* target_dis,
+        const double* weights) {
+    double cost = 0;
+    for (int i = 0; i < n; i++) {
+        for (int j = 0; j < n; j++) {
+            double wanted = target_dis[i * n + j];
+            double w = weights[i * n + j];
+            double actual = hamming_dis(perm[i], perm[j]);
+            cost += w * sqr(wanted - actual);
+        }
+    }
+    return cost;
+}
+
+template <>
+double hamming_cost_update<SIMDLevel::NONE>(
+        int n,
+        const int* perm,
+        int iw,
+        int jw,
+        const double* target_dis,
+        const double* weights) {
+    double delta_cost = 0;
+
+    for (int i = 0; i < n; i++) {
+        if (i == iw) {
+            for (int j = 0; j < n; j++) {
+                double wanted = target_dis[i * n + j], w = weights[i * n + j];
+                double actual = hamming_dis(perm[i], perm[j]);
+                delta_cost -= w * sqr(wanted - actual);
+                double new_actual = hamming_dis(
+                        perm[jw],
+                        perm[j == iw           ? jw
+                                     : j == jw ? iw
+                                               : j]);
+                delta_cost += w * sqr(wanted - new_actual);
+            }
+        } else if (i == jw) {
+            for (int j = 0; j < n; j++) {
+                double wanted = target_dis[i * n + j], w = weights[i * n + j];
+                double actual = hamming_dis(perm[i], perm[j]);
+                delta_cost -= w * sqr(wanted - actual);
+                double new_actual = hamming_dis(
+                        perm[iw],
+                        perm[j == iw           ? jw
+                                     : j == jw ? iw
+                                               : j]);
+                delta_cost += w * sqr(wanted - new_actual);
+            }
+        } else {
+            int j = iw;
+            {
+                double wanted = target_dis[i * n + j], w = weights[i * n + j];
+                double actual = hamming_dis(perm[i], perm[j]);
+                delta_cost -= w * sqr(wanted - actual);
+                double new_actual = hamming_dis(perm[i], perm[jw]);
+                delta_cost += w * sqr(wanted - new_actual);
+            }
+            j = jw;
+            {
+                double wanted = target_dis[i * n + j], w = weights[i * n + j];
+                double actual = hamming_dis(perm[i], perm[j]);
+                delta_cost -= w * sqr(wanted - actual);
+                double new_actual = hamming_dis(perm[i], perm[iw]);
+                delta_cost += w * sqr(wanted - new_actual);
+            }
+        }
+    }
+
+    return delta_cost;
+}
+
+} // namespace polysemous_training
+
 namespace {
 
 /// optimize permutation to reproduce a distance table with Hamming distances
@@ -175,10 +265,6 @@ struct ReproduceWithHammingObjective : PermutationObjective {
     int nbits;
     double dis_weight_factor;
 
-    static double sqr(double x) {
-        return x * x;
-    }
-
     // weighting of distances: it is more important to reproduce small
     // distances well
     double dis_weight(double x) const {
@@ -190,73 +276,21 @@ struct ReproduceWithHammingObjective : PermutationObjective {
 
     // cost = quadratic difference between actual distance and Hamming distance
     double compute_cost(const int* perm) const override {
-        double cost = 0;
-        for (int i = 0; i < n; i++) {
-            for (int j = 0; j < n; j++) {
-                double wanted = target_dis[i * n + j];
-                double w = weights[i * n + j];
-                double actual = hamming_dis(perm[i], perm[j]);
-                cost += w * sqr(wanted - actual);
-            }
-        }
-        return cost;
+        return with_selected_simd_levels<polysemous_training::SIMD_LEVELS>(
+                [&]<SIMDLevel SL>() {
+                    return polysemous_training::hamming_compute_cost<SL>(
+                            n, perm, target_dis.data(), weights.data());
+                });
     }
 
     // what would the cost update be if iw and jw were swapped?
     // computed in O(n) instead of O(n^2) for the full re-computation
     double cost_update(const int* perm, int iw, int jw) const override {
-        double delta_cost = 0;
-
-        for (int i = 0; i < n; i++) {
-            if (i == iw) {
-                for (int j = 0; j < n; j++) {
-                    double wanted = target_dis[i * n + j],
-                           w = weights[i * n + j];
-                    double actual = hamming_dis(perm[i], perm[j]);
-                    delta_cost -= w * sqr(wanted - actual);
-                    double new_actual = hamming_dis(
-                            perm[jw],
-                            perm[j == iw           ? jw
-                                         : j == jw ? iw
-                                                   : j]);
-                    delta_cost += w * sqr(wanted - new_actual);
-                }
-            } else if (i == jw) {
-                for (int j = 0; j < n; j++) {
-                    double wanted = target_dis[i * n + j],
-                           w = weights[i * n + j];
-                    double actual = hamming_dis(perm[i], perm[j]);
-                    delta_cost -= w * sqr(wanted - actual);
-                    double new_actual = hamming_dis(
-                            perm[iw],
-                            perm[j == iw           ? jw
-                                         : j == jw ? iw
-                                                   : j]);
-                    delta_cost += w * sqr(wanted - new_actual);
-                }
-            } else {
-                int j = iw;
-                {
-                    double wanted = target_dis[i * n + j],
-                           w = weights[i * n + j];
-                    double actual = hamming_dis(perm[i], perm[j]);
-                    delta_cost -= w * sqr(wanted - actual);
-                    double new_actual = hamming_dis(perm[i], perm[jw]);
-                    delta_cost += w * sqr(wanted - new_actual);
-                }
-                j = jw;
-                {
-                    double wanted = target_dis[i * n + j],
-                           w = weights[i * n + j];
-                    double actual = hamming_dis(perm[i], perm[j]);
-                    delta_cost -= w * sqr(wanted - actual);
-                    double new_actual = hamming_dis(perm[i], perm[iw]);
-                    delta_cost += w * sqr(wanted - new_actual);
-                }
-            }
-        }
-
-        return delta_cost;
+        return with_selected_simd_levels<polysemous_training::SIMD_LEVELS>(
+                [&]<SIMDLevel SL>() {
+                    return polysemous_training::hamming_cost_update<SL>(
+                            n, perm, iw, jw, target_dis.data(), weights.data());
+                });
     }
 
     ReproduceWithHammingObjective(
@@ -306,14 +340,20 @@ double ReproduceDistancesObjective::get_source_dis(int i, int j) const {
     return source_dis[i * n + j];
 }
 
+namespace polysemous_training {
+
 // cost = quadratic difference between actual distance and Hamming distance
-double ReproduceDistancesObjective::compute_cost(const int* perm) const {
+template <>
+double distances_compute_cost<SIMDLevel::NONE>(
+        const ReproduceDistancesObjective& obj,
+        const int* perm) {
+    const int n = obj.n;
     double cost = 0;
     for (int i = 0; i < n; i++) {
         for (int j = 0; j < n; j++) {
-            double wanted = target_dis[i * n + j];
-            double w = weights[i * n + j];
-            double actual = get_source_dis(perm[i], perm[j]);
+            double wanted = obj.target_dis[i * n + j];
+            double w = obj.weights[i * n + j];
+            double actual = obj.get_source_dis(perm[i], perm[j]);
             cost += w * sqr(wanted - actual);
         }
     }
@@ -322,16 +362,22 @@ double ReproduceDistancesObjective::compute_cost(const int* perm) const {
 
 // what would the cost update be if iw and jw were swapped?
 // computed in O(n) instead of O(n^2) for the full re-computation
-double ReproduceDistancesObjective::cost_update(const int* perm, int iw, int jw)
-        const {
+template <>
+double distances_cost_update<SIMDLevel::NONE>(
+        const ReproduceDistancesObjective& obj,
+        const int* perm,
+        int iw,
+        int jw) {
+    const int n = obj.n;
     double delta_cost = 0;
     for (int i = 0; i < n; i++) {
         if (i == iw) {
             for (int j = 0; j < n; j++) {
-                double wanted = target_dis[i * n + j], w = weights[i * n + j];
-                double actual = get_source_dis(perm[i], perm[j]);
+                double wanted = obj.target_dis[i * n + j],
+                       w = obj.weights[i * n + j];
+                double actual = obj.get_source_dis(perm[i], perm[j]);
                 delta_cost -= w * sqr(wanted - actual);
-                double new_actual = get_source_dis(
+                double new_actual = obj.get_source_dis(
                         perm[jw],
                         perm[j == iw           ? jw
                                      : j == jw ? iw
@@ -340,10 +386,11 @@ double ReproduceDistancesObjective::cost_update(const int* perm, int iw, int jw)
             }
         } else if (i == jw) {
             for (int j = 0; j < n; j++) {
-                double wanted = target_dis[i * n + j], w = weights[i * n + j];
-                double actual = get_source_dis(perm[i], perm[j]);
+                double wanted = obj.target_dis[i * n + j],
+                       w = obj.weights[i * n + j];
+                double actual = obj.get_source_dis(perm[i], perm[j]);
                 delta_cost -= w * sqr(wanted - actual);
-                double new_actual = get_source_dis(
+                double new_actual = obj.get_source_dis(
                         perm[iw],
                         perm[j == iw           ? jw
                                      : j == jw ? iw
@@ -353,18 +400,20 @@ double ReproduceDistancesObjective::cost_update(const int* perm, int iw, int jw)
         } else {
             int j = iw;
             {
-                double wanted = target_dis[i * n + j], w = weights[i * n + j];
-                double actual = get_source_dis(perm[i], perm[j]);
+                double wanted = obj.target_dis[i * n + j],
+                       w = obj.weights[i * n + j];
+                double actual = obj.get_source_dis(perm[i], perm[j]);
                 delta_cost -= w * sqr(wanted - actual);
-                double new_actual = get_source_dis(perm[i], perm[jw]);
+                double new_actual = obj.get_source_dis(perm[i], perm[jw]);
                 delta_cost += w * sqr(wanted - new_actual);
             }
             j = jw;
             {
-                double wanted = target_dis[i * n + j], w = weights[i * n + j];
-                double actual = get_source_dis(perm[i], perm[j]);
+                double wanted = obj.target_dis[i * n + j],
+                       w = obj.weights[i * n + j];
+                double actual = obj.get_source_dis(perm[i], perm[j]);
                 delta_cost -= w * sqr(wanted - actual);
-                double new_actual = get_source_dis(perm[i], perm[iw]);
+                double new_actual = obj.get_source_dis(perm[i], perm[iw]);
                 delta_cost += w * sqr(wanted - new_actual);
             }
         }
@@ -372,6 +421,25 @@ double ReproduceDistancesObjective::cost_update(const int* perm, int iw, int jw)
     return delta_cost;
 }
 
+} // namespace polysemous_training
+
+double ReproduceDistancesObjective::compute_cost(const int* perm) const {
+    return with_selected_simd_levels<polysemous_training::SIMD_LEVELS>(
+            [&]<SIMDLevel SL>() {
+                return polysemous_training::distances_compute_cost<SL>(
+                        *this, perm);
+            });
+}
+
+double ReproduceDistancesObjective::cost_update(const int* perm, int iw, int jw)
+        const {
+    return with_selected_simd_levels<polysemous_training::SIMD_LEVELS>(
+            [&]<SIMDLevel SL>() {
+                return polysemous_training::distances_cost_update<SL>(
+                        *this, perm, iw, jw);
+            });
+}
+
 ReproduceDistancesObjective::ReproduceDistancesObjective(
         int n_in,
         const double* source_dis_in,