faiss 0.6.2 → 0.6.4

data/vendor/faiss/faiss/impl/binary_hamming/IndexBinaryIVF_impl.h

@@ -39,17 +39,50 @@ namespace faiss {
 
 namespace {
 
+// Whether the computer can measure a whole batch of codes in one call.
 template <class HammingComputer>
-struct IVFBinaryScannerL2 : BinaryInvertedListScanner {
+constexpr bool has_hamming_batch =
+        requires(const uint8_t* tile, const uint8_t* codes, int32_t* dis) {
+            HammingComputer::batch_size;
+            HammingComputer::get_code_size();
+            HammingComputer::build_batch_query(tile, nullptr);
+            HammingComputer::hamming_batch(tile, codes, dis);
+        };
+
+// The query repeated batch_size times, which is what hamming_batch() XORs
+// against. Empty for a computer that does not batch, so those scanners carry
+// neither the buffer nor its alignment.
+template <class HammingComputer>
+struct BatchQueryTile {};
+
+template <class HammingComputer>
+    requires has_hamming_batch<HammingComputer>
+struct BatchQueryTile<HammingComputer> {
+    alignas(64) uint8_t batch_query
+            [HammingComputer::batch_size * HammingComputer::get_code_size()];
+};
+
+template <class HammingComputer>
+struct IVFBinaryScannerL2 : BinaryInvertedListScanner,
+                            BatchQueryTile<HammingComputer> {
     HammingComputer hc;
     size_t code_size;
     bool store_pairs;
 
     IVFBinaryScannerL2(size_t code_size_, bool store_pairs_)
-            : code_size(code_size_), store_pairs(store_pairs_) {}
+            : code_size(code_size_), store_pairs(store_pairs_) {
+        if constexpr (has_hamming_batch<HammingComputer>) {
+            // The batch kernel reads a fixed stride, so a caller that pairs
+            // this computer with another code size would read past the codes.
+            FAISS_THROW_IF_NOT(code_size == HammingComputer::get_code_size());
+        }
+    }
 
     void set_query(const uint8_t* query_vector) override {
         hc.set(query_vector, code_size);
+        if constexpr (has_hamming_batch<HammingComputer>) {
+            HammingComputer::build_batch_query(query_vector, this->batch_query);
+        }
     }
 
     idx_t list_no = 0;
@@ -61,6 +94,31 @@ struct IVFBinaryScannerL2 : BinaryInvertedListScanner {
         return hc.hamming(code);
     }
 
+    // Measures whole batches while at least batch_size codes remain, then
+    // leaves codes and j on the first code the caller must measure singly.
+    // bound is read per lane, so a caller that raises its heap top inside
+    // accept prunes the rest of the batch.
+    template <class Accept>
+    void scan_batch_prefix(
+            size_t n,
+            const uint8_t* __restrict& codes,
+            size_t& j,
+            const uint32_t& bound,
+            Accept&& accept) const {
+        if constexpr (has_hamming_batch<HammingComputer>) {
+            constexpr size_t B = HammingComputer::batch_size;
+            int32_t batch[B];
+            for (; j + B <= n; j += B, codes += B * code_size) {
+                HammingComputer::hamming_batch(this->batch_query, codes, batch);
+                for (size_t t = 0; t < B; t++) {
+                    if (static_cast<uint32_t>(batch[t]) < bound) {
+                        accept(batch[t], j + t);
+                    }
+                }
+            }
+        }
+    }
+
     size_t scan_codes(
             size_t n,
             const uint8_t* __restrict codes,
@@ -70,15 +128,25 @@ struct IVFBinaryScannerL2 : BinaryInvertedListScanner {
             size_t k) const override {
         using C = CMax<int32_t, idx_t>;
 
+        uint32_t bound = static_cast<uint32_t>(simi[0]);
+
         size_t nup = 0;
-        for (size_t j = 0; j < n; j++) {
+        size_t j = 0;
+
+        auto accept = [&](int32_t dis, size_t at) {
+            idx_t id = store_pairs ? lo_build(list_no, at) : ids[at];
+            heap_replace_top<C>(k, simi, idxi, dis, id);
+            bound = static_cast<uint32_t>(simi[0]);
+            nup++;
+        };
+
+        scan_batch_prefix(n, codes, j, bound, accept);
+
+        for (; j < n; j++, codes += code_size) {
             uint32_t dis = hc.hamming(codes);
-            if (dis < static_cast<uint32_t>(simi[0])) {
-                idx_t id = store_pairs ? lo_build(list_no, j) : ids[j];
-                heap_replace_top<C>(k, simi, idxi, dis, id);
-                nup++;
+            if (dis < bound) {
+                accept(static_cast<int32_t>(dis), j);
             }
-            codes += code_size;
         }
         return nup;
     }
@@ -89,13 +157,21 @@ struct IVFBinaryScannerL2 : BinaryInvertedListScanner {
             const idx_t* __restrict ids,
             int radius,
             RangeQueryResult& result) const override {
-        for (size_t j = 0; j < n; j++) {
+        const uint32_t bound = static_cast<uint32_t>(radius);
+        size_t j = 0;
+
+        auto accept = [&](int32_t dis, size_t at) {
+            int64_t id = store_pairs ? lo_build(list_no, at) : ids[at];
+            result.add(static_cast<uint32_t>(dis), id);
+        };
+
+        scan_batch_prefix(n, codes, j, bound, accept);
+
+        for (; j < n; j++, codes += code_size) {
             uint32_t dis = hc.hamming(codes);
-            if (dis < static_cast<uint32_t>(radius)) {
-                int64_t id = store_pairs ? lo_build(list_no, j) : ids[j];
-                result.add(dis, id);
+            if (dis < bound) {
+                accept(static_cast<int32_t>(dis), j);
             }
-            codes += code_size;
         }
     }
 };
@@ -292,7 +368,7 @@ void search_knn_hamming_per_invlist(
     nprobe = std::min((idx_t)ivf->nlist, nprobe);
     idx_t max_codes = params ? params->max_codes : ivf->max_codes;
     FAISS_THROW_IF_NOT(max_codes == 0);
-    FAISS_THROW_IF_NOT(!store_pairs);
+    FAISS_THROW_IF_MSG(store_pairs, "store_pairs is not supported here");
 
     // reorder buckets
     std::vector<int64_t> lims(n + 1);

data/vendor/faiss/faiss/impl/binary_hamming/avx2.cpp

@@ -9,17 +9,17 @@
 
 #define THE_SIMD_LEVEL SIMDLevel::AVX2
 
-// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+// NOLINTNEXTLINE(facebook-hte-InlineHeader,facebook-unused-include-check)
 #include <faiss/utils/hamming_distance/hamming_computer-avx2.h>
 
 // NOLINTNEXTLINE(facebook-hte-InlineHeader)
-#include <faiss/impl/binary_hamming/IndexBinaryHNSW_impl.h>
+#include <faiss/impl/binary_hamming/IndexBinaryHNSW_impl.h> // IWYU pragma: keep
 // NOLINTNEXTLINE(facebook-hte-InlineHeader)
-#include <faiss/impl/binary_hamming/IndexBinaryHash_impl.h>
+#include <faiss/impl/binary_hamming/IndexBinaryHash_impl.h> // IWYU pragma: keep
 // NOLINTNEXTLINE(facebook-hte-InlineHeader)
 #include <faiss/impl/binary_hamming/IndexBinaryIVF_impl.h>
 // NOLINTNEXTLINE(facebook-hte-InlineHeader)
-#include <faiss/impl/binary_hamming/IndexIVFSpectralHash_impl.h>
+#include <faiss/impl/binary_hamming/IndexIVFSpectralHash_impl.h> // IWYU pragma: keep
 // NOLINTNEXTLINE(facebook-hte-InlineHeader)
 #include <faiss/impl/binary_hamming/IndexPQ_impl.h>
 

data/vendor/faiss/faiss/impl/expanded_scanners.h

@@ -35,12 +35,16 @@ size_t run_scan_codes1(
     size_t list_no = scanner.list_no;
     size_t code_size = scanner.code_size;
     const IDSelector* sel = scanner.sel;
+    // If the selector implements IDSelectorWithContext, hand it the scan
+    // context (ids, list_size, j) so it can exploit scan-order locality; the
+    // dispatch caches the once-per-list RTTI detection.
+    const IDSelectorContextDispatch sel_dispatch(sel, store_pairs);
     float threshold = handler.threshold;
     for (size_t j = 0; j < list_size; j++) {
         if (use_sel) {
             int64_t id = store_pairs ? lo_build(list_no, j) : ids[j];
             // skip code without computing distance
-            if (!sel->is_member(id)) {
+            if (!sel_dispatch.is_member(id, IDScanContext{ids, list_size, j})) {
                 codes += code_size;
                 continue;
             }

data/vendor/faiss/faiss/impl/fast_scan/decompose_qbs.h

@@ -40,6 +40,7 @@ void kernel_accumulate_block(
 #ifdef __AVX512F__
     if constexpr (
             KernelSL == SIMDLevel::AVX512 ||
+            KernelSL == SIMDLevel::AVX512_VPOPCNT ||
             KernelSL == SIMDLevel::AVX512_SPR) {
         pq4_kernel_qbs_512<NQ>(nsq, codes, LUT, res, scaler);
     } else {

data/vendor/faiss/faiss/impl/fast_scan/dispatching.h

@@ -33,6 +33,7 @@
 
 #include <faiss/impl/fast_scan/accumulate_loops.h>
 #include <faiss/impl/fast_scan/fast_scan.h>
+#include <faiss/utils/simd_levels.h>
 
 #if defined(COMPILE_SIMD_AVX512) && defined(__AVX512F__)
 #include <faiss/impl/fast_scan/accumulate_loops_512.h>
@@ -114,8 +115,40 @@ struct ScannerMixIn : FastScanCodeScanner {
         constexpr bool use_avx512_qbs = false;
 #endif
         if constexpr (use_avx512_qbs) {
-            // Use 512-bit QBS kernels with properly-leveled scalers.
-            if (pq2x4_scale) {
+            // AMD Zen 4 / Zen 4c ("Bergamo", family 0x19) split 512-bit
+            // ops over a 256-bit datapath, so the 512-bit QBS kernel yields no
+            // throughput gain but pays extra per-block LUT-assembly and
+            // cross-lane reduction overhead (measured ~14% search regression
+            // for PQ8x4fs / PQ16x4fs). Route those CPUs to the 256-bit (AVX2)
+            // QBS kernel instead -- same output, no downside on Zen 4. This is
+            // a process-constant runtime branch, hoisted out of the inner
+            // accumulate loop. Intel AVX-512 keeps the 512-bit kernel.
+            if (SIMDConfig::avx512_split) {
+                if (pq2x4_scale) {
+                    NormTableScaler<SIMDLevel::AVX2> scaler(pq2x4_scale);
+                    pq4_accumulate_loop_qbs_fixed_scaler_256<SIMDLevel::AVX2>(
+                            qbs,
+                            nb,
+                            nsq,
+                            codes,
+                            LUT,
+                            handler_,
+                            scaler,
+                            block_stride);
+                } else {
+                    DummyScaler<SIMDLevel::AVX2> dummy;
+                    pq4_accumulate_loop_qbs_fixed_scaler_256<SIMDLevel::AVX2>(
+                            qbs,
+                            nb,
+                            nsq,
+                            codes,
+                            LUT,
+                            handler_,
+                            dummy,
+                            block_stride);
+                }
+            } else if (pq2x4_scale) {
+                // Use 512-bit QBS kernels with properly-leveled scalers.
                 NormTableScaler<THE_LEVEL_TO_DISPATCH> scaler(pq2x4_scale);
                 pq4_accumulate_loop_qbs_fixed_scaler_512(
                         qbs,

data/vendor/faiss/faiss/impl/hnsw/LockVector.cpp

@@ -30,7 +30,7 @@ void LockVector::prepare(size_t new_size) {
         // Just destroy old and init fresh; omp_lock_t is not copyable.
         clear();
         data_ = static_cast<omp_lock_t*>(malloc(new_cap * sizeof(omp_lock_t)));
-        FAISS_THROW_IF_NOT(data_ != nullptr);
+        FAISS_THROW_IF_NOT(data_);
         capacity_ = new_cap;
     }
     for (size_t i = size_; i < new_size; i++) {