faiss 0.6.0 → 0.6.1

data/vendor/faiss/faiss/impl/binary_hamming/avx2.cpp

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#ifdef COMPILE_SIMD_AVX2
+
+#define THE_SIMD_LEVEL SIMDLevel::AVX2
+
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/utils/hamming_distance/hamming_computer-avx2.h>
+
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexBinaryHNSW_impl.h>
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexBinaryHash_impl.h>
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexBinaryIVF_impl.h>
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexIVFSpectralHash_impl.h>
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexPQ_impl.h>
+
+#endif // COMPILE_SIMD_AVX2

data/vendor/faiss/faiss/impl/binary_hamming/avx512.cpp

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#ifdef COMPILE_SIMD_AVX512
+
+#define THE_SIMD_LEVEL SIMDLevel::AVX512
+
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/utils/hamming_distance/hamming_computer-avx512.h>
+
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexBinaryHNSW_impl.h>
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexBinaryHash_impl.h>
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexBinaryIVF_impl.h>
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexIVFSpectralHash_impl.h>
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexPQ_impl.h>
+
+#endif // COMPILE_SIMD_AVX512

data/vendor/faiss/faiss/impl/binary_hamming/dispatch.h

@@ -0,0 +1,143 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <faiss/utils/simd_levels.h>
+
+#include <cstddef>
+#include <cstdint>
+
+namespace faiss {
+
+// Forward declarations
+struct DistanceComputer;
+struct InvertedListScanner;
+struct BinaryInvertedListScanner;
+struct IndexBinaryFlat;
+struct IndexBinaryHash;
+struct IndexBinaryMultiHash;
+struct IndexBinaryIVF;
+struct IndexIVFSpectralHash;
+struct IndexPQ;
+struct SearchParametersIVF;
+using IVFSearchParameters = SearchParametersIVF;
+struct RangeQueryResult;
+using idx_t = int64_t;
+
+/** @name IndexBinaryHNSW dispatch
+ * @{ */
+template <SIMDLevel SL>
+DistanceComputer* make_binary_hnsw_distance_computer_fixSL(
+        int code_size,
+        IndexBinaryFlat* flat_storage);
+/** @} */
+
+/** @name IndexBinaryIVF dispatch
+ * @{ */
+template <SIMDLevel SL>
+BinaryInvertedListScanner* make_binary_ivf_scanner_fixSL(
+        size_t code_size,
+        bool store_pairs);
+
+template <SIMDLevel SL>
+void search_knn_hamming_per_invlist_fixSL(
+        int code_size,
+        const IndexBinaryIVF* ivf,
+        size_t n,
+        const uint8_t* x,
+        idx_t k,
+        const idx_t* keys_in,
+        const int32_t* coarse_dis,
+        int32_t* distances,
+        idx_t* labels,
+        bool store_pairs,
+        const IVFSearchParameters* params);
+
+template <SIMDLevel SL>
+void search_knn_hamming_count_fixSL(
+        int code_size,
+        bool store_pairs,
+        const IndexBinaryIVF* ivf,
+        size_t nx,
+        const uint8_t* x,
+        const idx_t* keys,
+        int k,
+        int32_t* distances,
+        idx_t* labels,
+        const IVFSearchParameters* params);
+/** @} */
+
+/** @name IndexBinaryHash dispatch
+ * @{ */
+template <SIMDLevel SL>
+void binary_hash_knn_search_fixSL(
+        const IndexBinaryHash& index,
+        const uint8_t* q,
+        idx_t k,
+        int32_t* heap_sim,
+        idx_t* heap_ids,
+        size_t& n0,
+        size_t& nlist,
+        size_t& ndis);
+
+template <SIMDLevel SL>
+void binary_hash_range_search_fixSL(
+        const IndexBinaryHash& index,
+        const uint8_t* q,
+        int radius,
+        RangeQueryResult& qres,
+        size_t& n0,
+        size_t& nlist,
+        size_t& ndis);
+
+template <SIMDLevel SL>
+void binary_multihash_knn_search_fixSL(
+        const IndexBinaryMultiHash& index,
+        const uint8_t* q,
+        idx_t k,
+        int32_t* heap_sim,
+        idx_t* heap_ids,
+        size_t& n0,
+        size_t& nlist,
+        size_t& ndis);
+
+template <SIMDLevel SL>
+void binary_multihash_range_search_fixSL(
+        const IndexBinaryMultiHash& index,
+        const uint8_t* q,
+        int radius,
+        RangeQueryResult& qres,
+        size_t& n0,
+        size_t& nlist,
+        size_t& ndis);
+/** @} */
+
+/** @name IndexIVFSpectralHash dispatch
+ * @{ */
+template <SIMDLevel SL>
+InvertedListScanner* make_spectral_hash_scanner_fixSL(
+        int code_size,
+        const IndexIVFSpectralHash* index,
+        bool store_pairs);
+/** @} */
+
+/** @name IndexPQ polysemous dispatch
+ * @{ */
+template <SIMDLevel SL>
+size_t polysemous_inner_loop_fixSL(
+        int code_size,
+        const IndexPQ* index,
+        const float* dis_table_qi,
+        const uint8_t* q_code,
+        size_t k,
+        float* heap_dis,
+        int64_t* heap_ids,
+        int ht);
+/** @} */
+
+} // namespace faiss

data/vendor/faiss/faiss/impl/binary_hamming/neon.cpp

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#ifdef COMPILE_SIMD_ARM_NEON
+
+#define THE_SIMD_LEVEL SIMDLevel::ARM_NEON
+
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/utils/hamming_distance/hamming_computer-neon.h>
+
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexBinaryHNSW_impl.h>
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexBinaryHash_impl.h>
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexBinaryIVF_impl.h>
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexIVFSpectralHash_impl.h>
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexPQ_impl.h>
+
+#endif // COMPILE_SIMD_ARM_NEON

data/vendor/faiss/faiss/impl/binary_hamming/rvv.cpp

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#ifdef COMPILE_SIMD_RISCV_RVV
+
+#define THE_SIMD_LEVEL SIMDLevel::RISCV_RVV
+
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/utils/hamming_distance/hamming_computer-rvv.h>
+
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexBinaryHNSW_impl.h>
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexBinaryHash_impl.h>
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexBinaryIVF_impl.h>
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexIVFSpectralHash_impl.h>
+// NOLINTNEXTLINE(facebook-hte-InlineHeader)
+#include <faiss/impl/binary_hamming/IndexPQ_impl.h>
+
+#endif // COMPILE_SIMD_RISCV_RVV

data/vendor/faiss/faiss/impl/expanded_scanners.h

@@ -10,6 +10,7 @@
 #include <cstdio>
 
 #include <faiss/IndexIVF.h>
+#include <faiss/impl/InvertedListScannerStats.h>
 #include <faiss/impl/ResultHandler.h>
 
 /* This is the inner loop of the inverted list scanners. The default version
@@ -45,12 +46,16 @@ size_t run_scan_codes1(
             }
         }
 
+        // post-IDSelector: distance is about to be computed for this code.
+        handler.stats.scan_cnt++;
         float dis = scanner.distance_to_code(codes); // will be inlined if final
         if (C::cmp(threshold, dis)) {
             int64_t id = store_pairs ? lo_build(list_no, j) : ids[j];
-            handler.add_result(dis, id);
-            threshold = handler.threshold;
-            nup++;
+            if (handler.add_result(dis, id)) {
+                handler.stats.nheap_updates++;
+                nup++;
+                threshold = handler.threshold;
+            }
         }
         codes += code_size;
     }

data/vendor/faiss/faiss/impl/{FastScanDistancePostProcessing.h → fast_scan/FastScanDistancePostProcessing.h}

@@ -11,9 +11,6 @@
 
 namespace faiss {
 
-// Forward declarations
-struct NormTableScaler;
-
 namespace rabitq_utils {
 struct QueryFactorsData;
 }
@@ -22,8 +19,10 @@ struct QueryFactorsData;
  * Simple context object that holds processors for FastScan operations.
  * */
 struct FastScanDistancePostProcessing {
-    /// Norm scaling processor for Additive Quantizers (nullptr if not needed)
-    const NormTableScaler* norm_scaler = nullptr;
+    /// Norm scaling processor for Additive Quantizers.
+    /// The scale is encoded in a 2x4 bit PQ table, then scaled by this int.
+    /// Set to 0 if unused.
+    int pq2x4_scale = 0;
 
     /// Query factors data pointer for RaBitQ (nullptr if not needed)
     /// This pointer should point to the beginning of the relevant
@@ -36,12 +35,20 @@ struct FastScanDistancePostProcessing {
     /// Set to 0 to use index->nprobe as fallback.
     size_t nprobe = 0;
 
+    /// RaBitQ query quantization bits override.
+    /// Set to 0 to use the index default (index->qb).
+    uint8_t qb = 0;
+
+    /// RaBitQ centered scalar quantizer override.
+    /// Only used when qb > 0 (i.e., when params are overridden).
+    bool centered = false;
+
     /// Default constructor - no processing
     FastScanDistancePostProcessing() = default;
 
     /// Check if norm scaling is enabled
     bool has_norm_scaling() const {
-        return norm_scaler != nullptr;
+        return pq2x4_scale != 0;
     }
 
     /// Check if query factors processing is enabled

data/vendor/faiss/faiss/impl/{LookupTableScaler.h → fast_scan/LookupTableScaler.h}

@@ -10,7 +10,7 @@
 #include <cstdint>
 #include <cstdlib>
 
-#include <faiss/utils/simdlib.h>
+#include <faiss/impl/simdlib/simdlib_dispatch.h>
 
 /*******************************************
  * The Scaler objects are used to specialize the handling of the
@@ -20,8 +20,15 @@
 namespace faiss {
 
 /// no-op handler
+template <SIMDLevel SL = SINGLE_SIMD_LEVEL>
 struct DummyScaler {
     static constexpr int nscale = 0;
+    static constexpr SIMDLevel SL256 = simd256_level_selector<SL>::value;
+    static constexpr SIMDLevel SL512 = simd512_level_selector<SL>::value;
+    using simd32uint8 = simd32uint8_tpl<SL256>;
+    using simd16uint16 = simd16uint16_tpl<SL256>;
+    using simd64uint8 = simd64uint8_tpl<SL512>;
+    using simd32uint16 = simd32uint16_tpl<SL512>;
 
     inline simd32uint8 lookup(const simd32uint8&, const simd32uint8&) const {
         FAISS_THROW_MSG("DummyScaler::lookup should not be called.");
@@ -38,7 +45,6 @@ struct DummyScaler {
         return simd16uint16(0);
     }
 
-#ifdef __AVX512F__
     inline simd64uint8 lookup(const simd64uint8&, const simd64uint8&) const {
         FAISS_THROW_MSG("DummyScaler::lookup should not be called.");
         return simd64uint8(0);
@@ -53,7 +59,6 @@ struct DummyScaler {
         FAISS_THROW_MSG("DummyScaler::scale_hi should not be called.");
         return simd32uint16(0);
     }
-#endif
 
     template <class dist_t>
     inline dist_t scale_one(const dist_t&) const {
@@ -64,8 +69,16 @@ struct DummyScaler {
 
 /// consumes 2x4 bits to encode a norm as a scalar additive quantizer
 /// the norm is scaled because its range is larger than other components
+template <SIMDLevel SL = SINGLE_SIMD_LEVEL>
 struct NormTableScaler {
     static constexpr int nscale = 2;
+    static constexpr SIMDLevel SL256 = simd256_level_selector<SL>::value;
+    static constexpr SIMDLevel SL512 = simd512_level_selector<SL>::value;
+    using simd32uint8 = simd32uint8_tpl<SL256>;
+    using simd16uint16 = simd16uint16_tpl<SL256>;
+    using simd64uint8 = simd64uint8_tpl<SL512>;
+    using simd32uint16 = simd32uint16_tpl<SL512>;
+
     int scale_int;
     simd16uint16 scale_simd;
 
@@ -84,7 +97,6 @@ struct NormTableScaler {
         return (simd16uint16(res) >> 8) * scale_simd;
     }
 
-#ifdef __AVX512F__
     inline simd64uint8 lookup(const simd64uint8& lut, const simd64uint8& c)
             const {
         return lut.lookup_4_lanes(c);
@@ -99,7 +111,6 @@ struct NormTableScaler {
         auto scale_simd_wide = simd32uint16(scale_simd, scale_simd);
         return (simd32uint16(res) >> 8) * scale_simd_wide;
     }
-#endif
 
     // for non-SIMD implem 2, 3, 4
     template <class dist_t>

data/vendor/faiss/faiss/impl/fast_scan/accumulate_loops.h

@@ -0,0 +1,237 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+/**
+ * @file accumulate_loops.h
+ * @brief Shared accumulation loop helpers for fast-scan search paths.
+ *
+ * Contains:
+ *   - accumulate_fixed_blocks / pq4_accumulate_loop_fixed_scaler
+ *     (search_1 multi-BB path, bbs > 32)
+ *   - accumulate_q_4step_256 / pq4_accumulate_loop_qbs_fixed_scaler_256
+ *     (QBS path, bbs == 32, 256-bit kernel only)
+ *
+ * The QBS helpers here use pq4_kernel_qbs_256 exclusively (not
+ * decompose_qbs.h) because decompose_qbs.h includes kernels_simd512.h
+ * whose 512-bit types need explicit SIMD levels.  The 512-bit QBS path
+ * lives in accumulate_loops_512.h, used by the AVX512 per-ISA TU.
+ *
+ * All functions live in `namespace faiss` (not anonymous) so they can be
+ * shared by both the per-SIMD TU dispatcher (dispatching.h) and the old
+ * free-function search paths (pq4_fast_scan_search_1.cpp).
+ *
+ * The QBS helpers here always use pq4_kernel_qbs_256 (never 512-bit).
+ * This is required for the per-SIMD DD TUs where SINGLE_SIMD_LEVEL=NONE
+ * leaves 512-bit types empty.  The old pq4_fast_scan_search_qbs.cpp
+ * continues to use decompose_qbs.h which includes both 256 and 512 paths.
+ */
+
+#include <cassert>
+
+#include <faiss/impl/FaissAssert.h>
+#include <faiss/impl/fast_scan/LookupTableScaler.h>
+#include <faiss/impl/fast_scan/kernels_simd256.h>
+#include <faiss/impl/fast_scan/simd_result_handlers.h>
+
+namespace faiss {
+
+using namespace simd_result_handlers;
+
+/***************************************************************
+ * Search_1 path helpers (multi-BB kernel, bbs > 32)
+ ***************************************************************/
+
+template <
+        int NQ,
+        int BB,
+        SIMDLevel KernelSL = SINGLE_SIMD_LEVEL,
+        class ResultHandler,
+        class Scaler>
+void accumulate_fixed_blocks(
+        size_t nb,
+        int nsq,
+        const uint8_t* codes,
+        const uint8_t* LUT,
+        ResultHandler& res,
+        const Scaler& scaler,
+        size_t block_stride) {
+    constexpr int bbs = 32 * BB;
+    for_each_block<bbs>(nb, codes, block_stride, res, [&](size_t) {
+        FixedStorageHandler<NQ, 2 * BB, KernelSL> res2;
+        kernel_accumulate_block<NQ, BB, KernelSL>(
+                nsq, codes, LUT, res2, scaler);
+        res2.to_other_handler(res);
+    });
+}
+
+template <
+        SIMDLevel KernelSL = SINGLE_SIMD_LEVEL,
+        class ResultHandler,
+        class Scaler>
+void pq4_accumulate_loop_fixed_scaler(
+        int nq,
+        size_t nb,
+        int bbs,
+        int nsq,
+        const uint8_t* codes,
+        const uint8_t* LUT,
+        ResultHandler& res,
+        const Scaler& scaler,
+        size_t block_stride) {
+    FAISS_THROW_IF_NOT(is_aligned_pointer(codes));
+    FAISS_THROW_IF_NOT(is_aligned_pointer(LUT));
+    FAISS_THROW_IF_NOT(bbs % 32 == 0);
+    FAISS_THROW_IF_NOT(nb % bbs == 0);
+
+#define FAISS_ACCLOOP_DISPATCH(NQ, BB)                           \
+    case NQ * 1000 + BB:                                         \
+        accumulate_fixed_blocks<NQ, BB, KernelSL>(               \
+                nb, nsq, codes, LUT, res, scaler, block_stride); \
+        break
+
+    switch (nq * 1000 + bbs / 32) {
+        FAISS_ACCLOOP_DISPATCH(1, 1);
+        FAISS_ACCLOOP_DISPATCH(1, 2);
+        FAISS_ACCLOOP_DISPATCH(1, 3);
+        FAISS_ACCLOOP_DISPATCH(1, 4);
+        FAISS_ACCLOOP_DISPATCH(1, 5);
+        FAISS_ACCLOOP_DISPATCH(2, 1);
+        FAISS_ACCLOOP_DISPATCH(2, 2);
+        FAISS_ACCLOOP_DISPATCH(3, 1);
+        FAISS_ACCLOOP_DISPATCH(4, 1);
+        default:
+            FAISS_THROW_FMT("nq=%d bbs=%d not instantiated", nq, bbs);
+    }
+#undef FAISS_ACCLOOP_DISPATCH
+}
+
+/***************************************************************
+ * QBS path helpers (bbs == 32, 256-bit kernel only)
+ ***************************************************************/
+
+template <
+        int QBS,
+        SIMDLevel KernelSL = SINGLE_SIMD_LEVEL,
+        class ResultHandler,
+        class Scaler>
+void accumulate_q_4step_256(
+        size_t ntotal2,
+        int nsq,
+        const uint8_t* codes,
+        const uint8_t* LUT0,
+        ResultHandler& res,
+        const Scaler& scaler,
+        size_t block_stride) {
+    constexpr int Q1 = QBS & 15;
+    constexpr int Q2 = (QBS >> 4) & 15;
+    constexpr int Q3 = (QBS >> 8) & 15;
+    constexpr int Q4 = (QBS >> 12) & 15;
+    constexpr int SQ = Q1 + Q2 + Q3 + Q4;
+
+    for_each_block<32>(ntotal2, codes, block_stride, res, [&](size_t) {
+        FixedStorageHandler<SQ, 2, KernelSL> res2;
+        const uint8_t* LUT = LUT0;
+        pq4_kernel_qbs_256<Q1, KernelSL>(nsq, codes, LUT, res2, scaler);
+        LUT += Q1 * nsq * 16;
+        if (Q2 > 0) {
+            res2.set_block_origin(Q1, 0);
+            pq4_kernel_qbs_256<Q2, KernelSL>(nsq, codes, LUT, res2, scaler);
+            LUT += Q2 * nsq * 16;
+        }
+        if (Q3 > 0) {
+            res2.set_block_origin(Q1 + Q2, 0);
+            pq4_kernel_qbs_256<Q3, KernelSL>(nsq, codes, LUT, res2, scaler);
+            LUT += Q3 * nsq * 16;
+        }
+        if (Q4 > 0) {
+            res2.set_block_origin(Q1 + Q2 + Q3, 0);
+            pq4_kernel_qbs_256<Q4, KernelSL>(nsq, codes, LUT, res2, scaler);
+        }
+        res2.to_other_handler(res);
+    });
+}
+
+template <
+        SIMDLevel KernelSL = SINGLE_SIMD_LEVEL,
+        class ResultHandler,
+        class Scaler>
+void pq4_accumulate_loop_qbs_fixed_scaler_256(
+        int qbs,
+        size_t ntotal2,
+        int nsq,
+        const uint8_t* codes,
+        const uint8_t* LUT0,
+        ResultHandler& res,
+        const Scaler& scaler,
+        size_t block_stride) {
+    assert(nsq % 2 == 0);
+    assert(is_aligned_pointer(codes));
+    assert(is_aligned_pointer(LUT0));
+
+    switch (qbs) {
+#define FAISS_QBS256_DISPATCH(QBS)                                     \
+    case QBS:                                                          \
+        accumulate_q_4step_256<QBS, KernelSL>(                         \
+                ntotal2, nsq, codes, LUT0, res, scaler, block_stride); \
+        return;
+        FAISS_QBS256_DISPATCH(0x3333); // 12
+        FAISS_QBS256_DISPATCH(0x2333); // 11
+        FAISS_QBS256_DISPATCH(0x2233); // 10
+        FAISS_QBS256_DISPATCH(0x333);  // 9
+        FAISS_QBS256_DISPATCH(0x2223); // 9
+        FAISS_QBS256_DISPATCH(0x233);  // 8
+        FAISS_QBS256_DISPATCH(0x1223); // 8
+        FAISS_QBS256_DISPATCH(0x223);  // 7
+        FAISS_QBS256_DISPATCH(0x34);   // 7
+        FAISS_QBS256_DISPATCH(0x133);  // 7
+        FAISS_QBS256_DISPATCH(0x6);    // 6
+        FAISS_QBS256_DISPATCH(0x33);   // 6
+        FAISS_QBS256_DISPATCH(0x123);  // 6
+        FAISS_QBS256_DISPATCH(0x222);  // 6
+        FAISS_QBS256_DISPATCH(0x23);   // 5
+        FAISS_QBS256_DISPATCH(0x5);    // 5
+        FAISS_QBS256_DISPATCH(0x13);   // 4
+        FAISS_QBS256_DISPATCH(0x22);   // 4
+        FAISS_QBS256_DISPATCH(0x4);    // 4
+        FAISS_QBS256_DISPATCH(0x3);    // 3
+        FAISS_QBS256_DISPATCH(0x21);   // 3
+        FAISS_QBS256_DISPATCH(0x2);    // 2
+        FAISS_QBS256_DISPATCH(0x1);    // 1
+#undef FAISS_QBS256_DISPATCH
+    }
+
+    // Default: qbs not known at compile time
+    for_each_block<32>(ntotal2, codes, block_stride, res, [&](size_t j0) {
+        const uint8_t* LUT = LUT0;
+        int qi = qbs;
+        int i0 = 0;
+        while (qi) {
+            int nq = qi & 15;
+            qi >>= 4;
+            res.set_block_origin(i0, j0);
+#define FAISS_NQ256_DISPATCH(NQ)                                        \
+    case NQ:                                                            \
+        pq4_kernel_qbs_256<NQ, KernelSL>(nsq, codes, LUT, res, scaler); \
+        break
+            switch (nq) {
+                FAISS_NQ256_DISPATCH(1);
+                FAISS_NQ256_DISPATCH(2);
+                FAISS_NQ256_DISPATCH(3);
+                FAISS_NQ256_DISPATCH(4);
+#undef FAISS_NQ256_DISPATCH
+                default:
+                    FAISS_THROW_FMT("accumulate nq=%d not instantiated", nq);
+            }
+            i0 += nq;
+            LUT += nq * nsq * 16;
+        }
+    });
+}
+
+} // namespace faiss