faiss 0.1.3 → 0.1.4

data/vendor/faiss/faiss/IndexIVFPQFastScan.h

@@ -0,0 +1,166 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <memory>
+
+#include <faiss/IndexIVFPQ.h>
+#include <faiss/impl/ProductQuantizer.h>
+#include <faiss/utils/AlignedTable.h>
+
+namespace faiss {
+
+
+/** Fast scan version of IVFPQ. Works for 4-bit PQ for now.
+ *
+ * The codes in the inverted lists are not stored sequentially but
+ * grouped in blocks of size bbs. This makes it possible to very quickly
+ * compute distances with SIMD instructions.
+ *
+ * Implementations (implem):
+ * 0: auto-select implementation (default)
+ * 1: orig's search, re-implemented
+ * 2: orig's search, re-ordered by invlist
+ * 10: optimizer int16 search, collect results in heap, no qbs
+ * 11: idem, collect results in reservoir
+ * 12: optimizer int16 search, collect results in heap, uses qbs
+ * 13: idem, collect results in reservoir
+ */
+
+struct IndexIVFPQFastScan: IndexIVF {
+
+    bool by_residual;              ///< Encode residual or plain vector?
+    ProductQuantizer pq;           ///< produces the codes
+
+    // size of the kernel
+    int bbs;  // set at build time
+
+    // M rounded up to a multiple of 2
+    size_t M2;
+
+    /// precomputed tables management
+    int use_precomputed_table = 0;
+    /// if use_precompute_table size (nlist, pq.M, pq.ksub)
+    AlignedTable<float> precomputed_table;
+
+    // search-time implementation
+    int implem = 0;
+    // skip some parts of the computation (for timing)
+    int skip = 0;
+
+    // batching factors at search time (0 = default)
+    int qbs = 0;
+    size_t qbs2 = 0;
+
+    IndexIVFPQFastScan (
+            Index * quantizer, size_t d, size_t nlist,
+            size_t M, size_t nbits_per_idx,
+            MetricType metric = METRIC_L2, int bbs = 32);
+
+    IndexIVFPQFastScan ();
+
+    // built from an IndexIVFPQ
+    explicit IndexIVFPQFastScan(const IndexIVFPQ & orig, int bbs = 32);
+
+    /// orig's inverted lists (for debugging)
+    InvertedLists * orig_invlists = nullptr;
+
+    void train_residual (idx_t n, const float *x) override;
+
+    /// build precomputed table, possibly updating use_precomputed_table
+    void precompute_table ();
+
+    /// same as the regular IVFPQ encoder. The codes are not reorganized by
+    /// blocks a that point
+    void encode_vectors(
+            idx_t n, const float* x,
+            const idx_t *list_nos, uint8_t * codes,
+            bool include_listno = false) const override;
+
+    void add_with_ids (
+        idx_t n, const float * x, const idx_t *xids) override;
+
+    void search(
+            idx_t n, const float* x, idx_t k,
+            float* distances, idx_t* labels) const override;
+
+    // prepare look-up tables
+
+    void compute_LUT(
+        size_t n, const float *x,
+        const idx_t *coarse_ids, const float *coarse_dis,
+        AlignedTable<float> & dis_tables,
+        AlignedTable<float> & biases
+    ) const;
+
+    void compute_LUT_uint8(
+        size_t n, const float *x,
+        const idx_t *coarse_ids, const float *coarse_dis,
+        AlignedTable<uint8_t> & dis_tables,
+        AlignedTable<uint16_t> & biases,
+        float * normalizers
+    ) const;
+
+    // internal search funcs
+
+    template<bool is_max>
+    void search_dispatch_implem(
+            idx_t n, const float* x, idx_t k,
+            float* distances, idx_t* labels) const;
+
+    template<class C>
+    void search_implem_1(
+            idx_t n, const float* x, idx_t k,
+            float* distances, idx_t* labels) const;
+
+    template<class C>
+    void search_implem_2(
+            idx_t n, const float* x, idx_t k,
+            float* distances, idx_t* labels) const;
+
+    // implem 10 and 12 are not multithreaded internally, so
+    // export search stats
+    template<class C>
+    void search_implem_10(
+            idx_t n, const float* x, idx_t k,
+            float* distances, idx_t* labels,
+            int impl, size_t *ndis_out, size_t *nlist_out) const;
+
+    template<class C>
+    void search_implem_12(
+            idx_t n, const float* x, idx_t k,
+            float* distances, idx_t* labels,
+            int impl, size_t *ndis_out, size_t *nlist_out) const;
+
+
+
+};
+
+struct IVFFastScanStats {
+    uint64_t times[10];
+    uint64_t t_compute_distance_tables, t_round;
+    uint64_t t_copy_pack, t_scan, t_to_flat;
+    uint64_t reservoir_times[4];
+
+    double Mcy_at(int i) {
+        return times[i] / (1000*1000.0);
+    }
+
+    double Mcy_reservoir_at(int i) {
+        return reservoir_times[i] / (1000*1000.0);
+    }
+    IVFFastScanStats() {reset();}
+    void reset() {
+        memset(this, 0, sizeof(*this));
+    }
+};
+
+FAISS_API extern IVFFastScanStats IVFFastScan_stats;
+
+
+} // namespace faiss

data/vendor/faiss/faiss/IndexIVFPQR.cpp

@@ -97,13 +97,13 @@ void IndexIVFPQR::add_core (idx_t n, const float *x, const idx_t *xids,
 #define TOC get_cycles () - t0
 
 
-void IndexIVFPQR::search_preassigned (idx_t n, const float *x, idx_t k,
-                                      const idx_t *idx,
-                                      const float *L1_dis,
-                                      float *distances, idx_t *labels,
-                                      bool store_pairs,
-                                      const IVFSearchParameters *params
-                                      ) const
+void IndexIVFPQR::search_preassigned (
+        idx_t n, const float *x, idx_t k,
+        const idx_t *idx, const float *L1_dis,
+        float *distances, idx_t *labels,
+        bool store_pairs,
+        const IVFSearchParameters *params, IndexIVFStats *stats
+        ) const
 {
     uint64_t t0;
     TIC;
@@ -172,9 +172,8 @@ void IndexIVFPQR::search_preassigned (idx_t n, const float *x, idx_t k,
                 float dis = fvec_L2sqr (residual_1, residual_2, d);
 
                 if (dis < heap_sim[0]) {
-                    maxheap_pop (k, heap_sim, heap_ids);
                     idx_t id_or_pair = store_pairs ? sl : id;
-                    maxheap_push (k, heap_sim, heap_ids, dis, id_or_pair);
+                    maxheap_replace_top (k, heap_sim, heap_ids, dis, id_or_pair);
                 }
                 n_refine ++;
             }

data/vendor/faiss/faiss/IndexIVFPQR.h

@@ -55,7 +55,8 @@ struct IndexIVFPQR: IndexIVFPQ {
                              const float *centroid_dis,
                              float *distances, idx_t *labels,
                              bool store_pairs,
-                             const IVFSearchParameters *params=nullptr
+                             const IVFSearchParameters *params=nullptr,
+                             IndexIVFStats *stats=nullptr
                              ) const override;
 
     IndexIVFPQR();

data/vendor/faiss/faiss/IndexIVFSpectralHash.cpp

@@ -269,9 +269,8 @@ struct IVFScanner: InvertedListScanner {
             float dis = hc.hamming (codes);
 
             if (dis < simi [0]) {
-                maxheap_pop (k, simi, idxi);
                 int64_t id = store_pairs ? lo_build (list_no, j) : ids[j];
-                maxheap_push (k, simi, idxi, dis, id);
+                maxheap_replace_top (k, simi, idxi, dis, id);
                 nup++;
             }
             codes += code_size;

data/vendor/faiss/faiss/IndexPQ.cpp

@@ -129,9 +129,10 @@ void IndexPQ::reconstruct (idx_t key, float * recons) const
 
 namespace {
 
-
-struct PQDis: DistanceComputer {
+template<class PQDecoder>
+struct PQDistanceComputer: DistanceComputer {
     size_t d;
+    MetricType metric;
     Index::idx_t nb;
     const uint8_t *codes;
     size_t code_size;
@@ -144,10 +145,11 @@ struct PQDis: DistanceComputer {
     {
         const uint8_t *code = codes + i * code_size;
         const float *dt = precomputed_table.data();
+        PQDecoder decoder(code, pq.nbits);
         float accu = 0;
         for (int j = 0; j < pq.M; j++) {
-            accu += dt[*code++];
-            dt += 256;
+            accu += dt[decoder.decode()];
+            dt += 1 << decoder.nbits;
         }
         ndis++;
         return accu;
@@ -155,33 +157,43 @@ struct PQDis: DistanceComputer {
 
     float symmetric_dis(idx_t i, idx_t j) override
     {
+        FAISS_THROW_IF_NOT(sdc);
         const float * sdci = sdc;
         float accu = 0;
-        const uint8_t *codei = codes + i * code_size;
-        const uint8_t *codej = codes + j * code_size;
+        PQDecoder codei (codes + i * code_size, pq.nbits);
+        PQDecoder codej (codes + j * code_size, pq.nbits);
 
         for (int l = 0; l < pq.M; l++) {
-            accu += sdci[(*codei++) + (*codej++) * 256];
-            sdci += 256 * 256;
+            accu += sdci[codei.decode() + (codej.decode() << codei.nbits)];
+            sdci += uint64_t(1) << (2 * codei.nbits);
         }
+        ndis++;
         return accu;
     }
 
-    explicit PQDis(const IndexPQ& storage, const float* /*q*/ = nullptr)
-        : pq(storage.pq) {
+    explicit PQDistanceComputer(const IndexPQ& storage)
+            : pq(storage.pq) {
         precomputed_table.resize(pq.M * pq.ksub);
         nb = storage.ntotal;
         d = storage.d;
+        metric = storage.metric_type;
         codes = storage.codes.data();
         code_size = pq.code_size;
-        FAISS_ASSERT(pq.ksub == 256);
-        FAISS_ASSERT(pq.sdc_table.size() == pq.ksub * pq.ksub * pq.M);
-        sdc = pq.sdc_table.data();
+        if (pq.sdc_table.size() == pq.ksub * pq.ksub * pq.M) {
+            sdc = pq.sdc_table.data();
+        } else {
+            sdc = nullptr;
+        }
         ndis = 0;
     }
 
     void set_query(const float *x) override {
-        pq.compute_distance_table(x, precomputed_table.data());
+        if (metric == METRIC_L2) {
+            pq.compute_distance_table(x, precomputed_table.data());
+        } else {
+            pq.compute_inner_prod_table(x, precomputed_table.data());
+        }
+
     }
 };
 
@@ -190,8 +202,13 @@ struct PQDis: DistanceComputer {
 
 
 DistanceComputer * IndexPQ::get_distance_computer() const {
-    FAISS_THROW_IF_NOT(pq.nbits == 8);
-    return new PQDis(*this);
+    if (pq.nbits == 8) {
+        return new PQDistanceComputer<PQDecoder8>(*this);
+    } else if (pq.nbits == 16) {
+        return new PQDistanceComputer<PQDecoder16>(*this);
+    } else {
+        return new PQDistanceComputer<PQDecoderGeneric>(*this);
+    }
 }
 
 
@@ -329,8 +346,7 @@ static size_t polysemous_inner_loop (
             }
 
             if (dis < heap_dis[0]) {
-                maxheap_pop (k, heap_dis, heap_ids);
-                maxheap_push (k, heap_dis, heap_ids, dis, bi);
+                maxheap_replace_top (k, heap_dis, heap_ids, dis, bi);
             }
         }
         b_code += code_size;

data/vendor/faiss/faiss/IndexPQFastScan.cpp

@@ -0,0 +1,536 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+
+#include <faiss/IndexPQFastScan.h>
+
+#include <cassert>
+#include <memory>
+#include <limits.h>
+
+#include <omp.h>
+
+
+#include <faiss/impl/FaissAssert.h>
+#include <faiss/utils/utils.h>
+#include <faiss/utils/random.h>
+
+#include <faiss/impl/simd_result_handlers.h>
+#include <faiss/utils/quantize_lut.h>
+#include <faiss/impl/pq4_fast_scan.h>
+
+
+namespace faiss {
+
+using namespace simd_result_handlers;
+
+inline size_t roundup(size_t a, size_t b) {
+    return (a + b - 1) / b * b;
+}
+
+IndexPQFastScan::IndexPQFastScan(
+        int d, size_t M, size_t nbits,
+        MetricType metric,
+        int bbs):
+    Index(d, metric), pq(d, M, nbits),
+    bbs(bbs), ntotal2(0), M2(roundup(M, 2))
+{
+    FAISS_THROW_IF_NOT(nbits == 4);
+    is_trained = false;
+}
+
+IndexPQFastScan::IndexPQFastScan():
+    bbs(0), ntotal2(0), M2(0)
+{}
+
+IndexPQFastScan::IndexPQFastScan(const IndexPQ & orig, int bbs):
+    Index(orig.d, orig.metric_type),
+    pq(orig.pq),
+    bbs(bbs)
+{
+    FAISS_THROW_IF_NOT(orig.pq.nbits == 4);
+    ntotal = orig.ntotal;
+    is_trained = orig.is_trained;
+    orig_codes = orig.codes.data();
+
+    qbs = 0; // means use default
+
+    // pack the codes
+
+    size_t M = pq.M;
+
+    FAISS_THROW_IF_NOT(bbs % 32 == 0);
+    M2 = roundup(M, 2);
+    ntotal2 = roundup(ntotal, bbs);
+
+    codes.resize(ntotal2 * M2 / 2);
+
+    // printf("M=%d M2=%d code_size=%d\n", M, M2, pq.code_size);
+    pq4_pack_codes(
+            orig.codes.data(),
+            ntotal, M,
+            ntotal2, bbs, M2,
+            codes.get()
+    );
+}
+
+void IndexPQFastScan::train (idx_t n, const float *x)
+{
+    if (is_trained) {
+        return;
+    }
+    pq.train(n, x);
+    is_trained = true;
+}
+
+
+void IndexPQFastScan::add (idx_t n, const float *x) {
+    FAISS_THROW_IF_NOT (is_trained);
+    AlignedTable<uint8_t> tmp_codes(n * pq.code_size);
+    pq.compute_codes (x, tmp_codes.get(), n);
+    ntotal2 = roundup(ntotal + n, bbs);
+    size_t new_size = ntotal2 * M2 / 2;
+    size_t old_size = codes.size();
+    if (new_size > old_size) {
+        codes.resize(new_size);
+        memset(codes.get() + old_size, 0, new_size - old_size);
+    }
+    pq4_pack_codes_range(
+        tmp_codes.get(), pq.M, ntotal, ntotal + n,
+        bbs, M2, codes.get()
+    );
+    ntotal += n;
+}
+
+void IndexPQFastScan::reset()
+{
+    codes.resize(0);
+    ntotal = 0;
+ }
+
+
+
+namespace {
+
+// from impl/ProductQuantizer.cpp
+template <class C, typename dis_t>
+void pq_estimators_from_tables_generic(
+        const ProductQuantizer& pq, size_t nbits,
+        const uint8_t *codes, size_t ncodes,
+        const dis_t *dis_table, size_t k,
+        typename C::T *heap_dis, int64_t *heap_ids)
+{
+    using accu_t = typename C::T;
+    const size_t M = pq.M;
+    const size_t ksub = pq.ksub;
+    for (size_t j = 0; j < ncodes; ++j) {
+        PQDecoderGeneric decoder(
+                codes + j * pq.code_size, nbits
+        );
+        accu_t dis = 0;
+        const dis_t * __restrict dt = dis_table;
+        for (size_t m = 0; m < M; m++) {
+            uint64_t c = decoder.decode();
+            dis += dt[c];
+            dt += ksub;
+        }
+
+        if (C::cmp(heap_dis[0], dis)) {
+            heap_pop<C>(k, heap_dis, heap_ids);
+            heap_push<C>(k, heap_dis, heap_ids, dis, j);
+        }
+    }
+}
+
+
+} // anonymous namespace
+
+
+using namespace quantize_lut;
+
+void IndexPQFastScan::compute_quantized_LUT(
+        idx_t n, const float* x,
+        uint8_t *lut, float *normalizers) const
+{
+    size_t dim12 = pq.ksub * pq.M;
+    std::unique_ptr<float[]> dis_tables(new float [n * dim12]);
+    if (metric_type == METRIC_L2) {
+        pq.compute_distance_tables (n, x, dis_tables.get());
+    } else {
+        pq.compute_inner_prod_tables (n, x, dis_tables.get());
+    }
+
+    for(uint64_t i = 0; i < n; i++) {
+        round_uint8_per_column(
+                dis_tables.get() + i * dim12, pq.M, pq.ksub,
+                &normalizers[2 * i], &normalizers[2 * i + 1]
+        );
+    }
+
+    for(uint64_t i = 0; i < n; i++) {
+        const float *t_in = dis_tables.get() + i * dim12;
+        uint8_t *t_out = lut + i * M2 * pq.ksub;
+
+        for(int j = 0; j < dim12; j++) {
+            t_out[j] = int(t_in[j]);
+        }
+        memset(t_out + dim12, 0, (M2 - pq.M) * pq.ksub);
+    }
+}
+
+
+
+/******************************************************************************
+ * Search driver routine
+ ******************************************************************************/
+
+
+void IndexPQFastScan::search(
+                idx_t n, const float* x, idx_t k,
+                float* distances, idx_t* labels) const
+{
+    if (metric_type == METRIC_L2) {
+        search_dispatch_implem<true>(n, x, k, distances, labels);
+    } else {
+        search_dispatch_implem<false>(n, x, k, distances, labels);
+    }
+}
+
+
+template<bool is_max>
+void IndexPQFastScan::search_dispatch_implem(
+                idx_t n,
+                const float* x,
+                idx_t k,
+                float* distances,
+                idx_t* labels) const
+{
+    using Cfloat = typename std::conditional<is_max,
+        CMax<float, int64_t>, CMin<float, int64_t> >::type;
+
+    using C = typename std::conditional<is_max,
+        CMax<uint16_t, int>, CMin<uint16_t, int> >::type;
+
+    if (n == 0) {
+        return;
+    }
+
+    // actual implementation used
+    int impl = implem;
+
+    if (impl == 0) {
+        if (bbs == 32) {
+            impl = 12;
+        } else {
+            impl = 14;
+        }
+        if (k > 20) {
+            impl ++;
+        }
+    }
+
+
+    if (implem == 1) {
+        FAISS_THROW_IF_NOT(orig_codes);
+        FAISS_THROW_IF_NOT(is_max);
+        float_maxheap_array_t res = {
+            size_t(n), size_t(k), labels, distances };
+        pq.search (x, n, orig_codes, ntotal, &res, true);
+    } else if (implem == 2 || implem == 3 || implem == 4) {
+        FAISS_THROW_IF_NOT(orig_codes);
+
+        size_t dim12 = pq.ksub * pq.M;
+        std::unique_ptr<float[]> dis_tables(new float [n * dim12]);
+        if (is_max) {
+            pq.compute_distance_tables (n, x, dis_tables.get());
+        } else {
+            pq.compute_inner_prod_tables (n, x, dis_tables.get());
+        }
+
+        std::vector<float> normalizers(n * 2);
+
+        if (implem == 2) {
+            // default float
+        } else if (implem == 3 || implem == 4) {
+            for(uint64_t i = 0; i < n; i++) {
+                round_uint8_per_column(
+                        dis_tables.get() + i * dim12, pq.M,
+                        pq.ksub,
+                        &normalizers[2 * i], &normalizers[2 * i + 1]
+                );
+            }
+        }
+
+        for (int64_t i = 0; i < n; i++) {
+            int64_t *heap_ids = labels + i * k;
+            float *heap_dis = distances + i * k;
+
+            heap_heapify<Cfloat> (k, heap_dis, heap_ids);
+
+            pq_estimators_from_tables_generic<Cfloat>(
+                pq, pq.nbits, orig_codes, ntotal,
+                dis_tables.get() + i * dim12,
+                k, heap_dis, heap_ids
+            );
+
+            heap_reorder<Cfloat> (k, heap_dis, heap_ids);
+
+            if (implem == 4) {
+                float a = normalizers[2 * i];
+                float b = normalizers[2 * i + 1];
+
+                for(int j = 0; j < k; j++) {
+                    heap_dis[j] = heap_dis[j] / a + b;
+                }
+            }
+        }
+    } else if (impl >= 12 && impl <= 15) {
+        FAISS_THROW_IF_NOT(ntotal < INT_MAX);
+        int nt = std::min(omp_get_max_threads(), int(n));
+        if (nt < 2) {
+            if (impl == 12 || impl == 13) {
+                search_implem_12<C>(n, x, k, distances, labels, impl);
+            } else {
+                search_implem_14<C>(n, x, k, distances, labels, impl);
+            }
+        } else {
+            // explicitly slice over threads
+#pragma omp parallel for num_threads(nt)
+            for (int slice = 0; slice < nt; slice++) {
+                idx_t i0 = n * slice / nt;
+                idx_t i1 = n * (slice + 1) / nt;
+                float *dis_i = distances + i0 * k;
+                idx_t *lab_i = labels + i0 * k;
+                if (impl == 12 || impl == 13) {
+                    search_implem_12<C>(
+                        i1 - i0, x + i0 * d, k, dis_i, lab_i, impl);
+                } else {
+                    search_implem_14<C>(
+                        i1 - i0, x + i0 * d, k, dis_i, lab_i, impl);
+                }
+            }
+        }
+    } else {
+        FAISS_THROW_FMT("invalid implem %d impl=%d", implem, impl);
+    }
+
+}
+
+template<class C>
+void IndexPQFastScan::search_implem_12(
+                idx_t n, const float* x, idx_t k,
+                float* distances, idx_t* labels,
+                int impl) const
+{
+
+    FAISS_THROW_IF_NOT(bbs == 32);
+
+    // handle qbs2 blocking by recursive call
+    int64_t qbs2 = this->qbs == 0 ? 11 : pq4_qbs_to_nq(this->qbs);
+    if (n > qbs2) {
+        for (int64_t i0 = 0; i0 < n; i0 += qbs2) {
+            int64_t i1 = std::min(i0 + qbs2, n);
+            search_implem_12<C>(
+                    i1 - i0, x + d * i0, k,
+                    distances + i0 * k, labels + i0 * k, impl
+            );
+        }
+        return;
+    }
+
+    size_t dim12 = pq.ksub * M2;
+    AlignedTable<uint8_t> quantized_dis_tables(n * dim12);
+    std::unique_ptr<float []> normalizers(new float[2 * n]);
+
+    if (skip & 1) {
+        quantized_dis_tables.clear();
+    } else {
+        compute_quantized_LUT(
+            n, x, quantized_dis_tables.get(), normalizers.get()
+        );
+    }
+
+    AlignedTable<uint8_t> LUT(n * dim12);
+
+    // block sizes are encoded in qbs, 4 bits at a time
+
+    // caution: we override an object field
+    int qbs = this->qbs;
+
+    if (n != pq4_qbs_to_nq(qbs)) {
+        qbs = pq4_preferred_qbs(n);
+    }
+
+    int LUT_nq = pq4_pack_LUT_qbs(
+        qbs, M2, quantized_dis_tables.get(), LUT.get()
+    );
+    FAISS_THROW_IF_NOT(LUT_nq == n);
+
+    if (k == 1) {
+        SingleResultHandler<C> handler(n, ntotal);
+        if (skip & 4) {
+            // pass
+        } else {
+            handler.disable = bool(skip & 2);
+            pq4_accumulate_loop_qbs(
+                qbs, ntotal2, M2,
+                codes.get(), LUT.get(),
+                handler
+            );
+        }
+
+        handler.to_flat_arrays(distances, labels, normalizers.get());
+
+    } else if (impl == 12) {
+
+        std::vector<uint16_t> tmp_dis(n * k);
+        std::vector<int32_t> tmp_ids(n * k);
+
+        if (skip & 4) {
+            // skip
+        } else {
+            HeapHandler<C> handler(n, tmp_dis.data(), tmp_ids.data(), k, ntotal);
+            handler.disable = bool(skip & 2);
+
+            pq4_accumulate_loop_qbs(
+                qbs, ntotal2, M2,
+                codes.get(), LUT.get(),
+                handler
+            );
+
+            if (!(skip & 8)) {
+                handler.to_flat_arrays(distances, labels, normalizers.get());
+            }
+        }
+
+
+    } else { // impl == 13
+
+        ReservoirHandler<C> handler(n, ntotal, k, 2 * k);
+        handler.disable = bool(skip & 2);
+
+        if (skip & 4) {
+            // skip
+        } else {
+            pq4_accumulate_loop_qbs(
+                qbs, ntotal2, M2,
+                codes.get(), LUT.get(),
+                handler
+            );
+        }
+
+        if (!(skip & 8)) {
+            handler.to_flat_arrays(distances, labels, normalizers.get());
+        }
+
+        FastScan_stats.t0 += handler.times[0];
+        FastScan_stats.t1 += handler.times[1];
+        FastScan_stats.t2 += handler.times[2];
+        FastScan_stats.t3 += handler.times[3];
+
+    }
+}
+
+FastScanStats FastScan_stats;
+
+template<class C>
+void IndexPQFastScan::search_implem_14(
+                idx_t n, const float* x, idx_t k,
+                float* distances, idx_t* labels, int impl) const
+{
+
+    FAISS_THROW_IF_NOT(bbs % 32 == 0);
+
+    int qbs2 = qbs == 0 ? 4 : qbs;
+
+    // handle qbs2 blocking by recursive call
+    if (n > qbs2) {
+        for (int64_t i0 = 0; i0 < n; i0 += qbs2) {
+            int64_t i1 = std::min(i0 + qbs2, n);
+            search_implem_14<C>(
+                    i1 - i0, x + d * i0, k,
+                    distances + i0 * k, labels + i0 * k, impl
+            );
+        }
+        return;
+    }
+
+    size_t dim12 = pq.ksub * M2;
+    AlignedTable<uint8_t> quantized_dis_tables(n * dim12);
+    std::unique_ptr<float []> normalizers(new float[2 * n]);
+
+    if (skip & 1) {
+        quantized_dis_tables.clear();
+    } else {
+        compute_quantized_LUT(
+            n, x, quantized_dis_tables.get(), normalizers.get()
+        );
+    }
+
+    AlignedTable<uint8_t> LUT(n * dim12);
+    pq4_pack_LUT(n, M2, quantized_dis_tables.get(), LUT.get());
+
+    if (k == 1) {
+        SingleResultHandler<C> handler(n, ntotal);
+        if (skip & 4) {
+            // pass
+        } else {
+            handler.disable = bool(skip & 2);
+            pq4_accumulate_loop (
+                n, ntotal2, bbs, M2,
+                codes.get(), LUT.get(),
+                handler
+            );
+        }
+        handler.to_flat_arrays(distances, labels, normalizers.get());
+
+    } else if (impl == 14) {
+
+        std::vector<uint16_t> tmp_dis(n * k);
+        std::vector<int32_t> tmp_ids(n * k);
+
+        if (skip & 4) {
+            // skip
+        } else if (k > 1) {
+            HeapHandler<C> handler(n, tmp_dis.data(), tmp_ids.data(), k, ntotal);
+            handler.disable = bool(skip & 2);
+
+            pq4_accumulate_loop (
+                n, ntotal2, bbs, M2,
+                codes.get(), LUT.get(),
+                handler
+            );
+
+            if (!(skip & 8)) {
+                handler.to_flat_arrays(distances, labels, normalizers.get());
+            }
+        }
+
+
+    } else { // impl == 15
+
+        ReservoirHandler<C> handler(n, ntotal, k, 2 * k);
+        handler.disable = bool(skip & 2);
+
+        if (skip & 4) {
+            // skip
+        } else {
+            pq4_accumulate_loop (
+                n, ntotal2, bbs, M2,
+                codes.get(), LUT.get(),
+                handler
+            );
+        }
+
+        if (!(skip & 8)) {
+            handler.to_flat_arrays(distances, labels, normalizers.get());
+        }
+    }
+}
+
+
+} // namespace faiss