faiss 0.6.1 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/lib/faiss/version.rb +1 -1
- data/vendor/faiss/faiss/AutoTune.cpp +3 -1
- data/vendor/faiss/faiss/Clustering.cpp +9 -1
- data/vendor/faiss/faiss/IVFlib.cpp +14 -3
- data/vendor/faiss/faiss/Index.h +2 -2
- data/vendor/faiss/faiss/IndexAdditiveQuantizer.cpp +9 -10
- data/vendor/faiss/faiss/IndexAdditiveQuantizerFastScan.cpp +2 -3
- data/vendor/faiss/faiss/IndexBinaryFromFloat.cpp +1 -2
- data/vendor/faiss/faiss/IndexBinaryHNSW.cpp +10 -12
- data/vendor/faiss/faiss/IndexBinaryHash.cpp +5 -9
- data/vendor/faiss/faiss/IndexBinaryIVF.cpp +5 -7
- data/vendor/faiss/faiss/IndexEDEN.cpp +273 -0
- data/vendor/faiss/faiss/IndexEDEN.h +57 -0
- data/vendor/faiss/faiss/IndexFastScan.cpp +15 -4
- data/vendor/faiss/faiss/IndexFlat.cpp +13 -50
- data/vendor/faiss/faiss/IndexHNSW.cpp +177 -148
- data/vendor/faiss/faiss/IndexIDMap.cpp +16 -3
- data/vendor/faiss/faiss/IndexIDMap.h +2 -0
- data/vendor/faiss/faiss/IndexIVF.cpp +19 -8
- data/vendor/faiss/faiss/IndexIVFAdditiveQuantizer.cpp +3 -3
- data/vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.cpp +3 -4
- data/vendor/faiss/faiss/IndexIVFEDEN.cpp +302 -0
- data/vendor/faiss/faiss/IndexIVFEDEN.h +70 -0
- data/vendor/faiss/faiss/IndexIVFFastScan.cpp +5 -6
- data/vendor/faiss/faiss/IndexIVFFlat.cpp +6 -5
- data/vendor/faiss/faiss/IndexIVFFlatPanorama.cpp +3 -3
- data/vendor/faiss/faiss/IndexIVFIndependentQuantizer.cpp +1 -1
- data/vendor/faiss/faiss/IndexIVFPQ.cpp +42 -25
- data/vendor/faiss/faiss/IndexIVFPQFastScan.cpp +0 -1
- data/vendor/faiss/faiss/IndexIVFPQR.cpp +2 -3
- data/vendor/faiss/faiss/IndexIVFRaBitQ.cpp +23 -62
- data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.cpp +180 -76
- data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.h +5 -4
- data/vendor/faiss/faiss/IndexIVFSpectralHash.cpp +8 -6
- data/vendor/faiss/faiss/IndexLSH.cpp +2 -3
- data/vendor/faiss/faiss/IndexLattice.cpp +5 -0
- data/vendor/faiss/faiss/IndexNNDescent.cpp +10 -3
- data/vendor/faiss/faiss/IndexNSG.cpp +8 -4
- data/vendor/faiss/faiss/IndexPQ.cpp +6 -8
- data/vendor/faiss/faiss/IndexPreTransform.cpp +15 -0
- data/vendor/faiss/faiss/IndexRaBitQ.cpp +2 -2
- data/vendor/faiss/faiss/IndexRaBitQFastScan.cpp +1 -2
- data/vendor/faiss/faiss/IndexRaBitQFastScan.h +5 -1
- data/vendor/faiss/faiss/IndexRefine.cpp +30 -1
- data/vendor/faiss/faiss/IndexReplicas.cpp +1 -2
- data/vendor/faiss/faiss/IndexScalarQuantizer.cpp +68 -6
- data/vendor/faiss/faiss/IndexScalarQuantizer.h +10 -0
- data/vendor/faiss/faiss/IndexShards.cpp +2 -2
- data/vendor/faiss/faiss/IndexShardsIVF.cpp +2 -2
- data/vendor/faiss/faiss/MetaIndexes.cpp +2 -4
- data/vendor/faiss/faiss/SuperKMeans.cpp +256 -240
- data/vendor/faiss/faiss/SuperKMeans.h +30 -0
- data/vendor/faiss/faiss/VectorTransform.cpp +33 -2
- data/vendor/faiss/faiss/clone_index.cpp +5 -0
- data/vendor/faiss/faiss/cppcontrib/SaDecodeKernels.h +1 -1
- data/vendor/faiss/faiss/cppcontrib/sa_decode/Level2-neon-inl.h +902 -12
- data/vendor/faiss/faiss/cppcontrib/sa_decode/PQ-neon-inl.h +702 -10
- data/vendor/faiss/faiss/factory_tools.cpp +51 -4
- data/vendor/faiss/faiss/gpu/GpuCloner.cpp +11 -11
- data/vendor/faiss/faiss/gpu/GpuIndex.h +34 -11
- data/vendor/faiss/faiss/gpu/GpuIndexCagra.h +47 -0
- data/vendor/faiss/faiss/gpu/GpuIndexIVF.h +17 -0
- data/vendor/faiss/faiss/gpu/GpuIndexIVFScalarQuantizer.h +16 -0
- data/vendor/faiss/faiss/gpu/GpuResources.h +3 -2
- data/vendor/faiss/faiss/gpu/StandardGpuResources.cpp +11 -12
- data/vendor/faiss/faiss/gpu/StandardGpuResources.h +3 -3
- data/vendor/faiss/faiss/gpu/perf/PerfClustering.cpp +1 -1
- data/vendor/faiss/faiss/gpu/perf/PerfIVFPQAdd.cpp +2 -2
- data/vendor/faiss/faiss/gpu/test/TestGpuIndexIVFScalarQuantizer.cpp +180 -0
- data/vendor/faiss/faiss/gpu_metal/MetalDistance.h +87 -0
- data/vendor/faiss/faiss/gpu_metal/MetalIndex.h +7 -0
- data/vendor/faiss/faiss/gpu_metal/MetalIndexIVFFlat.h +177 -0
- data/vendor/faiss/faiss/gpu_metal/MetalIndexIVFPQ.h +88 -0
- data/vendor/faiss/faiss/gpu_metal/MetalKernels.h +48 -3
- data/vendor/faiss/faiss/gpu_metal/MetalPythonBridge.h +45 -0
- data/vendor/faiss/faiss/gpu_metal/impl/MetalIVFFlat.h +193 -0
- data/vendor/faiss/faiss/gpu_metal/impl/MetalIVFPQ.h +134 -0
- data/vendor/faiss/faiss/impl/ClusteringInitialization.cpp +2 -2
- data/vendor/faiss/faiss/impl/DistanceComputer.h +34 -0
- data/vendor/faiss/faiss/impl/EDENQuantizer.h +119 -0
- data/vendor/faiss/faiss/impl/HNSW.cpp +658 -344
- data/vendor/faiss/faiss/impl/HNSW.h +51 -13
- data/vendor/faiss/faiss/impl/LocalSearchQuantizer.cpp +2 -2
- data/vendor/faiss/faiss/impl/NSG.cpp +18 -12
- data/vendor/faiss/faiss/impl/Panorama.h +20 -7
- data/vendor/faiss/faiss/impl/PolysemousTraining.cpp +152 -84
- data/vendor/faiss/faiss/impl/ProductQuantizer.cpp +59 -24
- data/vendor/faiss/faiss/impl/RaBitQUtils.cpp +45 -37
- data/vendor/faiss/faiss/impl/RaBitQUtils.h +35 -0
- data/vendor/faiss/faiss/impl/RaBitQuantizer.cpp +175 -68
- data/vendor/faiss/faiss/impl/RaBitQuantizer.h +19 -0
- data/vendor/faiss/faiss/impl/RaBitQuantizerMultiBit.cpp +2 -11
- data/vendor/faiss/faiss/impl/ResultHandler.h +26 -31
- data/vendor/faiss/faiss/impl/ScalarQuantizer.cpp +522 -58
- data/vendor/faiss/faiss/impl/ScalarQuantizer.h +70 -0
- data/vendor/faiss/faiss/impl/ThreadedIndex-inl.h +2 -2
- data/vendor/faiss/faiss/impl/VisitedTable.cpp +33 -13
- data/vendor/faiss/faiss/impl/VisitedTable.h +88 -33
- data/vendor/faiss/faiss/impl/binary_hamming/IndexBinaryIVF_impl.h +1 -1
- data/vendor/faiss/faiss/impl/binary_hamming/avx2.cpp +4 -4
- data/vendor/faiss/faiss/impl/fast_scan/dispatching.h +38 -3
- data/vendor/faiss/faiss/impl/hnsw/LockVector.cpp +1 -1
- data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.cpp +35 -43
- data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.h +64 -15
- data/vendor/faiss/faiss/impl/hnsw/avx2.cpp +86 -40
- data/vendor/faiss/faiss/impl/hnsw/avx512.cpp +81 -50
- data/vendor/faiss/faiss/impl/index_read.cpp +476 -75
- data/vendor/faiss/faiss/impl/index_write.cpp +56 -4
- data/vendor/faiss/faiss/impl/io_macros.h +25 -0
- data/vendor/faiss/faiss/impl/lattice_Zn.cpp +8 -9
- data/vendor/faiss/faiss/impl/platform_macros.h +15 -9
- data/vendor/faiss/faiss/impl/polysemous_training/avx512.cpp +284 -0
- data/vendor/faiss/faiss/impl/polysemous_training/dispatch.h +115 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/IVFPQ_QueryTables.cpp +0 -1
- data/vendor/faiss/faiss/impl/pq_code_distance/PQDistanceComputer_impl.h +26 -15
- data/vendor/faiss/faiss/impl/pq_code_distance/avx2.cpp +6 -4
- data/vendor/faiss/faiss/impl/pq_code_distance/avx512.cpp +2 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/neon.cpp +2 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-generic.cpp +20 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-inl.h +36 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-sve.cpp +5 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_scan_impl.h +105 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/rvv.cpp +2 -0
- data/vendor/faiss/faiss/impl/result_handler/ResultHandler.cpp +195 -0
- data/vendor/faiss/faiss/impl/result_handler/avx2.cpp +133 -0
- data/vendor/faiss/faiss/impl/result_handler/avx512.cpp +281 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx2.cpp +72 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx512.cpp +228 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer.cpp +882 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/distance_computers.h +6 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/quantizers.h +336 -26
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx2.cpp +331 -32
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-impl.h +553 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-spr.cpp +558 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512.cpp +284 -45
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-dispatch.h +502 -3
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-neon.cpp +157 -32
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-rvv.cpp +26 -0
- data/vendor/faiss/faiss/impl/simd_dispatch.h +86 -8
- data/vendor/faiss/faiss/index_factory.cpp +37 -7
- data/vendor/faiss/faiss/index_io.h +16 -0
- data/vendor/faiss/faiss/invlists/DirectMap.cpp +5 -2
- data/vendor/faiss/faiss/invlists/InvertedLists.cpp +15 -15
- data/vendor/faiss/faiss/invlists/InvertedLists.h +2 -2
- data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.cpp +19 -4
- data/vendor/faiss/faiss/python/python_callbacks.cpp +3 -1
- data/vendor/faiss/faiss/svs/IndexSVSFaissUtils.h +60 -0
- data/vendor/faiss/faiss/svs/IndexSVSFlat.cpp +26 -1
- data/vendor/faiss/faiss/svs/IndexSVSFlat.h +13 -0
- data/vendor/faiss/faiss/svs/IndexSVSIVF.cpp +1 -1
- data/vendor/faiss/faiss/svs/IndexSVSIVFLeanVec.cpp +1 -1
- data/vendor/faiss/faiss/svs/IndexSVSVamana.cpp +150 -23
- data/vendor/faiss/faiss/svs/IndexSVSVamana.h +30 -7
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.cpp +3 -2
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.h +2 -1
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.cpp +65 -25
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.h +3 -2
- data/vendor/faiss/faiss/utils/approx_topk_hamming/approx_topk_hamming.h +1 -1
- data/vendor/faiss/faiss/utils/bf16.h +34 -0
- data/vendor/faiss/faiss/utils/distances.cpp +14 -2
- data/vendor/faiss/faiss/utils/distances_simd.cpp +4 -4
- data/vendor/faiss/faiss/utils/extra_distances.cpp +4 -14
- data/vendor/faiss/faiss/utils/extra_distances.h +1 -2
- data/vendor/faiss/faiss/utils/hamming.cpp +9 -9
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx2.cpp +2 -1
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx512_spr.cpp +15 -0
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512.h +6 -30
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512_spr.h +171 -0
- data/vendor/faiss/faiss/utils/partitioning.cpp +0 -2
- data/vendor/faiss/faiss/utils/quantize_lut.cpp +29 -8
- data/vendor/faiss/faiss/utils/rabitq_simd.h +202 -0
- data/vendor/faiss/faiss/utils/simd_impl/distances_avx2.cpp +0 -1
- data/vendor/faiss/faiss/utils/simd_impl/distances_avx512.cpp +263 -15
- data/vendor/faiss/faiss/utils/simd_impl/distances_rvv.cpp +160 -18
- data/vendor/faiss/faiss/utils/simd_impl/partitioning_simdlib256.h +14 -68
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx2.cpp +245 -0
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512.cpp +273 -0
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512_spr.cpp +435 -0
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_neon.cpp +11 -0
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_rvv.cpp +143 -6
- data/vendor/faiss/faiss/utils/simd_levels.cpp +56 -2
- data/vendor/faiss/faiss/utils/simd_levels.h +14 -0
- data/vendor/faiss/faiss/utils/utils.cpp +9 -27
- metadata +27 -2
|
@@ -20,23 +20,39 @@ namespace faiss {
|
|
|
20
20
|
namespace pq_code_distance {
|
|
21
21
|
|
|
22
22
|
template <class PQCodeDist>
|
|
23
|
-
struct PQDistanceComputer : FlatCodesDistanceComputer {
|
|
23
|
+
struct PQDistanceComputer final : FlatCodesDistanceComputer {
|
|
24
24
|
using PQDecoder = typename PQCodeDist::PQDecoder;
|
|
25
|
-
size_t d;
|
|
26
25
|
MetricType metric;
|
|
27
|
-
idx_t nb;
|
|
28
26
|
const ProductQuantizer& pq;
|
|
29
27
|
const float* sdc;
|
|
30
28
|
std::vector<float> precomputed_table;
|
|
31
|
-
size_t ndis;
|
|
32
|
-
const float* q;
|
|
33
29
|
|
|
34
30
|
float distance_to_code(const uint8_t* code) final {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
float dis = PQCodeDist::distance_single_code(
|
|
31
|
+
return PQCodeDist::distance_single_code(
|
|
38
32
|
pq.M, pq.nbits, precomputed_table.data(), code);
|
|
39
|
-
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
void distance_to_code_batch_4(
|
|
36
|
+
const uint8_t* c1,
|
|
37
|
+
const uint8_t* c2,
|
|
38
|
+
const uint8_t* c3,
|
|
39
|
+
const uint8_t* c4,
|
|
40
|
+
float& d1,
|
|
41
|
+
float& d2,
|
|
42
|
+
float& d3,
|
|
43
|
+
float& d4) override {
|
|
44
|
+
PQCodeDist::distance_four_codes(
|
|
45
|
+
pq.M,
|
|
46
|
+
pq.nbits,
|
|
47
|
+
precomputed_table.data(),
|
|
48
|
+
c1,
|
|
49
|
+
c2,
|
|
50
|
+
c3,
|
|
51
|
+
c4,
|
|
52
|
+
d1,
|
|
53
|
+
d2,
|
|
54
|
+
d3,
|
|
55
|
+
d4);
|
|
40
56
|
}
|
|
41
57
|
|
|
42
58
|
float symmetric_dis(idx_t i, idx_t j) override {
|
|
@@ -50,7 +66,6 @@ struct PQDistanceComputer : FlatCodesDistanceComputer {
|
|
|
50
66
|
accu += sdci[codei.decode() + (codej.decode() << codei.nbits)];
|
|
51
67
|
sdci += uint64_t(1) << (2 * codei.nbits);
|
|
52
68
|
}
|
|
53
|
-
ndis++;
|
|
54
69
|
return accu;
|
|
55
70
|
}
|
|
56
71
|
|
|
@@ -58,18 +73,14 @@ struct PQDistanceComputer : FlatCodesDistanceComputer {
|
|
|
58
73
|
: FlatCodesDistanceComputer(
|
|
59
74
|
storage.codes.data(),
|
|
60
75
|
storage.code_size),
|
|
61
|
-
pq(storage.pq)
|
|
62
|
-
q(nullptr) {
|
|
76
|
+
pq(storage.pq) {
|
|
63
77
|
precomputed_table.resize(pq.M * pq.ksub);
|
|
64
|
-
nb = storage.ntotal;
|
|
65
|
-
d = storage.d;
|
|
66
78
|
metric = storage.metric_type;
|
|
67
79
|
if (pq.sdc_table.size() == pq.ksub * pq.ksub * pq.M) {
|
|
68
80
|
sdc = pq.sdc_table.data();
|
|
69
81
|
} else {
|
|
70
82
|
sdc = nullptr;
|
|
71
83
|
}
|
|
72
|
-
ndis = 0;
|
|
73
84
|
}
|
|
74
85
|
|
|
75
86
|
void set_query(const float* x) override {
|
|
@@ -9,13 +9,15 @@
|
|
|
9
9
|
|
|
10
10
|
#define THE_SIMD_LEVEL SIMDLevel::AVX2
|
|
11
11
|
|
|
12
|
-
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
12
|
+
// NOLINTNEXTLINE(facebook-hte-InlineHeader,facebook-unused-include-check)
|
|
13
13
|
#include <faiss/impl/pq_code_distance/pq_code_distance-avx2.h>
|
|
14
|
-
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
14
|
+
// NOLINTNEXTLINE(facebook-hte-InlineHeader,facebook-unused-include-check)
|
|
15
15
|
#include <faiss/utils/hamming_distance/hamming_computer-avx2.h>
|
|
16
|
-
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
16
|
+
// NOLINTNEXTLINE(facebook-hte-InlineHeader,facebook-unused-include-check)
|
|
17
|
+
#include <faiss/impl/pq_code_distance/pq_scan_impl.h>
|
|
18
|
+
// NOLINTNEXTLINE(facebook-hte-InlineHeader,facebook-unused-include-check)
|
|
17
19
|
#include <faiss/impl/pq_code_distance/PQDistanceComputer_impl.h>
|
|
18
|
-
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
20
|
+
// NOLINTNEXTLINE(facebook-hte-InlineHeader,facebook-unused-include-check)
|
|
19
21
|
#include <faiss/impl/pq_code_distance/IVFPQScanner_impl.h>
|
|
20
22
|
|
|
21
23
|
#endif // COMPILE_SIMD_AVX2
|
|
@@ -14,6 +14,8 @@
|
|
|
14
14
|
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
15
15
|
#include <faiss/utils/hamming_distance/hamming_computer-avx512.h>
|
|
16
16
|
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
17
|
+
#include <faiss/impl/pq_code_distance/pq_scan_impl.h>
|
|
18
|
+
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
17
19
|
#include <faiss/impl/pq_code_distance/PQDistanceComputer_impl.h>
|
|
18
20
|
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
19
21
|
#include <faiss/impl/pq_code_distance/IVFPQScanner_impl.h>
|
|
@@ -14,6 +14,8 @@
|
|
|
14
14
|
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
15
15
|
#include <faiss/utils/hamming_distance/hamming_computer-neon.h>
|
|
16
16
|
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
17
|
+
#include <faiss/impl/pq_code_distance/pq_scan_impl.h>
|
|
18
|
+
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
17
19
|
#include <faiss/impl/pq_code_distance/PQDistanceComputer_impl.h>
|
|
18
20
|
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
19
21
|
#include <faiss/impl/pq_code_distance/IVFPQScanner_impl.h>
|
|
@@ -17,9 +17,29 @@
|
|
|
17
17
|
|
|
18
18
|
#include <faiss/impl/pq_code_distance/pq_code_distance-generic.h>
|
|
19
19
|
|
|
20
|
+
#define THE_SIMD_LEVEL SIMDLevel::NONE
|
|
21
|
+
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
22
|
+
#include <faiss/impl/pq_code_distance/pq_scan_impl.h>
|
|
23
|
+
#undef THE_SIMD_LEVEL
|
|
24
|
+
|
|
20
25
|
namespace faiss {
|
|
21
26
|
namespace pq_code_distance {
|
|
22
27
|
|
|
28
|
+
void pq_scan_8bit(
|
|
29
|
+
size_t M,
|
|
30
|
+
const float* dis_table,
|
|
31
|
+
const uint8_t* codes,
|
|
32
|
+
size_t ncodes,
|
|
33
|
+
size_t k,
|
|
34
|
+
float* heap_dis,
|
|
35
|
+
int64_t* heap_ids,
|
|
36
|
+
bool max_heap) {
|
|
37
|
+
with_simd_level([&]<SIMDLevel SL>() {
|
|
38
|
+
pq_scan_8bit_impl<SL>(
|
|
39
|
+
M, dis_table, codes, ncodes, k, heap_dis, heap_ids, max_heap);
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
|
|
23
43
|
float pq_code_distance_8bit_single(
|
|
24
44
|
size_t M,
|
|
25
45
|
const float* sim_table,
|
|
@@ -245,11 +245,47 @@ FAISS_API void pq_code_distance_8bit_four(
|
|
|
245
245
|
float& result2,
|
|
246
246
|
float& result3);
|
|
247
247
|
|
|
248
|
+
/*********************************************************************
|
|
249
|
+
* Standalone PQ scan — SIMD-dispatched full-index scan.
|
|
250
|
+
*
|
|
251
|
+
* Scans all ncodes PQ codes against a precomputed distance table,
|
|
252
|
+
* maintaining a k-nearest-neighbor heap. Uses the SIMD PQ distance
|
|
253
|
+
* kernels (AVX2 gathers, etc.) for the inner loop, with the SIMD
|
|
254
|
+
* gathers inlined into the scan loop in each per-SIMD TU.
|
|
255
|
+
*
|
|
256
|
+
* Definitions are in pq_scan_impl.h (per-SIMD TUs) and
|
|
257
|
+
* pq_code_distance-generic.cpp (dispatch wrapper).
|
|
258
|
+
*********************************************************************/
|
|
259
|
+
|
|
260
|
+
template <SIMDLevel SL>
|
|
261
|
+
void pq_scan_8bit_impl(
|
|
262
|
+
size_t M,
|
|
263
|
+
const float* dis_table,
|
|
264
|
+
const uint8_t* codes,
|
|
265
|
+
size_t ncodes,
|
|
266
|
+
size_t k,
|
|
267
|
+
float* heap_dis,
|
|
268
|
+
int64_t* heap_ids,
|
|
269
|
+
bool max_heap);
|
|
270
|
+
|
|
271
|
+
/// Scan all ncodes 8-bit PQ codes, dispatching to the best SIMD level.
|
|
272
|
+
/// max_heap=true for L2 (CMax), false for IP (CMin).
|
|
273
|
+
FAISS_API void pq_scan_8bit(
|
|
274
|
+
size_t M,
|
|
275
|
+
const float* dis_table,
|
|
276
|
+
const uint8_t* codes,
|
|
277
|
+
size_t ncodes,
|
|
278
|
+
size_t k,
|
|
279
|
+
float* heap_dis,
|
|
280
|
+
int64_t* heap_ids,
|
|
281
|
+
bool max_heap);
|
|
282
|
+
|
|
248
283
|
} // namespace pq_code_distance
|
|
249
284
|
|
|
250
285
|
// Re-export public API into namespace faiss for convenience
|
|
251
286
|
using pq_code_distance::pq_code_distance_8bit_four;
|
|
252
287
|
using pq_code_distance::pq_code_distance_8bit_single;
|
|
288
|
+
using pq_code_distance::pq_scan_8bit;
|
|
253
289
|
using pq_code_distance::PQCodeDistance;
|
|
254
290
|
using pq_code_distance::PQCodeDistanceScalar;
|
|
255
291
|
|
|
@@ -347,4 +347,9 @@ void pq_code_distance_8bit_four_impl<SIMDLevel::ARM_SVE>(
|
|
|
347
347
|
} // namespace pq_code_distance
|
|
348
348
|
} // namespace faiss
|
|
349
349
|
|
|
350
|
+
#define THE_SIMD_LEVEL SIMDLevel::ARM_SVE
|
|
351
|
+
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
352
|
+
#include <faiss/impl/pq_code_distance/pq_scan_impl.h>
|
|
353
|
+
#undef THE_SIMD_LEVEL
|
|
354
|
+
|
|
350
355
|
#endif // COMPILE_SIMD_ARM_SVE
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
3
|
+
*
|
|
4
|
+
* This source code is licensed under the MIT license found in the
|
|
5
|
+
* LICENSE file in the root directory of this source tree.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
#pragma once
|
|
9
|
+
|
|
10
|
+
#ifndef THE_SIMD_LEVEL
|
|
11
|
+
#error "THE_SIMD_LEVEL must be defined before including pq_scan_impl.h"
|
|
12
|
+
#endif
|
|
13
|
+
|
|
14
|
+
#include <faiss/impl/pq_code_distance/pq_code_distance-inl.h>
|
|
15
|
+
#include <faiss/utils/Heap.h>
|
|
16
|
+
|
|
17
|
+
namespace faiss {
|
|
18
|
+
namespace pq_code_distance {
|
|
19
|
+
|
|
20
|
+
namespace {
|
|
21
|
+
|
|
22
|
+
template <class C>
|
|
23
|
+
void pq_scan_8bit_at_level(
|
|
24
|
+
size_t M,
|
|
25
|
+
const float* dis_table,
|
|
26
|
+
const uint8_t* codes,
|
|
27
|
+
size_t ncodes,
|
|
28
|
+
size_t k,
|
|
29
|
+
float* heap_dis,
|
|
30
|
+
int64_t* heap_ids) {
|
|
31
|
+
const size_t code_size = M;
|
|
32
|
+
|
|
33
|
+
size_t j = 0;
|
|
34
|
+
for (; j + 3 < ncodes; j += 4) {
|
|
35
|
+
float d0, d1, d2, d3;
|
|
36
|
+
pq_code_distance_8bit_four_impl<THE_SIMD_LEVEL>( // NOLINT(facebook-modularize-issue-check)
|
|
37
|
+
M,
|
|
38
|
+
dis_table,
|
|
39
|
+
codes + j * code_size,
|
|
40
|
+
codes + (j + 1) * code_size,
|
|
41
|
+
codes + (j + 2) * code_size,
|
|
42
|
+
codes + (j + 3) * code_size,
|
|
43
|
+
d0,
|
|
44
|
+
d1,
|
|
45
|
+
d2,
|
|
46
|
+
d3);
|
|
47
|
+
|
|
48
|
+
if (C::cmp(heap_dis[0], d0)) {
|
|
49
|
+
heap_replace_top<C>(k, heap_dis, heap_ids, d0, j);
|
|
50
|
+
}
|
|
51
|
+
if (C::cmp(heap_dis[0], d1)) {
|
|
52
|
+
heap_replace_top<C>(k, heap_dis, heap_ids, d1, j + 1);
|
|
53
|
+
}
|
|
54
|
+
if (C::cmp(heap_dis[0], d2)) {
|
|
55
|
+
heap_replace_top<C>(k, heap_dis, heap_ids, d2, j + 2);
|
|
56
|
+
}
|
|
57
|
+
if (C::cmp(heap_dis[0], d3)) {
|
|
58
|
+
heap_replace_top<C>(k, heap_dis, heap_ids, d3, j + 3);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
for (; j < ncodes; j++) {
|
|
63
|
+
float dis = pq_code_distance_8bit_single_impl<THE_SIMD_LEVEL>( // NOLINT(facebook-modularize-issue-check)
|
|
64
|
+
M, dis_table, codes + j * code_size);
|
|
65
|
+
if (C::cmp(heap_dis[0], dis)) {
|
|
66
|
+
heap_replace_top<C>(k, heap_dis, heap_ids, dis, j);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
} // anonymous namespace
|
|
72
|
+
|
|
73
|
+
template <SIMDLevel SL>
|
|
74
|
+
void pq_scan_8bit_impl(
|
|
75
|
+
size_t M,
|
|
76
|
+
const float* dis_table,
|
|
77
|
+
const uint8_t* codes,
|
|
78
|
+
size_t ncodes,
|
|
79
|
+
size_t k,
|
|
80
|
+
float* heap_dis,
|
|
81
|
+
int64_t* heap_ids,
|
|
82
|
+
bool max_heap);
|
|
83
|
+
|
|
84
|
+
// NOLINTNEXTLINE(facebook-hte-MisplacedTemplateSpecialization,facebook-modularize-issue-check)
|
|
85
|
+
template <>
|
|
86
|
+
void pq_scan_8bit_impl<THE_SIMD_LEVEL>(
|
|
87
|
+
size_t M,
|
|
88
|
+
const float* dis_table,
|
|
89
|
+
const uint8_t* codes,
|
|
90
|
+
size_t ncodes,
|
|
91
|
+
size_t k,
|
|
92
|
+
float* heap_dis,
|
|
93
|
+
int64_t* heap_ids,
|
|
94
|
+
bool max_heap) {
|
|
95
|
+
if (max_heap) {
|
|
96
|
+
pq_scan_8bit_at_level<CMax<float, int64_t>>(
|
|
97
|
+
M, dis_table, codes, ncodes, k, heap_dis, heap_ids);
|
|
98
|
+
} else {
|
|
99
|
+
pq_scan_8bit_at_level<CMin<float, int64_t>>(
|
|
100
|
+
M, dis_table, codes, ncodes, k, heap_dis, heap_ids);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
} // namespace pq_code_distance
|
|
105
|
+
} // namespace faiss
|
|
@@ -56,6 +56,8 @@ void pq_code_distance_8bit_four_impl<SIMDLevel::RISCV_RVV>(
|
|
|
56
56
|
|
|
57
57
|
#define THE_SIMD_LEVEL SIMDLevel::RISCV_RVV
|
|
58
58
|
|
|
59
|
+
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
60
|
+
#include <faiss/impl/pq_code_distance/pq_scan_impl.h>
|
|
59
61
|
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
60
62
|
#include <faiss/utils/hamming_distance/hamming_computer-rvv.h>
|
|
61
63
|
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
3
|
+
*
|
|
4
|
+
* This source code is licensed under the MIT license found in the
|
|
5
|
+
* LICENSE file in the root directory of this source tree.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
// Generic (NONE) implementations of Top1 and Reservoir add_results, plus the
|
|
9
|
+
// runtime-dispatch method bodies. SIMD specialisations live in
|
|
10
|
+
// result_handler_avx2.cpp and result_handler_avx512.cpp.
|
|
11
|
+
|
|
12
|
+
#include <faiss/impl/ResultHandler.h>
|
|
13
|
+
#include <faiss/impl/simd_dispatch.h>
|
|
14
|
+
|
|
15
|
+
namespace faiss {
|
|
16
|
+
|
|
17
|
+
// ----------------------------------------------------------------
|
|
18
|
+
// SIMD-level masks
|
|
19
|
+
// ----------------------------------------------------------------
|
|
20
|
+
|
|
21
|
+
// Top-1: scalar fallback + AVX2 (8-wide) + AVX512 (16-wide).
|
|
22
|
+
constexpr int TOP1_SIMD_LEVELS = (1 << int(SIMDLevel::NONE)) |
|
|
23
|
+
(1 << int(SIMDLevel::AVX2)) | (1 << int(SIMDLevel::AVX512));
|
|
24
|
+
|
|
25
|
+
// Reservoir: scalar fallback + AVX512 compress path.
|
|
26
|
+
// VPCOMPRESSPS/VPCOMPRESSD require AVX512F so there is no AVX2 path.
|
|
27
|
+
// On non-AVX512 hosts the dispatch falls back to NONE automatically.
|
|
28
|
+
constexpr int RESERVOIR_SIMD_LEVELS =
|
|
29
|
+
(1 << int(SIMDLevel::NONE)) | (1 << int(SIMDLevel::AVX512));
|
|
30
|
+
|
|
31
|
+
// ----------------------------------------------------------------
|
|
32
|
+
// Scalar (NONE) helper implementations
|
|
33
|
+
// ----------------------------------------------------------------
|
|
34
|
+
|
|
35
|
+
namespace {
|
|
36
|
+
|
|
37
|
+
template <class C, bool use_sel>
|
|
38
|
+
void top1_add_results_none(
|
|
39
|
+
Top1BlockResultHandler<C, use_sel>* self,
|
|
40
|
+
size_t j0,
|
|
41
|
+
size_t j1,
|
|
42
|
+
const float* dis_tab_in) {
|
|
43
|
+
using T = typename C::T;
|
|
44
|
+
using TI = typename C::TI;
|
|
45
|
+
|
|
46
|
+
for (size_t qi = self->i0; qi < self->i1; qi++) {
|
|
47
|
+
const T* dis_tab_i = dis_tab_in + (j1 - j0) * (qi - self->i0) - j0;
|
|
48
|
+
|
|
49
|
+
// Hoist best_dis / best_idx into locals so the compiler keeps them in
|
|
50
|
+
// registers across the inner loop (no aliasing with dis_tab reads).
|
|
51
|
+
T best_dis = self->dis_tab[qi];
|
|
52
|
+
TI best_idx = self->ids_tab[qi];
|
|
53
|
+
|
|
54
|
+
for (size_t j = j0; j < j1; j++) {
|
|
55
|
+
if (C::cmp(best_dis, dis_tab_i[j])) {
|
|
56
|
+
best_dis = dis_tab_i[j];
|
|
57
|
+
best_idx = (TI)j;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
self->dis_tab[qi] = best_dis;
|
|
62
|
+
self->ids_tab[qi] = best_idx;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
template <class C, bool use_sel>
|
|
67
|
+
void reservoir_add_results_none(
|
|
68
|
+
ReservoirBlockResultHandler<C, use_sel>* self,
|
|
69
|
+
size_t j0,
|
|
70
|
+
size_t j1,
|
|
71
|
+
const float* dis_in) {
|
|
72
|
+
using T = typename C::T;
|
|
73
|
+
using TI = typename C::TI;
|
|
74
|
+
|
|
75
|
+
#pragma omp parallel for
|
|
76
|
+
for (int64_t qi = (int64_t)self->i0; qi < (int64_t)self->i1; qi++) {
|
|
77
|
+
ReservoirTopN<C>& res = self->reservoirs[qi - (int64_t)self->i0];
|
|
78
|
+
const T* dis_tab_i = dis_in + (j1 - j0) * (qi - (int64_t)self->i0) - j0;
|
|
79
|
+
|
|
80
|
+
// Hoist res.i and res.threshold into locals so the compiler keeps
|
|
81
|
+
// them in registers.
|
|
82
|
+
size_t ri = res.i;
|
|
83
|
+
T thresh = res.threshold;
|
|
84
|
+
|
|
85
|
+
for (size_t j = j0; j < j1; j++) {
|
|
86
|
+
T dis = dis_tab_i[j];
|
|
87
|
+
if (C::cmp(thresh, dis)) {
|
|
88
|
+
res.vals[ri] = dis;
|
|
89
|
+
res.ids[ri] = (TI)j;
|
|
90
|
+
ri++;
|
|
91
|
+
if (ri >= res.capacity) {
|
|
92
|
+
res.i = ri;
|
|
93
|
+
res.shrink_fuzzy();
|
|
94
|
+
ri = res.i;
|
|
95
|
+
thresh = res.threshold;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
res.i = ri;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
} // namespace
|
|
104
|
+
|
|
105
|
+
// ----------------------------------------------------------------
|
|
106
|
+
// SIMDLevel::NONE explicit specialisations
|
|
107
|
+
// ----------------------------------------------------------------
|
|
108
|
+
|
|
109
|
+
// Instantiate top1_add_results_tpl<C, use_sel, SIMDLevel::NONE> and
|
|
110
|
+
// reservoir_add_results_tpl<C, use_sel, SIMDLevel::NONE> for all
|
|
111
|
+
// (C, use_sel) combinations that the rest of FAISS uses.
|
|
112
|
+
#define INSTANTIATE_NONE(C, use_sel) \
|
|
113
|
+
template <> \
|
|
114
|
+
void top1_add_results_tpl<C, use_sel, SIMDLevel::NONE>( \
|
|
115
|
+
Top1BlockResultHandler<C, use_sel> * self, \
|
|
116
|
+
size_t j0, \
|
|
117
|
+
size_t j1, \
|
|
118
|
+
const float* dis_tab) { \
|
|
119
|
+
top1_add_results_none<C, use_sel>(self, j0, j1, dis_tab); \
|
|
120
|
+
} \
|
|
121
|
+
template <> \
|
|
122
|
+
void reservoir_add_results_tpl<C, use_sel, SIMDLevel::NONE>( \
|
|
123
|
+
ReservoirBlockResultHandler<C, use_sel> * self, \
|
|
124
|
+
size_t j0, \
|
|
125
|
+
size_t j1, \
|
|
126
|
+
const float* dis_in) { \
|
|
127
|
+
reservoir_add_results_none<C, use_sel>(self, j0, j1, dis_in); \
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// Type aliases so the comma in CMax<float, int64_t> doesn't split macro args.
|
|
131
|
+
using CMaxFI = CMax<float, int64_t>;
|
|
132
|
+
using CMinFI = CMin<float, int64_t>;
|
|
133
|
+
|
|
134
|
+
INSTANTIATE_NONE(CMaxFI, false)
|
|
135
|
+
INSTANTIATE_NONE(CMaxFI, true)
|
|
136
|
+
INSTANTIATE_NONE(CMinFI, false)
|
|
137
|
+
INSTANTIATE_NONE(CMinFI, true)
|
|
138
|
+
|
|
139
|
+
#undef INSTANTIATE_NONE
|
|
140
|
+
|
|
141
|
+
// ----------------------------------------------------------------
|
|
142
|
+
// add_results method definitions — dispatch to the right SL kernel
|
|
143
|
+
// ----------------------------------------------------------------
|
|
144
|
+
|
|
145
|
+
template <class C, bool use_sel>
|
|
146
|
+
void Top1BlockResultHandler<C, use_sel>::add_results(
|
|
147
|
+
size_t j0,
|
|
148
|
+
size_t j1,
|
|
149
|
+
const T* dis_tab_2) {
|
|
150
|
+
with_selected_simd_levels<TOP1_SIMD_LEVELS>([&]<SIMDLevel SL>() {
|
|
151
|
+
top1_add_results_tpl<C, use_sel, SL>(this, j0, j1, dis_tab_2);
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
template <class C, bool use_sel>
|
|
156
|
+
void ReservoirBlockResultHandler<C, use_sel>::add_results(
|
|
157
|
+
size_t j0,
|
|
158
|
+
size_t j1,
|
|
159
|
+
const T* dis_in) {
|
|
160
|
+
with_selected_simd_levels<RESERVOIR_SIMD_LEVELS>([&]<SIMDLevel SL>() {
|
|
161
|
+
reservoir_add_results_tpl<C, use_sel, SL>(this, j0, j1, dis_in);
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// ----------------------------------------------------------------
|
|
166
|
+
// Explicit class-template instantiations (force linkage)
|
|
167
|
+
// ----------------------------------------------------------------
|
|
168
|
+
|
|
169
|
+
template void Top1BlockResultHandler<CMax<float, int64_t>, false>::add_results(
|
|
170
|
+
size_t,
|
|
171
|
+
size_t,
|
|
172
|
+
const float*);
|
|
173
|
+
template void Top1BlockResultHandler<CMax<float, int64_t>, true>::add_results(
|
|
174
|
+
size_t,
|
|
175
|
+
size_t,
|
|
176
|
+
const float*);
|
|
177
|
+
template void Top1BlockResultHandler<CMin<float, int64_t>, false>::add_results(
|
|
178
|
+
size_t,
|
|
179
|
+
size_t,
|
|
180
|
+
const float*);
|
|
181
|
+
template void Top1BlockResultHandler<CMin<float, int64_t>, true>::add_results(
|
|
182
|
+
size_t,
|
|
183
|
+
size_t,
|
|
184
|
+
const float*);
|
|
185
|
+
|
|
186
|
+
template void ReservoirBlockResultHandler<CMax<float, int64_t>, false>::
|
|
187
|
+
add_results(size_t, size_t, const float*);
|
|
188
|
+
template void ReservoirBlockResultHandler<CMax<float, int64_t>, true>::
|
|
189
|
+
add_results(size_t, size_t, const float*);
|
|
190
|
+
template void ReservoirBlockResultHandler<CMin<float, int64_t>, false>::
|
|
191
|
+
add_results(size_t, size_t, const float*);
|
|
192
|
+
template void ReservoirBlockResultHandler<CMin<float, int64_t>, true>::
|
|
193
|
+
add_results(size_t, size_t, const float*);
|
|
194
|
+
|
|
195
|
+
} // namespace faiss
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
3
|
+
*
|
|
4
|
+
* This source code is licensed under the MIT license found in the
|
|
5
|
+
* LICENSE file in the root directory of this source tree.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
// AVX2 specialisation of Top1 add_results (8-wide branchless argmin/argmax).
|
|
9
|
+
// Reservoir stays on the NONE path — VPCOMPRESSPS requires AVX512F.
|
|
10
|
+
|
|
11
|
+
#ifdef COMPILE_SIMD_AVX2
|
|
12
|
+
|
|
13
|
+
#include <faiss/impl/ResultHandler.h>
|
|
14
|
+
|
|
15
|
+
#include <immintrin.h>
|
|
16
|
+
#include <type_traits>
|
|
17
|
+
|
|
18
|
+
namespace faiss {
|
|
19
|
+
|
|
20
|
+
namespace {
|
|
21
|
+
|
|
22
|
+
/// Templated AVX2 implementation of Top1 add_results for both CMax (keeps the
|
|
23
|
+
/// smallest distance) and CMin (keeps the largest similarity).
|
|
24
|
+
template <class C, bool use_sel>
|
|
25
|
+
void top1_add_results_avx2(
|
|
26
|
+
Top1BlockResultHandler<C, use_sel>* self,
|
|
27
|
+
size_t j0,
|
|
28
|
+
size_t j1,
|
|
29
|
+
const float* dis_tab_in) {
|
|
30
|
+
static_assert(
|
|
31
|
+
std::is_same<typename C::T, float>::value,
|
|
32
|
+
"This code expects float distances");
|
|
33
|
+
using TI = typename C::TI;
|
|
34
|
+
const __m256i vstep = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
|
|
35
|
+
|
|
36
|
+
for (size_t qi = self->i0; qi < self->i1; qi++) {
|
|
37
|
+
const float* dis_tab_i = dis_tab_in + (j1 - j0) * (qi - self->i0) - j0;
|
|
38
|
+
|
|
39
|
+
// Hoist best_dis / best_idx into locals so the compiler keeps them in
|
|
40
|
+
// registers across the inner loop (no aliasing with dis_tab reads).
|
|
41
|
+
float best_dis = self->dis_tab[qi];
|
|
42
|
+
TI best_idx = self->ids_tab[qi];
|
|
43
|
+
size_t j = j0;
|
|
44
|
+
|
|
45
|
+
__m256 vbest = _mm256_set1_ps(best_dis);
|
|
46
|
+
__m256i vbest_idx = _mm256_set1_epi32((int32_t)best_idx);
|
|
47
|
+
|
|
48
|
+
for (; j + 8 <= j1; j += 8) {
|
|
49
|
+
__m256 vdis = _mm256_loadu_ps(dis_tab_i + j);
|
|
50
|
+
__m256i vidx =
|
|
51
|
+
_mm256_add_epi32(_mm256_set1_epi32((int32_t)j), vstep);
|
|
52
|
+
|
|
53
|
+
// CMax (L2 nearest neighbour): keep lane if dis < best.
|
|
54
|
+
// CMin (inner product): keep lane if dis > best.
|
|
55
|
+
__m256 mask;
|
|
56
|
+
if constexpr (C::is_max) {
|
|
57
|
+
mask = _mm256_cmp_ps(vdis, vbest, _CMP_LT_OS);
|
|
58
|
+
} else {
|
|
59
|
+
mask = _mm256_cmp_ps(vdis, vbest, _CMP_GT_OS);
|
|
60
|
+
}
|
|
61
|
+
vbest = _mm256_blendv_ps(vbest, vdis, mask);
|
|
62
|
+
vbest_idx = _mm256_blendv_epi8(
|
|
63
|
+
vbest_idx, vidx, _mm256_castps_si256(mask));
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// Horizontal reduction across 8 lanes.
|
|
67
|
+
alignas(32) float best_arr[8];
|
|
68
|
+
alignas(32) int32_t idx_arr[8];
|
|
69
|
+
_mm256_store_ps(best_arr, vbest);
|
|
70
|
+
_mm256_store_si256((__m256i*)idx_arr, vbest_idx);
|
|
71
|
+
for (int k = 0; k < 8; k++) {
|
|
72
|
+
if (C::cmp(best_dis, best_arr[k])) {
|
|
73
|
+
best_dis = best_arr[k];
|
|
74
|
+
best_idx = (TI)idx_arr[k];
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Scalar tail.
|
|
79
|
+
for (; j < j1; j++) {
|
|
80
|
+
if (C::cmp(best_dis, dis_tab_i[j])) {
|
|
81
|
+
best_dis = dis_tab_i[j];
|
|
82
|
+
best_idx = (TI)j;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
self->dis_tab[qi] = best_dis;
|
|
87
|
+
self->ids_tab[qi] = best_idx;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
} // namespace
|
|
92
|
+
|
|
93
|
+
// Explicit specialisations for AVX2
|
|
94
|
+
|
|
95
|
+
template <>
|
|
96
|
+
void top1_add_results_tpl<CMax<float, int64_t>, false, SIMDLevel::AVX2>(
|
|
97
|
+
Top1BlockResultHandler<CMax<float, int64_t>, false>* self,
|
|
98
|
+
size_t j0,
|
|
99
|
+
size_t j1,
|
|
100
|
+
const float* dis_tab) {
|
|
101
|
+
top1_add_results_avx2<CMax<float, int64_t>, false>(self, j0, j1, dis_tab);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
template <>
|
|
105
|
+
void top1_add_results_tpl<CMax<float, int64_t>, true, SIMDLevel::AVX2>(
|
|
106
|
+
Top1BlockResultHandler<CMax<float, int64_t>, true>* self,
|
|
107
|
+
size_t j0,
|
|
108
|
+
size_t j1,
|
|
109
|
+
const float* dis_tab) {
|
|
110
|
+
top1_add_results_avx2<CMax<float, int64_t>, true>(self, j0, j1, dis_tab);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
template <>
|
|
114
|
+
void top1_add_results_tpl<CMin<float, int64_t>, false, SIMDLevel::AVX2>(
|
|
115
|
+
Top1BlockResultHandler<CMin<float, int64_t>, false>* self,
|
|
116
|
+
size_t j0,
|
|
117
|
+
size_t j1,
|
|
118
|
+
const float* dis_tab) {
|
|
119
|
+
top1_add_results_avx2<CMin<float, int64_t>, false>(self, j0, j1, dis_tab);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
template <>
|
|
123
|
+
void top1_add_results_tpl<CMin<float, int64_t>, true, SIMDLevel::AVX2>(
|
|
124
|
+
Top1BlockResultHandler<CMin<float, int64_t>, true>* self,
|
|
125
|
+
size_t j0,
|
|
126
|
+
size_t j1,
|
|
127
|
+
const float* dis_tab) {
|
|
128
|
+
top1_add_results_avx2<CMin<float, int64_t>, true>(self, j0, j1, dis_tab);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
} // namespace faiss
|
|
132
|
+
|
|
133
|
+
#endif // COMPILE_SIMD_AVX2
|