faiss 0.6.4 → 0.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/ext/faiss/extconf.rb +1 -1
- data/lib/faiss/version.rb +1 -1
- data/vendor/faiss/faiss/AutoTune.cpp +175 -0
- data/vendor/faiss/faiss/Clustering.cpp +150 -60
- data/vendor/faiss/faiss/Clustering.h +29 -0
- data/vendor/faiss/faiss/IVFlib.cpp +99 -0
- data/vendor/faiss/faiss/IVFlib.h +15 -0
- data/vendor/faiss/faiss/Index.cpp +44 -0
- data/vendor/faiss/faiss/Index.h +19 -14
- data/vendor/faiss/faiss/IndexBinary.cpp +8 -0
- data/vendor/faiss/faiss/IndexBinary.h +8 -0
- data/vendor/faiss/faiss/IndexBinaryHNSW.cpp +82 -4
- data/vendor/faiss/faiss/IndexBinaryHNSW.h +19 -0
- data/vendor/faiss/faiss/IndexBinaryIVF.cpp +34 -0
- data/vendor/faiss/faiss/IndexFastScan.cpp +33 -17
- data/vendor/faiss/faiss/IndexFastScan.h +6 -3
- data/vendor/faiss/faiss/IndexFlat.cpp +56 -0
- data/vendor/faiss/faiss/IndexFlat.h +31 -0
- data/vendor/faiss/faiss/IndexFlatCodes.cpp +6 -1
- data/vendor/faiss/faiss/IndexHNSW.cpp +456 -10
- data/vendor/faiss/faiss/IndexHNSW.h +38 -0
- data/vendor/faiss/faiss/IndexIDMap.cpp +32 -0
- data/vendor/faiss/faiss/IndexIDMap.h +7 -0
- data/vendor/faiss/faiss/IndexIVF.cpp +281 -31
- data/vendor/faiss/faiss/IndexIVF.h +53 -1
- data/vendor/faiss/faiss/IndexIVFFastScan.cpp +46 -14
- data/vendor/faiss/faiss/IndexIVFFastScan.h +4 -2
- data/vendor/faiss/faiss/IndexIVFFlat.cpp +33 -0
- data/vendor/faiss/faiss/IndexIVFFlat.h +16 -0
- data/vendor/faiss/faiss/IndexIVFPQFastScan.cpp +1 -0
- data/vendor/faiss/faiss/IndexIVFRaBitQ.cpp +3 -20
- data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.cpp +50 -14
- data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.h +5 -2
- data/vendor/faiss/faiss/IndexIVFSQFastScan.cpp +855 -0
- data/vendor/faiss/faiss/IndexIVFSQFastScan.h +145 -0
- data/vendor/faiss/faiss/IndexNNDescent.cpp +0 -3
- data/vendor/faiss/faiss/IndexPiPNN.cpp +201 -0
- data/vendor/faiss/faiss/IndexPiPNN.h +83 -0
- data/vendor/faiss/faiss/IndexRaBitQFastScan.cpp +6 -3
- data/vendor/faiss/faiss/IndexRaBitQFastScan.h +10 -4
- data/vendor/faiss/faiss/IndexReplicas.cpp +20 -0
- data/vendor/faiss/faiss/IndexReplicas.h +9 -0
- data/vendor/faiss/faiss/IndexSQFastScan.cpp +648 -0
- data/vendor/faiss/faiss/IndexSQFastScan.h +129 -0
- data/vendor/faiss/faiss/IndexShards.cpp +10 -0
- data/vendor/faiss/faiss/IndexShards.h +5 -0
- data/vendor/faiss/faiss/SuperKMeans.cpp +57 -15
- data/vendor/faiss/faiss/SuperKMeans.h +9 -4
- data/vendor/faiss/faiss/VectorTransform.cpp +3 -2
- data/vendor/faiss/faiss/clone_index.cpp +24 -0
- data/vendor/faiss/faiss/gpu/Dataset.h +112 -0
- data/vendor/faiss/faiss/gpu/GpuCloner.cpp +16 -13
- data/vendor/faiss/faiss/gpu/GpuCloner.h +4 -0
- data/vendor/faiss/faiss/gpu/GpuClonerOptions.h +3 -0
- data/vendor/faiss/faiss/gpu/GpuIndex.h +4 -13
- data/vendor/faiss/faiss/gpu/GpuIndexBinaryCagra.h +26 -13
- data/vendor/faiss/faiss/gpu/GpuIndexCagra.h +45 -22
- data/vendor/faiss/faiss/gpu/GpuIndexFlat.h +11 -0
- data/vendor/faiss/faiss/gpu/GpuIndexIVF.h +6 -0
- data/vendor/faiss/faiss/gpu/test/TestGpuDatasetHeader.cpp +18 -0
- data/vendor/faiss/faiss/gpu/test/TestGpuIndexFlat.cpp +215 -0
- data/vendor/faiss/faiss/gpu/test/TestGpuIndexIVFFlat.cpp +98 -0
- data/vendor/faiss/faiss/gpu/utils/DeviceUtils.h +11 -0
- data/vendor/faiss/faiss/gpu/utils/StackDeviceMemory.cpp +1 -1
- data/vendor/faiss/faiss/gpu_metal/MetalCloner.h +11 -0
- data/vendor/faiss/faiss/gpu_metal/MetalDistance.h +92 -0
- data/vendor/faiss/faiss/gpu_metal/MetalIndex.h +4 -0
- data/vendor/faiss/faiss/gpu_metal/MetalIndexBinaryFlat.h +69 -0
- data/vendor/faiss/faiss/gpu_metal/MetalIndexIVFPQ.h +38 -0
- data/vendor/faiss/faiss/gpu_metal/MetalKernels.h +106 -0
- data/vendor/faiss/faiss/gpu_metal/MetalPythonBridge.h +10 -0
- data/vendor/faiss/faiss/impl/AuxIndexStructures.cpp +2 -0
- data/vendor/faiss/faiss/impl/AuxIndexStructures.h +1 -6
- data/vendor/faiss/faiss/impl/ClusteringHelpers.cpp +134 -45
- data/vendor/faiss/faiss/impl/ClusteringHelpers.h +36 -0
- data/vendor/faiss/faiss/impl/ClusteringInitialization.cpp +286 -136
- data/vendor/faiss/faiss/impl/DistanceComputer.cpp +63 -0
- data/vendor/faiss/faiss/impl/DistanceComputer.h +3 -29
- data/vendor/faiss/faiss/impl/HNSW.cpp +46 -10
- data/vendor/faiss/faiss/impl/HNSW.h +72 -0
- data/vendor/faiss/faiss/impl/LocalSearchQuantizer.cpp +6 -19
- data/vendor/faiss/faiss/impl/NNDescent.cpp +2 -1
- data/vendor/faiss/faiss/impl/NSG.cpp +3 -2
- data/vendor/faiss/faiss/impl/Panorama.h +3 -2
- data/vendor/faiss/faiss/impl/PiPNN.cpp +822 -0
- data/vendor/faiss/faiss/impl/PiPNN.h +122 -0
- data/vendor/faiss/faiss/impl/ProductQuantizer.cpp +54 -11
- data/vendor/faiss/faiss/impl/RaBitQUtils.h +1 -1
- data/vendor/faiss/faiss/impl/RaBitQuantizer.cpp +81 -57
- data/vendor/faiss/faiss/impl/RaBitQuantizer.h +15 -0
- data/vendor/faiss/faiss/impl/ScalarQuantizer.cpp +29 -0
- data/vendor/faiss/faiss/impl/ScalarQuantizer.h +1 -6
- data/vendor/faiss/faiss/impl/approx_topk/avx2.cpp +0 -4
- data/vendor/faiss/faiss/impl/approx_topk/neon.cpp +0 -4
- data/vendor/faiss/faiss/impl/binary_hamming/IndexBinaryHash_impl.h +5 -2
- data/vendor/faiss/faiss/impl/binary_hamming/avx2.cpp +0 -4
- data/vendor/faiss/faiss/impl/binary_hamming/avx512.cpp +0 -4
- data/vendor/faiss/faiss/impl/binary_hamming/neon.cpp +0 -4
- data/vendor/faiss/faiss/impl/binary_hamming/rvv.cpp +0 -4
- data/vendor/faiss/faiss/impl/expanded_scanners.h +88 -0
- data/vendor/faiss/faiss/impl/fast_scan/FastScanDistancePostProcessing.h +4 -0
- data/vendor/faiss/faiss/impl/fast_scan/accumulate_loops.h +61 -64
- data/vendor/faiss/faiss/impl/fast_scan/accumulate_loops_512.h +13 -14
- data/vendor/faiss/faiss/impl/fast_scan/decompose_qbs.h +17 -129
- data/vendor/faiss/faiss/impl/fast_scan/dispatching.h +25 -19
- data/vendor/faiss/faiss/impl/fast_scan/fast_scan.cpp +5 -0
- data/vendor/faiss/faiss/impl/fast_scan/impl-avx2.cpp +0 -4
- data/vendor/faiss/faiss/impl/fast_scan/impl-avx512.cpp +0 -4
- data/vendor/faiss/faiss/impl/fast_scan/impl-neon.cpp +0 -88
- data/vendor/faiss/faiss/impl/fast_scan/impl-riscv.cpp +0 -4
- data/vendor/faiss/faiss/impl/fast_scan/kernels_neon.h +155 -0
- data/vendor/faiss/faiss/impl/fast_scan/kernels_simd256.h +22 -1
- data/vendor/faiss/faiss/impl/fast_scan/kernels_simd512.h +3 -2
- data/vendor/faiss/faiss/impl/fast_scan/simd_result_handlers.h +41 -23
- data/vendor/faiss/faiss/impl/hnsw/avx2.cpp +0 -4
- data/vendor/faiss/faiss/impl/hnsw/avx512.cpp +0 -4
- data/vendor/faiss/faiss/impl/index_read.cpp +172 -0
- data/vendor/faiss/faiss/impl/index_write.cpp +95 -0
- data/vendor/faiss/faiss/impl/pipnn/HashPrune.cpp +159 -0
- data/vendor/faiss/faiss/impl/pipnn/HashPrune.h +69 -0
- data/vendor/faiss/faiss/impl/pipnn/Partition.cpp +587 -0
- data/vendor/faiss/faiss/impl/pipnn/Partition.h +93 -0
- data/vendor/faiss/faiss/impl/pipnn/kernels.cpp +326 -0
- data/vendor/faiss/faiss/impl/pipnn/kernels.h +92 -0
- data/vendor/faiss/faiss/impl/platform_macros.h +45 -0
- data/vendor/faiss/faiss/impl/polysemous_training/avx512.cpp +0 -4
- data/vendor/faiss/faiss/impl/pq_code_distance/IVFPQScanner_impl.h +79 -4
- data/vendor/faiss/faiss/impl/pq_code_distance/PQDistanceComputer_impl.h +6 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/avx2.cpp +0 -4
- data/vendor/faiss/faiss/impl/pq_code_distance/avx512.cpp +0 -4
- data/vendor/faiss/faiss/impl/pq_code_distance/neon.cpp +0 -4
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-avx2.h +4 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-avx512.h +4 -1
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-generic.cpp +19 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-generic.h +6 -6
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-inl.h +131 -6
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-sve.cpp +0 -4
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_rvv_kernel.h +89 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_scan_impl.h +21 -10
- data/vendor/faiss/faiss/impl/pq_code_distance/rvv.cpp +45 -18
- data/vendor/faiss/faiss/impl/residual_quantizer_encode_steps.cpp +2 -1
- data/vendor/faiss/faiss/impl/result_handler/avx2.cpp +0 -4
- data/vendor/faiss/faiss/impl/result_handler/avx512.cpp +0 -4
- data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx2.cpp +0 -4
- data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx512.cpp +0 -4
- data/vendor/faiss/faiss/impl/scalar_quantizer/distance_computers.h +5 -1
- data/vendor/faiss/faiss/impl/scalar_quantizer/quantizers.h +28 -52
- data/vendor/faiss/faiss/impl/scalar_quantizer/scanners.h +59 -11
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx2.cpp +226 -77
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-impl.h +482 -103
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-spr.cpp +2 -149
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512.cpp +3 -698
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-dispatch.h +66 -57
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-neon.cpp +101 -96
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-rvv.cpp +3185 -144
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq8_coefficients.h +48 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq8_rvv_kernel.h +546 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/training.cpp +2 -2
- data/vendor/faiss/faiss/impl/simd_dispatch.h +7 -0
- data/vendor/faiss/faiss/impl/simdlib/simdlib_avx2.h +14 -0
- data/vendor/faiss/faiss/impl/simdlib/simdlib_dispatch.h +7 -8
- data/vendor/faiss/faiss/impl/sq_fastscan_utils.h +232 -0
- data/vendor/faiss/faiss/index_factory.cpp +76 -0
- data/vendor/faiss/faiss/invlists/InvertedLists.cpp +3 -1
- data/vendor/faiss/faiss/svs/IndexSVSVamana.h +3 -2
- data/vendor/faiss/faiss/utils/AlignedTable.h +6 -1
- data/vendor/faiss/faiss/utils/Heap.cpp +4 -3
- data/vendor/faiss/faiss/utils/approx_topk_hamming/approx_topk_hamming.h +16 -5
- data/vendor/faiss/faiss/utils/bf16.h +36 -0
- data/vendor/faiss/faiss/utils/distances.cpp +226 -15
- data/vendor/faiss/faiss/utils/distances.h +34 -0
- data/vendor/faiss/faiss/utils/distances_dispatch.h +6 -4
- data/vendor/faiss/faiss/utils/distances_fused/avx512.cpp +2 -5
- data/vendor/faiss/faiss/utils/distances_fused/distances_fused.cpp +0 -14
- data/vendor/faiss/faiss/utils/distances_fused/rvv.cpp +25 -0
- data/vendor/faiss/faiss/utils/distances_fused/simdlib_based.cpp +0 -4
- data/vendor/faiss/faiss/utils/distances_fused/simdlib_based_neon.cpp +0 -4
- data/vendor/faiss/faiss/utils/distances_fused/simdlib_kernel-inl.h +2 -1
- data/vendor/faiss/faiss/utils/distances_simd.cpp +9 -0
- data/vendor/faiss/faiss/utils/fp16-fp16c.h +4 -0
- data/vendor/faiss/faiss/utils/fp16-inl.h +26 -52
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx2.cpp +0 -4
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx512.cpp +0 -4
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx512_vpopcnt.cpp +0 -4
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512.h +33 -5
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512_vpopcnt.h +15 -30
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-neon.h +0 -4
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_impl.h +2 -1
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_neon.cpp +0 -4
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_rvv.cpp +0 -4
- data/vendor/faiss/faiss/utils/pq_code_distance.h +8 -4
- data/vendor/faiss/faiss/utils/prefetch.h +1 -26
- data/vendor/faiss/faiss/utils/quantize_lut.cpp +26 -27
- data/vendor/faiss/faiss/utils/quantize_lut.h +11 -0
- data/vendor/faiss/faiss/utils/rabitq_simd.h +51 -16
- data/vendor/faiss/faiss/utils/simd_impl/distances_aarch64.cpp +9 -4
- data/vendor/faiss/faiss/utils/simd_impl/distances_arm_sve.cpp +37 -5
- data/vendor/faiss/faiss/utils/simd_impl/distances_avx2.cpp +64 -5
- data/vendor/faiss/faiss/utils/simd_impl/distances_avx512.cpp +64 -5
- data/vendor/faiss/faiss/utils/simd_impl/distances_rvv.cpp +353 -71
- data/vendor/faiss/faiss/utils/simd_impl/distances_rvv_kernel.h +68 -0
- data/vendor/faiss/faiss/utils/simd_impl/exhaustive_L2sqr_blas_cmax.h +93 -0
- data/vendor/faiss/faiss/utils/simd_impl/fp16_kernels.h +76 -0
- data/vendor/faiss/faiss/utils/simd_impl/fp16_kernels_avx2.cpp +52 -0
- data/vendor/faiss/faiss/utils/simd_impl/fp16_kernels_avx512.cpp +52 -0
- data/vendor/faiss/faiss/utils/simd_impl/fp16_kernels_neon.cpp +48 -0
- data/vendor/faiss/faiss/utils/simd_impl/masked_sum-avx2.h +44 -0
- data/vendor/faiss/faiss/utils/simd_impl/masked_sum-avx512.h +43 -0
- data/vendor/faiss/faiss/utils/simd_impl/masked_sum.h +38 -0
- data/vendor/faiss/faiss/utils/simd_impl/partitioning_avx2.cpp +0 -4
- data/vendor/faiss/faiss/utils/simd_impl/partitioning_neon.cpp +0 -4
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx2.cpp +54 -15
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512.cpp +38 -21
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512_vpopcnt.cpp +51 -14
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_neon.cpp +297 -10
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_rvv.cpp +0 -4
- data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels_avx2.cpp +0 -4
- data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels_avx512.cpp +0 -4
- data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels_sve.cpp +0 -4
- data/vendor/faiss/faiss/utils/simd_levels.cpp +19 -3
- data/vendor/faiss/faiss/utils/simd_levels.h +74 -1
- data/vendor/faiss/faiss/utils/transpose/transpose-avx2-inl.h +0 -4
- data/vendor/faiss/faiss/utils/transpose/transpose-avx512-inl.h +0 -4
- data/vendor/faiss/faiss/utils/utils.cpp +80 -0
- data/vendor/faiss/faiss/utils/utils.h +24 -0
- metadata +33 -1
|
@@ -5,8 +5,6 @@
|
|
|
5
5
|
* LICENSE file in the root directory of this source tree.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
#ifdef COMPILE_SIMD_AVX2
|
|
9
|
-
|
|
10
8
|
#define THE_LEVEL_TO_DISPATCH SIMDLevel::AVX2
|
|
11
9
|
#include <faiss/impl/fast_scan/dispatching.h> // IWYU pragma: keep
|
|
12
10
|
#include <faiss/impl/fast_scan/rabitq_dispatching.h> // IWYU pragma: keep
|
|
@@ -32,5 +30,3 @@ void accumulate_to_mem_impl<SIMDLevel::AVX2>(
|
|
|
32
30
|
}
|
|
33
31
|
|
|
34
32
|
} // namespace faiss
|
|
35
|
-
|
|
36
|
-
#endif // COMPILE_SIMD_AVX2
|
|
@@ -5,8 +5,6 @@
|
|
|
5
5
|
* LICENSE file in the root directory of this source tree.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
#ifdef COMPILE_SIMD_AVX512
|
|
9
|
-
|
|
10
8
|
#define THE_LEVEL_TO_DISPATCH SIMDLevel::AVX512
|
|
11
9
|
#include <faiss/impl/fast_scan/dispatching.h> // IWYU pragma: keep
|
|
12
10
|
#include <faiss/impl/fast_scan/rabitq_dispatching.h> // IWYU pragma: keep
|
|
@@ -36,5 +34,3 @@ void accumulate_to_mem_impl<SIMDLevel::AVX512>(
|
|
|
36
34
|
}
|
|
37
35
|
|
|
38
36
|
} // namespace faiss
|
|
39
|
-
|
|
40
|
-
#endif // COMPILE_SIMD_AVX512
|
|
@@ -5,8 +5,6 @@
|
|
|
5
5
|
* LICENSE file in the root directory of this source tree.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
#ifdef COMPILE_SIMD_ARM_NEON
|
|
9
|
-
|
|
10
8
|
#define THE_LEVEL_TO_DISPATCH SIMDLevel::ARM_NEON
|
|
11
9
|
#include <faiss/impl/fast_scan/dispatching.h> // IWYU pragma: keep
|
|
12
10
|
#include <faiss/impl/fast_scan/rabitq_dispatching.h> // IWYU pragma: keep
|
|
@@ -32,89 +30,3 @@ void accumulate_to_mem_impl<SIMDLevel::ARM_NEON>(
|
|
|
32
30
|
}
|
|
33
31
|
|
|
34
32
|
} // namespace faiss
|
|
35
|
-
|
|
36
|
-
// ARM_SVE: forward to ARM_NEON implementation until a dedicated SVE
|
|
37
|
-
// specialization is written (same pattern as scalar_quantizer/sq-neon.cpp).
|
|
38
|
-
#ifdef COMPILE_SIMD_ARM_SVE
|
|
39
|
-
|
|
40
|
-
namespace faiss {
|
|
41
|
-
|
|
42
|
-
template <>
|
|
43
|
-
std::unique_ptr<FastScanCodeScanner> make_fast_scan_scanner_impl<
|
|
44
|
-
SIMDLevel::ARM_SVE>(
|
|
45
|
-
bool is_max,
|
|
46
|
-
int impl,
|
|
47
|
-
size_t nq,
|
|
48
|
-
size_t ntotal,
|
|
49
|
-
int64_t k,
|
|
50
|
-
float* distances,
|
|
51
|
-
int64_t* ids,
|
|
52
|
-
const IDSelector* sel,
|
|
53
|
-
bool with_id_map) {
|
|
54
|
-
return make_fast_scan_scanner_impl<SIMDLevel::ARM_NEON>(
|
|
55
|
-
is_max, impl, nq, ntotal, k, distances, ids, sel, with_id_map);
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
template <>
|
|
59
|
-
std::unique_ptr<FastScanCodeScanner> make_range_scanner_impl<
|
|
60
|
-
SIMDLevel::ARM_SVE>(
|
|
61
|
-
bool is_max,
|
|
62
|
-
RangeSearchResult& rres,
|
|
63
|
-
float radius,
|
|
64
|
-
size_t ntotal,
|
|
65
|
-
const IDSelector* sel) {
|
|
66
|
-
return make_range_scanner_impl<SIMDLevel::ARM_NEON>(
|
|
67
|
-
is_max, rres, radius, ntotal, sel);
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
template <>
|
|
71
|
-
std::unique_ptr<FastScanCodeScanner> make_partial_range_scanner_impl<
|
|
72
|
-
SIMDLevel::ARM_SVE>(
|
|
73
|
-
bool is_max,
|
|
74
|
-
RangeSearchPartialResult& pres,
|
|
75
|
-
float radius,
|
|
76
|
-
size_t ntotal,
|
|
77
|
-
size_t q0,
|
|
78
|
-
size_t q1,
|
|
79
|
-
const IDSelector* sel) {
|
|
80
|
-
return make_partial_range_scanner_impl<SIMDLevel::ARM_NEON>(
|
|
81
|
-
is_max, pres, radius, ntotal, q0, q1, sel);
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
template <>
|
|
85
|
-
std::unique_ptr<FastScanCodeScanner> rabitq_make_knn_scanner_impl<
|
|
86
|
-
SIMDLevel::ARM_SVE>(
|
|
87
|
-
const IndexRaBitQFastScan* index,
|
|
88
|
-
bool is_max,
|
|
89
|
-
size_t nq,
|
|
90
|
-
int64_t k,
|
|
91
|
-
float* distances,
|
|
92
|
-
int64_t* ids,
|
|
93
|
-
const IDSelector* sel,
|
|
94
|
-
const FastScanDistancePostProcessing& context,
|
|
95
|
-
bool is_multi_bit) {
|
|
96
|
-
return rabitq_make_knn_scanner_impl<SIMDLevel::ARM_NEON>(
|
|
97
|
-
index, is_max, nq, k, distances, ids, sel, context, is_multi_bit);
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
template <>
|
|
101
|
-
std::unique_ptr<FastScanCodeScanner> rabitq_ivf_make_knn_scanner_impl<
|
|
102
|
-
SIMDLevel::ARM_SVE>(
|
|
103
|
-
bool is_max,
|
|
104
|
-
const IndexIVFRaBitQFastScan* index,
|
|
105
|
-
size_t nq,
|
|
106
|
-
size_t k,
|
|
107
|
-
float* distances,
|
|
108
|
-
int64_t* ids,
|
|
109
|
-
const IDSelector* sel,
|
|
110
|
-
const FastScanDistancePostProcessing* context,
|
|
111
|
-
bool multi_bit) {
|
|
112
|
-
return rabitq_ivf_make_knn_scanner_impl<SIMDLevel::ARM_NEON>(
|
|
113
|
-
is_max, index, nq, k, distances, ids, sel, context, multi_bit);
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
} // namespace faiss
|
|
117
|
-
|
|
118
|
-
#endif // COMPILE_SIMD_ARM_SVE
|
|
119
|
-
|
|
120
|
-
#endif // COMPILE_SIMD_ARM_NEON
|
|
@@ -8,8 +8,6 @@
|
|
|
8
8
|
// RISC-V RVV: forward all fast_scan specializations to NONE until
|
|
9
9
|
// dedicated RVV implementations are written.
|
|
10
10
|
|
|
11
|
-
#ifdef COMPILE_SIMD_RISCV_RVV
|
|
12
|
-
|
|
13
11
|
#include <faiss/impl/fast_scan/fast_scan.h>
|
|
14
12
|
|
|
15
13
|
namespace faiss {
|
|
@@ -100,5 +98,3 @@ std::unique_ptr<FastScanCodeScanner> rabitq_ivf_make_knn_scanner_impl<
|
|
|
100
98
|
}
|
|
101
99
|
|
|
102
100
|
} // namespace faiss
|
|
103
|
-
|
|
104
|
-
#endif // COMPILE_SIMD_RISCV_RVV
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Portions Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
3
|
+
*
|
|
4
|
+
* This source code is licensed under the MIT license found in the
|
|
5
|
+
* LICENSE file in the root directory of this source tree.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/*
|
|
9
|
+
* Portions Copyright 2026 Arm Limited and/or its affiliates
|
|
10
|
+
* <open-source-office@arm.com>
|
|
11
|
+
*
|
|
12
|
+
* This source code is licensed under the MIT license found in the
|
|
13
|
+
* LICENSE file in the root directory of this source tree.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
#pragma once
|
|
17
|
+
|
|
18
|
+
#include <arm_neon.h>
|
|
19
|
+
|
|
20
|
+
#include <faiss/impl/fast_scan/kernels_simd256.h>
|
|
21
|
+
#include <faiss/impl/platform_macros.h>
|
|
22
|
+
#include <faiss/impl/simdlib/simdlib_dispatch.h>
|
|
23
|
+
|
|
24
|
+
namespace faiss {
|
|
25
|
+
|
|
26
|
+
/*
|
|
27
|
+
* Single-BB QBS variant: accumulates NQ queries x 32 db elements (BB=1).
|
|
28
|
+
* Uses native 128-bit AdvSIMD registers instead of logical 256-bit values.
|
|
29
|
+
*/
|
|
30
|
+
template <int NQ, class ResultHandler, class Scaler>
|
|
31
|
+
FAISS_ALWAYS_INLINE void pq4_kernel_qbs_neon(
|
|
32
|
+
int nsq,
|
|
33
|
+
const uint8_t* codes,
|
|
34
|
+
const uint8_t* LUT,
|
|
35
|
+
ResultHandler& res,
|
|
36
|
+
const Scaler& scaler) {
|
|
37
|
+
// Avoid a zero-length array for NQ=0 instantiations.
|
|
38
|
+
constexpr int NQA = NQ > 0 ? NQ : 1;
|
|
39
|
+
const uint16x8_t zero = vdupq_n_u16(0);
|
|
40
|
+
// accu[q][b] holds distances for vectors 8*b..8*b+7.
|
|
41
|
+
uint16x8_t accu[NQA][4];
|
|
42
|
+
|
|
43
|
+
for (int q = 0; q < NQ; q++) {
|
|
44
|
+
for (int b = 0; b < 4; b++) {
|
|
45
|
+
accu[q][b] = zero;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Process two subquantizers per iteration.
|
|
50
|
+
for (int sq = 0; sq < nsq - scaler.nscale; sq += 2) {
|
|
51
|
+
const uint8x16_t code0 = vld1q_u8(codes);
|
|
52
|
+
const uint8x16_t code1 = vld1q_u8(codes + 16);
|
|
53
|
+
codes += 32;
|
|
54
|
+
|
|
55
|
+
const uint8x16_t mask = vdupq_n_u8(0x0f);
|
|
56
|
+
const uint8x16_t code0_lo = vandq_u8(code0, mask);
|
|
57
|
+
const uint8x16_t code1_lo = vandq_u8(code1, mask);
|
|
58
|
+
const uint8x16_t code0_hi = vshrq_n_u8(code0, 4);
|
|
59
|
+
const uint8x16_t code1_hi = vshrq_n_u8(code1, 4);
|
|
60
|
+
|
|
61
|
+
for (int q = 0; q < NQ; q++) {
|
|
62
|
+
// Load LUTs for two subquantizers.
|
|
63
|
+
const uint8x16_t lut0 = vld1q_u8(LUT);
|
|
64
|
+
const uint8x16_t lut1 = vld1q_u8(LUT + 16);
|
|
65
|
+
LUT += 32;
|
|
66
|
+
|
|
67
|
+
const uint8x16_t d0_lo = vqtbl1q_u8(lut0, code0_lo);
|
|
68
|
+
const uint8x16_t d0_hi = vqtbl1q_u8(lut0, code0_hi);
|
|
69
|
+
const uint8x16_t d1_lo = vqtbl1q_u8(lut1, code1_lo);
|
|
70
|
+
const uint8x16_t d1_hi = vqtbl1q_u8(lut1, code1_hi);
|
|
71
|
+
|
|
72
|
+
accu[q][0] = vaddq_u16(
|
|
73
|
+
accu[q][0],
|
|
74
|
+
vaddl_u8(vget_low_u8(d0_lo), vget_low_u8(d1_lo)));
|
|
75
|
+
accu[q][1] = vaddq_u16(accu[q][1], vaddl_high_u8(d0_lo, d1_lo));
|
|
76
|
+
accu[q][2] = vaddq_u16(
|
|
77
|
+
accu[q][2],
|
|
78
|
+
vaddl_u8(vget_low_u8(d0_hi), vget_low_u8(d1_hi)));
|
|
79
|
+
accu[q][3] = vaddq_u16(accu[q][3], vaddl_high_u8(d0_hi, d1_hi));
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Scaled tail used by additive-quantizer FastScan indexes.
|
|
84
|
+
// (this is compiled out for DummyScaler)
|
|
85
|
+
for (int sq = 0; sq < scaler.nscale; sq += 2) {
|
|
86
|
+
const uint16_t scale = scaler.scale_one(uint16_t{1});
|
|
87
|
+
const uint8x16_t code0 = vld1q_u8(codes);
|
|
88
|
+
const uint8x16_t code1 = vld1q_u8(codes + 16);
|
|
89
|
+
codes += 32;
|
|
90
|
+
|
|
91
|
+
const uint8x16_t mask = vdupq_n_u8(0x0f);
|
|
92
|
+
const uint8x16_t code0_lo = vandq_u8(code0, mask);
|
|
93
|
+
const uint8x16_t code1_lo = vandq_u8(code1, mask);
|
|
94
|
+
const uint8x16_t code0_hi = vshrq_n_u8(code0, 4);
|
|
95
|
+
const uint8x16_t code1_hi = vshrq_n_u8(code1, 4);
|
|
96
|
+
|
|
97
|
+
for (int q = 0; q < NQ; q++) {
|
|
98
|
+
// Load LUTs for two subquantizers.
|
|
99
|
+
const uint8x16_t lut0 = vld1q_u8(LUT);
|
|
100
|
+
const uint8x16_t lut1 = vld1q_u8(LUT + 16);
|
|
101
|
+
LUT += 32;
|
|
102
|
+
|
|
103
|
+
const uint8x16_t d0_lo = vqtbl1q_u8(lut0, code0_lo);
|
|
104
|
+
const uint8x16_t d0_hi = vqtbl1q_u8(lut0, code0_hi);
|
|
105
|
+
const uint8x16_t d1_lo = vqtbl1q_u8(lut1, code1_lo);
|
|
106
|
+
const uint8x16_t d1_hi = vqtbl1q_u8(lut1, code1_hi);
|
|
107
|
+
|
|
108
|
+
accu[q][0] = vaddq_u16(
|
|
109
|
+
accu[q][0],
|
|
110
|
+
vmulq_n_u16(
|
|
111
|
+
vaddl_u8(vget_low_u8(d0_lo), vget_low_u8(d1_lo)),
|
|
112
|
+
scale));
|
|
113
|
+
accu[q][1] = vaddq_u16(
|
|
114
|
+
accu[q][1],
|
|
115
|
+
vmulq_n_u16(vaddl_high_u8(d0_lo, d1_lo), scale));
|
|
116
|
+
accu[q][2] = vaddq_u16(
|
|
117
|
+
accu[q][2],
|
|
118
|
+
vmulq_n_u16(
|
|
119
|
+
vaddl_u8(vget_low_u8(d0_hi), vget_low_u8(d1_hi)),
|
|
120
|
+
scale));
|
|
121
|
+
accu[q][3] = vaddq_u16(
|
|
122
|
+
accu[q][3],
|
|
123
|
+
vmulq_n_u16(vaddl_high_u8(d0_hi, d1_hi), scale));
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// Reorder the four 8-lane accumulators for the result handler.
|
|
128
|
+
using simd16uint16 = simd16uint16_tpl<SIMDLevel::ARM_NEON>;
|
|
129
|
+
for (int q = 0; q < NQ; q++) {
|
|
130
|
+
const uint16x8_t d0 = vuzp1q_u16(accu[q][0], accu[q][1]);
|
|
131
|
+
const uint16x8_t d1 = vuzp2q_u16(accu[q][0], accu[q][1]);
|
|
132
|
+
const uint16x8_t d2 = vuzp1q_u16(accu[q][2], accu[q][3]);
|
|
133
|
+
const uint16x8_t d3 = vuzp2q_u16(accu[q][2], accu[q][3]);
|
|
134
|
+
res.handle(
|
|
135
|
+
q,
|
|
136
|
+
0,
|
|
137
|
+
simd16uint16(uint16x8x2_t{d0, d1}),
|
|
138
|
+
simd16uint16(uint16x8x2_t{d2, d3}));
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
template <>
|
|
143
|
+
struct PQ4QBSKernel<SIMDLevel::ARM_NEON> {
|
|
144
|
+
template <int NQ, class ResultHandler, class Scaler>
|
|
145
|
+
FAISS_ALWAYS_INLINE static void run(
|
|
146
|
+
int nsq,
|
|
147
|
+
const uint8_t* codes,
|
|
148
|
+
const uint8_t* LUT,
|
|
149
|
+
ResultHandler& res,
|
|
150
|
+
const Scaler& scaler) {
|
|
151
|
+
pq4_kernel_qbs_neon<NQ>(nsq, codes, LUT, res, scaler);
|
|
152
|
+
}
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
} // namespace faiss
|
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
/*
|
|
2
|
-
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
2
|
+
* Portions Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
3
|
+
*
|
|
4
|
+
* This source code is licensed under the MIT license found in the
|
|
5
|
+
* LICENSE file in the root directory of this source tree.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/*
|
|
9
|
+
* Portions Copyright 2026 Arm Limited and/or its affiliates
|
|
10
|
+
* <open-source-office@arm.com>
|
|
3
11
|
*
|
|
4
12
|
* This source code is licensed under the MIT license found in the
|
|
5
13
|
* LICENSE file in the root directory of this source tree.
|
|
@@ -210,4 +218,17 @@ void pq4_kernel_qbs_256(
|
|
|
210
218
|
}
|
|
211
219
|
}
|
|
212
220
|
|
|
221
|
+
template <SIMDLevel SL>
|
|
222
|
+
struct PQ4QBSKernel {
|
|
223
|
+
template <int NQ, class ResultHandler, class Scaler>
|
|
224
|
+
FAISS_ALWAYS_INLINE static void run(
|
|
225
|
+
int nsq,
|
|
226
|
+
const uint8_t* codes,
|
|
227
|
+
const uint8_t* LUT,
|
|
228
|
+
ResultHandler& res,
|
|
229
|
+
const Scaler& scaler) {
|
|
230
|
+
pq4_kernel_qbs_256<NQ, SL>(nsq, codes, LUT, res, scaler);
|
|
231
|
+
}
|
|
232
|
+
};
|
|
233
|
+
|
|
213
234
|
} // namespace faiss
|
|
@@ -9,8 +9,9 @@
|
|
|
9
9
|
|
|
10
10
|
#include <faiss/impl/platform_macros.h>
|
|
11
11
|
#include <faiss/impl/simdlib/simdlib_dispatch.h>
|
|
12
|
+
#include <faiss/utils/simd_levels.h>
|
|
12
13
|
|
|
13
|
-
#
|
|
14
|
+
#if FAISS_SIMD_TU(AVX512)
|
|
14
15
|
|
|
15
16
|
namespace faiss {
|
|
16
17
|
|
|
@@ -476,4 +477,4 @@ void pq4_kernel_qbs_512(
|
|
|
476
477
|
|
|
477
478
|
} // namespace faiss
|
|
478
479
|
|
|
479
|
-
#endif //
|
|
480
|
+
#endif // FAISS_SIMD_TU(AVX512)
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
#pragma once
|
|
9
9
|
|
|
10
10
|
#include <algorithm>
|
|
11
|
+
#include <bit>
|
|
11
12
|
#include <type_traits>
|
|
12
13
|
#include <utility>
|
|
13
14
|
#include <vector>
|
|
@@ -122,10 +123,7 @@ struct SIMDResultHandlerToFloat {
|
|
|
122
123
|
|
|
123
124
|
// called at end of search to convert int16 distances to float, before
|
|
124
125
|
// normalizers are deallocated
|
|
125
|
-
virtual void end()
|
|
126
|
-
normalizers = nullptr;
|
|
127
|
-
scan_cnt = 0;
|
|
128
|
-
}
|
|
126
|
+
virtual void end();
|
|
129
127
|
|
|
130
128
|
// Get the number of scanned vectors
|
|
131
129
|
size_t count_scanned_rows() {
|
|
@@ -217,8 +215,17 @@ struct StoreResultHandler : SIMDResultHandler<SL> {
|
|
|
217
215
|
}
|
|
218
216
|
};
|
|
219
217
|
|
|
220
|
-
/** stores results in fixed-size matrix.
|
|
221
|
-
|
|
218
|
+
/** stores results in fixed-size matrix.
|
|
219
|
+
*
|
|
220
|
+
* LinkageTag is unused by the handler. A tag type that only one TU uses gives
|
|
221
|
+
* the instantiation (and its out-of-line virtual handle()) a symbol name of
|
|
222
|
+
* its own, so a copy compiled with other SIMD flags cannot replace the copy
|
|
223
|
+
* of another TU. */
|
|
224
|
+
template <
|
|
225
|
+
int NQ,
|
|
226
|
+
int BB,
|
|
227
|
+
SIMDLevel SL = SINGLE_SIMD_LEVEL_256,
|
|
228
|
+
class LinkageTag = void>
|
|
222
229
|
struct FixedStorageHandler : SIMDResultHandler<SL> {
|
|
223
230
|
using typename SIMDResultHandler<SL>::simd16uint16;
|
|
224
231
|
|
|
@@ -374,7 +381,7 @@ struct SingleResultHandler : ResultHandlerCompare<C, with_id_map, SL> {
|
|
|
374
381
|
if (this->sel != nullptr) {
|
|
375
382
|
while (lt_mask) {
|
|
376
383
|
// find first non-zero
|
|
377
|
-
int j =
|
|
384
|
+
int j = std::countr_zero(lt_mask);
|
|
378
385
|
auto real_idx = this->adjust_id(b, j);
|
|
379
386
|
lt_mask -= 1 << j;
|
|
380
387
|
if (this->sel->is_member(real_idx)) {
|
|
@@ -391,7 +398,7 @@ struct SingleResultHandler : ResultHandlerCompare<C, with_id_map, SL> {
|
|
|
391
398
|
} else {
|
|
392
399
|
while (lt_mask) {
|
|
393
400
|
// find first non-zero
|
|
394
|
-
int j =
|
|
401
|
+
int j = std::countr_zero(lt_mask);
|
|
395
402
|
lt_mask -= 1 << j;
|
|
396
403
|
this->scan_cnt++;
|
|
397
404
|
T d = d32tab[j];
|
|
@@ -410,7 +417,8 @@ struct SingleResultHandler : ResultHandlerCompare<C, with_id_map, SL> {
|
|
|
410
417
|
if (!normalizers) {
|
|
411
418
|
dis[q] = idis[q];
|
|
412
419
|
} else {
|
|
413
|
-
float
|
|
420
|
+
float a = normalizers[2 * q];
|
|
421
|
+
float one_a = a != 0 ? 1 / a : 0;
|
|
414
422
|
float b = normalizers[2 * q + 1];
|
|
415
423
|
dis[q] = b + idis[q] * one_a;
|
|
416
424
|
}
|
|
@@ -497,7 +505,7 @@ struct HeapHandler : ResultHandlerCompare<C, with_id_map, SL> {
|
|
|
497
505
|
if (this->sel != nullptr) {
|
|
498
506
|
while (lt_mask) {
|
|
499
507
|
// find first non-zero
|
|
500
|
-
int j =
|
|
508
|
+
int j = std::countr_zero(lt_mask);
|
|
501
509
|
auto real_idx = this->adjust_id(b, j);
|
|
502
510
|
lt_mask -= 1 << j;
|
|
503
511
|
if (this->sel->is_member(real_idx)) {
|
|
@@ -519,7 +527,7 @@ struct HeapHandler : ResultHandlerCompare<C, with_id_map, SL> {
|
|
|
519
527
|
} else {
|
|
520
528
|
while (lt_mask) {
|
|
521
529
|
// find first non-zero
|
|
522
|
-
int j =
|
|
530
|
+
int j = std::countr_zero(lt_mask);
|
|
523
531
|
lt_mask -= 1 << j;
|
|
524
532
|
this->scan_cnt++;
|
|
525
533
|
T dis_for_j = d32tab[j];
|
|
@@ -549,7 +557,8 @@ struct HeapHandler : ResultHandlerCompare<C, with_id_map, SL> {
|
|
|
549
557
|
|
|
550
558
|
float one_a = 1.0, b = 0.0;
|
|
551
559
|
if (normalizers) {
|
|
552
|
-
|
|
560
|
+
float a = normalizers[2 * q];
|
|
561
|
+
one_a = a != 0 ? 1 / a : 0;
|
|
553
562
|
b = normalizers[2 * q + 1];
|
|
554
563
|
}
|
|
555
564
|
for (int j = 0; j < k; j++) {
|
|
@@ -590,7 +599,12 @@ struct ReservoirHandler : ResultHandlerCompare<C, with_id_map, SL> {
|
|
|
590
599
|
|
|
591
600
|
std::vector<TI> all_ids;
|
|
592
601
|
AlignedTable<T> all_vals;
|
|
593
|
-
|
|
602
|
+
// Declared in this class, which takes the SIMD level, so that
|
|
603
|
+
// std::vector<Reservoir> is a different type at each level.
|
|
604
|
+
struct Reservoir : ReservoirTopN<C> {
|
|
605
|
+
using ReservoirTopN<C>::ReservoirTopN;
|
|
606
|
+
};
|
|
607
|
+
std::vector<Reservoir> reservoirs;
|
|
594
608
|
|
|
595
609
|
ReservoirHandler(
|
|
596
610
|
size_t nq,
|
|
@@ -635,7 +649,7 @@ struct ReservoirHandler : ResultHandlerCompare<C, with_id_map, SL> {
|
|
|
635
649
|
if (this->sel != nullptr) {
|
|
636
650
|
while (lt_mask) {
|
|
637
651
|
// find first non-zero
|
|
638
|
-
int j =
|
|
652
|
+
int j = std::countr_zero(lt_mask);
|
|
639
653
|
auto real_idx = this->adjust_id(b, j);
|
|
640
654
|
lt_mask -= 1 << j;
|
|
641
655
|
if (this->sel->is_member(real_idx)) {
|
|
@@ -649,7 +663,7 @@ struct ReservoirHandler : ResultHandlerCompare<C, with_id_map, SL> {
|
|
|
649
663
|
} else {
|
|
650
664
|
while (lt_mask) {
|
|
651
665
|
// find first non-zero
|
|
652
|
-
int j =
|
|
666
|
+
int j = std::countr_zero(lt_mask);
|
|
653
667
|
lt_mask -= 1 << j;
|
|
654
668
|
T dis_for_j = d32tab[j];
|
|
655
669
|
this->scan_cnt++;
|
|
@@ -679,7 +693,8 @@ struct ReservoirHandler : ResultHandlerCompare<C, with_id_map, SL> {
|
|
|
679
693
|
|
|
680
694
|
float one_a = 1.0, b = 0.0;
|
|
681
695
|
if (normalizers) {
|
|
682
|
-
|
|
696
|
+
float a = normalizers[2 * q];
|
|
697
|
+
one_a = a != 0 ? 1 / a : 0;
|
|
683
698
|
b = normalizers[2 * q + 1];
|
|
684
699
|
}
|
|
685
700
|
for (size_t i = 0; i < res.i; i++) {
|
|
@@ -768,7 +783,7 @@ struct RangeHandler : ResultHandlerCompare<C, with_id_map, SL> {
|
|
|
768
783
|
if (this->sel != nullptr) {
|
|
769
784
|
while (lt_mask) {
|
|
770
785
|
// find first non-zero
|
|
771
|
-
int j =
|
|
786
|
+
int j = std::countr_zero(lt_mask);
|
|
772
787
|
lt_mask -= 1 << j;
|
|
773
788
|
|
|
774
789
|
auto real_idx = this->adjust_id(b, j);
|
|
@@ -783,7 +798,7 @@ struct RangeHandler : ResultHandlerCompare<C, with_id_map, SL> {
|
|
|
783
798
|
} else {
|
|
784
799
|
while (lt_mask) {
|
|
785
800
|
// find first non-zero
|
|
786
|
-
int j =
|
|
801
|
+
int j = std::countr_zero(lt_mask);
|
|
787
802
|
lt_mask -= 1 << j;
|
|
788
803
|
T dis = d32tab[j];
|
|
789
804
|
n_per_query[q]++;
|
|
@@ -807,7 +822,8 @@ struct RangeHandler : ResultHandlerCompare<C, with_id_map, SL> {
|
|
|
807
822
|
rres.lims[0] = 0;
|
|
808
823
|
|
|
809
824
|
for (int q = 0; q < nq; q++) {
|
|
810
|
-
float
|
|
825
|
+
float a = normalizers[2 * q];
|
|
826
|
+
float one_a = a != 0 ? 1 / a : 0;
|
|
811
827
|
float b = normalizers[2 * q + 1];
|
|
812
828
|
for (size_t i = rres.lims[q]; i < rres.lims[q + 1]; i++) {
|
|
813
829
|
rres.distances[i] = rres.distances[i] * one_a + b;
|
|
@@ -873,7 +889,8 @@ struct PartialRangeHandler : RangeHandler<C, with_id_map, SL> {
|
|
|
873
889
|
size_t* lims = n_per_query.data();
|
|
874
890
|
|
|
875
891
|
for (int q = 0; q < nq; q++) {
|
|
876
|
-
float
|
|
892
|
+
float a = normalizers[2 * q];
|
|
893
|
+
float one_a = a != 0 ? 1 / a : 0;
|
|
877
894
|
float b = normalizers[2 * q + 1];
|
|
878
895
|
RangeQueryResult& qres = pres.new_result(q + q0);
|
|
879
896
|
for (size_t i = lims[q]; i < lims[q + 1]; i++) {
|
|
@@ -939,7 +956,7 @@ struct SingleQueryResultCollectHandler
|
|
|
939
956
|
|
|
940
957
|
if (this->sel != nullptr) {
|
|
941
958
|
while (lt_mask) {
|
|
942
|
-
int j =
|
|
959
|
+
int j = std::countr_zero(lt_mask);
|
|
943
960
|
auto real_idx = this->adjust_id(b, j);
|
|
944
961
|
lt_mask -= 1 << j;
|
|
945
962
|
if (this->sel->is_member(real_idx)) {
|
|
@@ -950,7 +967,7 @@ struct SingleQueryResultCollectHandler
|
|
|
950
967
|
}
|
|
951
968
|
} else {
|
|
952
969
|
while (lt_mask) {
|
|
953
|
-
int j =
|
|
970
|
+
int j = std::countr_zero(lt_mask);
|
|
954
971
|
lt_mask -= 1 << j;
|
|
955
972
|
T dis = d32tab[j];
|
|
956
973
|
int64_t idx = this->adjust_id(b, j);
|
|
@@ -962,7 +979,8 @@ struct SingleQueryResultCollectHandler
|
|
|
962
979
|
|
|
963
980
|
void end() override {
|
|
964
981
|
if (normalizers) {
|
|
965
|
-
float
|
|
982
|
+
float a = normalizers[0];
|
|
983
|
+
float one_a = a != 0 ? 1 / a : 0;
|
|
966
984
|
float b = normalizers[1];
|
|
967
985
|
for (size_t i = 0; i < collect.size(); i++) {
|
|
968
986
|
collect[i].second = collect[i].second * one_a + b;
|
|
@@ -5,8 +5,6 @@
|
|
|
5
5
|
* LICENSE file in the root directory of this source tree.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
#ifdef COMPILE_SIMD_AVX2
|
|
9
|
-
|
|
10
8
|
#include <faiss/impl/hnsw/MinimaxHeap.h>
|
|
11
9
|
|
|
12
10
|
#include <immintrin.h>
|
|
@@ -146,5 +144,3 @@ int pop_min_tpl<CMin<float, int32_t>, SIMDLevel::AVX2>(
|
|
|
146
144
|
}
|
|
147
145
|
|
|
148
146
|
} // namespace faiss
|
|
149
|
-
|
|
150
|
-
#endif // COMPILE_SIMD_AVX2
|
|
@@ -5,8 +5,6 @@
|
|
|
5
5
|
* LICENSE file in the root directory of this source tree.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
#ifdef COMPILE_SIMD_AVX512
|
|
9
|
-
|
|
10
8
|
#include <faiss/impl/hnsw/MinimaxHeap.h>
|
|
11
9
|
|
|
12
10
|
#include <immintrin.h>
|
|
@@ -138,5 +136,3 @@ int pop_min_tpl<CMin<float, int32_t>, SIMDLevel::AVX512>(
|
|
|
138
136
|
}
|
|
139
137
|
|
|
140
138
|
} // namespace faiss
|
|
141
|
-
|
|
142
|
-
#endif // COMPILE_SIMD_AVX512
|