faiss 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/ext/faiss/extconf.rb +2 -1
- data/ext/faiss/{index_rb.cpp → index.cpp} +1 -1
- data/ext/faiss/index_binary.cpp +1 -1
- data/ext/faiss/kmeans.cpp +1 -1
- data/ext/faiss/pca_matrix.cpp +1 -1
- data/ext/faiss/product_quantizer.cpp +1 -1
- data/ext/faiss/{utils_rb.cpp → utils.cpp} +1 -1
- data/lib/faiss/version.rb +1 -1
- data/vendor/faiss/faiss/AutoTune.cpp +93 -80
- data/vendor/faiss/faiss/Clustering.cpp +39 -240
- data/vendor/faiss/faiss/Clustering.h +6 -0
- data/vendor/faiss/faiss/IVFlib.cpp +41 -21
- data/vendor/faiss/faiss/Index.cpp +6 -5
- data/vendor/faiss/faiss/Index.h +5 -5
- data/vendor/faiss/faiss/Index2Layer.cpp +37 -53
- data/vendor/faiss/faiss/IndexAdditiveQuantizer.cpp +49 -37
- data/vendor/faiss/faiss/IndexAdditiveQuantizerFastScan.cpp +36 -34
- data/vendor/faiss/faiss/IndexAdditiveQuantizerFastScan.h +4 -1
- data/vendor/faiss/faiss/IndexBinary.cpp +5 -3
- data/vendor/faiss/faiss/IndexBinary.h +4 -4
- data/vendor/faiss/faiss/IndexBinaryFlat.cpp +1 -1
- data/vendor/faiss/faiss/IndexBinaryFlat.h +1 -1
- data/vendor/faiss/faiss/IndexBinaryFromFloat.cpp +4 -4
- data/vendor/faiss/faiss/IndexBinaryHNSW.cpp +84 -92
- data/vendor/faiss/faiss/IndexBinaryHNSW.h +9 -3
- data/vendor/faiss/faiss/IndexBinaryHash.cpp +45 -236
- data/vendor/faiss/faiss/IndexBinaryHash.h +6 -6
- data/vendor/faiss/faiss/IndexBinaryIVF.cpp +87 -415
- data/vendor/faiss/faiss/IndexFastScan.cpp +72 -109
- data/vendor/faiss/faiss/IndexFastScan.h +25 -23
- data/vendor/faiss/faiss/IndexFlat.cpp +27 -20
- data/vendor/faiss/faiss/IndexFlat.h +21 -18
- data/vendor/faiss/faiss/IndexFlatCodes.cpp +42 -19
- data/vendor/faiss/faiss/IndexHNSW.cpp +283 -145
- data/vendor/faiss/faiss/IndexHNSW.h +16 -2
- data/vendor/faiss/faiss/IndexIDMap.cpp +25 -21
- data/vendor/faiss/faiss/IndexIDMap.h +9 -7
- data/vendor/faiss/faiss/IndexIVF.cpp +465 -362
- data/vendor/faiss/faiss/IndexIVF.h +33 -12
- data/vendor/faiss/faiss/IndexIVFAdditiveQuantizer.cpp +77 -74
- data/vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.cpp +96 -93
- data/vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.h +4 -1
- data/vendor/faiss/faiss/IndexIVFFastScan.cpp +357 -238
- data/vendor/faiss/faiss/IndexIVFFastScan.h +42 -41
- data/vendor/faiss/faiss/IndexIVFFlat.cpp +36 -68
- data/vendor/faiss/faiss/IndexIVFFlat.h +32 -0
- data/vendor/faiss/faiss/IndexIVFFlatPanorama.cpp +53 -30
- data/vendor/faiss/faiss/IndexIVFFlatPanorama.h +3 -1
- data/vendor/faiss/faiss/IndexIVFIndependentQuantizer.cpp +18 -15
- data/vendor/faiss/faiss/IndexIVFPQ.cpp +71 -843
- data/vendor/faiss/faiss/IndexIVFPQFastScan.cpp +151 -121
- data/vendor/faiss/faiss/IndexIVFPQFastScan.h +3 -0
- data/vendor/faiss/faiss/IndexIVFPQR.cpp +21 -17
- data/vendor/faiss/faiss/IndexIVFRaBitQ.cpp +26 -39
- data/vendor/faiss/faiss/IndexIVFRaBitQ.h +2 -1
- data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.cpp +475 -476
- data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.h +248 -93
- data/vendor/faiss/faiss/IndexIVFSpectralHash.cpp +41 -127
- data/vendor/faiss/faiss/IndexIVFSpectralHash.h +1 -1
- data/vendor/faiss/faiss/IndexLSH.cpp +36 -19
- data/vendor/faiss/faiss/IndexLattice.cpp +13 -13
- data/vendor/faiss/faiss/IndexNNDescent.cpp +36 -21
- data/vendor/faiss/faiss/IndexNNDescent.h +2 -2
- data/vendor/faiss/faiss/IndexNSG.cpp +39 -23
- data/vendor/faiss/faiss/IndexNeuralNetCodec.cpp +31 -11
- data/vendor/faiss/faiss/IndexPQ.cpp +128 -221
- data/vendor/faiss/faiss/IndexPQ.h +3 -2
- data/vendor/faiss/faiss/IndexPQFastScan.cpp +20 -14
- data/vendor/faiss/faiss/IndexPQFastScan.h +3 -0
- data/vendor/faiss/faiss/IndexPreTransform.cpp +25 -18
- data/vendor/faiss/faiss/IndexPreTransform.h +1 -1
- data/vendor/faiss/faiss/IndexRaBitQ.cpp +11 -36
- data/vendor/faiss/faiss/IndexRaBitQ.h +2 -1
- data/vendor/faiss/faiss/IndexRaBitQFastScan.cpp +41 -277
- data/vendor/faiss/faiss/IndexRaBitQFastScan.h +183 -27
- data/vendor/faiss/faiss/IndexRefine.cpp +30 -25
- data/vendor/faiss/faiss/IndexRefine.h +4 -4
- data/vendor/faiss/faiss/IndexReplicas.cpp +6 -6
- data/vendor/faiss/faiss/IndexRowwiseMinMax.cpp +15 -14
- data/vendor/faiss/faiss/IndexRowwiseMinMax.h +1 -1
- data/vendor/faiss/faiss/IndexScalarQuantizer.cpp +82 -14
- data/vendor/faiss/faiss/IndexShards.cpp +10 -9
- data/vendor/faiss/faiss/IndexShardsIVF.cpp +21 -15
- data/vendor/faiss/faiss/MatrixStats.cpp +5 -4
- data/vendor/faiss/faiss/MetaIndexes.cpp +19 -17
- data/vendor/faiss/faiss/MetaIndexes.h +1 -1
- data/vendor/faiss/faiss/MetricType.h +14 -7
- data/vendor/faiss/faiss/SuperKMeans.cpp +656 -0
- data/vendor/faiss/faiss/SuperKMeans.h +97 -0
- data/vendor/faiss/faiss/VectorTransform.cpp +237 -149
- data/vendor/faiss/faiss/VectorTransform.h +16 -16
- data/vendor/faiss/faiss/build.cpp +23 -0
- data/vendor/faiss/faiss/build.h +15 -0
- data/vendor/faiss/faiss/clone_index.cpp +48 -47
- data/vendor/faiss/faiss/cppcontrib/sa_decode/Level2-avx2-inl.h +47 -47
- data/vendor/faiss/faiss/cppcontrib/sa_decode/Level2-inl.h +11 -0
- data/vendor/faiss/faiss/cppcontrib/sa_decode/PQ-avx2-inl.h +38 -38
- data/vendor/faiss/faiss/cppcontrib/sa_decode/PQ-inl.h +11 -0
- data/vendor/faiss/faiss/factory_tools.cpp +5 -0
- data/vendor/faiss/faiss/gpu/GpuIndexCagra.h +6 -5
- data/vendor/faiss/faiss/gpu/GpuResources.h +1 -1
- data/vendor/faiss/faiss/gpu/StandardGpuResources.cpp +9 -9
- data/vendor/faiss/faiss/gpu/StandardGpuResources.h +4 -3
- data/vendor/faiss/faiss/gpu/test/TestGpuIndexFlat.cpp +46 -0
- data/vendor/faiss/faiss/gpu/test/TestGpuIndexIVFFlat.cpp +56 -0
- data/vendor/faiss/faiss/gpu/test/TestGpuIndexIVFPQ.cpp +78 -1
- data/vendor/faiss/faiss/gpu/test/TestUtils.cpp +72 -0
- data/vendor/faiss/faiss/gpu/test/TestUtils.h +23 -0
- data/vendor/faiss/faiss/gpu/utils/CuvsFilterConvert.h +1 -1
- data/vendor/faiss/faiss/gpu/utils/CuvsUtils.h +21 -10
- data/vendor/faiss/faiss/gpu_metal/GpuIndexFlat.h +22 -0
- data/vendor/faiss/faiss/gpu_metal/MetalCloner.h +35 -0
- data/vendor/faiss/faiss/gpu_metal/MetalFlatKernels.h +40 -0
- data/vendor/faiss/faiss/gpu_metal/MetalIndex.h +51 -0
- data/vendor/faiss/faiss/gpu_metal/MetalIndexFlat.h +65 -0
- data/vendor/faiss/faiss/gpu_metal/MetalKernels.h +66 -0
- data/vendor/faiss/faiss/gpu_metal/MetalResources.h +79 -0
- data/vendor/faiss/faiss/gpu_metal/StandardMetalResources.h +35 -0
- data/vendor/faiss/faiss/impl/AdSampling.cpp +103 -0
- data/vendor/faiss/faiss/impl/AdSampling.h +35 -0
- data/vendor/faiss/faiss/impl/AdditiveQuantizer.cpp +29 -25
- data/vendor/faiss/faiss/impl/AdditiveQuantizer.h +1 -0
- data/vendor/faiss/faiss/impl/AuxIndexStructures.cpp +10 -9
- data/vendor/faiss/faiss/impl/AuxIndexStructures.h +3 -0
- data/vendor/faiss/faiss/impl/ClusteringHelpers.cpp +244 -0
- data/vendor/faiss/faiss/impl/ClusteringHelpers.h +94 -0
- data/vendor/faiss/faiss/impl/ClusteringInitialization.cpp +16 -16
- data/vendor/faiss/faiss/impl/CodePacker.cpp +3 -3
- data/vendor/faiss/faiss/impl/CodePackerRaBitQ.cpp +1 -1
- data/vendor/faiss/faiss/impl/DistanceComputer.h +8 -8
- data/vendor/faiss/faiss/impl/FaissAssert.h +6 -3
- data/vendor/faiss/faiss/impl/FaissException.h +50 -3
- data/vendor/faiss/faiss/impl/HNSW.cpp +92 -317
- data/vendor/faiss/faiss/impl/HNSW.h +13 -34
- data/vendor/faiss/faiss/impl/IDSelector.cpp +15 -11
- data/vendor/faiss/faiss/impl/IDSelector.h +8 -8
- data/vendor/faiss/faiss/impl/InvertedListScannerStats.h +26 -0
- data/vendor/faiss/faiss/impl/LocalSearchQuantizer.cpp +82 -77
- data/vendor/faiss/faiss/impl/NNDescent.cpp +62 -25
- data/vendor/faiss/faiss/impl/NNDescent.h +6 -2
- data/vendor/faiss/faiss/impl/NSG.cpp +38 -21
- data/vendor/faiss/faiss/impl/NSG.h +4 -4
- data/vendor/faiss/faiss/impl/Panorama.cpp +23 -6
- data/vendor/faiss/faiss/impl/Panorama.h +258 -87
- data/vendor/faiss/faiss/impl/PdxLayout.cpp +93 -0
- data/vendor/faiss/faiss/impl/PdxLayout.h +41 -0
- data/vendor/faiss/faiss/impl/PolysemousTraining.cpp +46 -32
- data/vendor/faiss/faiss/impl/PolysemousTraining.h +3 -3
- data/vendor/faiss/faiss/impl/ProductAdditiveQuantizer.cpp +35 -35
- data/vendor/faiss/faiss/impl/ProductQuantizer-inl.h +21 -16
- data/vendor/faiss/faiss/impl/ProductQuantizer.cpp +30 -23
- data/vendor/faiss/faiss/impl/Quantizer.h +2 -2
- data/vendor/faiss/faiss/impl/RaBitQUtils.cpp +55 -49
- data/vendor/faiss/faiss/impl/RaBitQUtils.h +65 -0
- data/vendor/faiss/faiss/impl/RaBitQuantizer.cpp +296 -283
- data/vendor/faiss/faiss/impl/ResidualQuantizer.cpp +26 -23
- data/vendor/faiss/faiss/impl/ResidualQuantizer.h +1 -1
- data/vendor/faiss/faiss/impl/ResultHandler.h +99 -75
- data/vendor/faiss/faiss/impl/ScalarQuantizer.cpp +52 -4
- data/vendor/faiss/faiss/impl/ScalarQuantizer.h +27 -1
- data/vendor/faiss/faiss/impl/ThreadedIndex-inl.h +14 -11
- data/vendor/faiss/faiss/impl/VisitedTable.h +7 -0
- data/vendor/faiss/faiss/impl/approx_topk/approx_topk.h +276 -0
- data/vendor/faiss/faiss/impl/approx_topk/avx2.cpp +68 -0
- data/vendor/faiss/faiss/{utils → impl}/approx_topk/generic.h +15 -8
- data/vendor/faiss/faiss/impl/approx_topk/neon.cpp +68 -0
- data/vendor/faiss/faiss/impl/approx_topk/rq_beam_search_tab-inl.h +169 -0
- data/vendor/faiss/faiss/impl/approx_topk/rq_beam_search_tab.h +117 -0
- data/vendor/faiss/faiss/impl/approx_topk/simdlib256-inl.h +146 -0
- data/vendor/faiss/faiss/impl/binary_hamming/IndexBinaryHNSW_impl.h +73 -0
- data/vendor/faiss/faiss/impl/binary_hamming/IndexBinaryHash_impl.h +270 -0
- data/vendor/faiss/faiss/impl/binary_hamming/IndexBinaryIVF_impl.h +460 -0
- data/vendor/faiss/faiss/impl/binary_hamming/IndexIVFSpectralHash_impl.h +159 -0
- data/vendor/faiss/faiss/impl/binary_hamming/IndexPQ_impl.h +92 -0
- data/vendor/faiss/faiss/impl/binary_hamming/avx2.cpp +26 -0
- data/vendor/faiss/faiss/impl/binary_hamming/avx512.cpp +26 -0
- data/vendor/faiss/faiss/impl/binary_hamming/dispatch.h +143 -0
- data/vendor/faiss/faiss/impl/binary_hamming/neon.cpp +26 -0
- data/vendor/faiss/faiss/impl/binary_hamming/rvv.cpp +26 -0
- data/vendor/faiss/faiss/impl/expanded_scanners.h +8 -3
- data/vendor/faiss/faiss/impl/{FastScanDistancePostProcessing.h → fast_scan/FastScanDistancePostProcessing.h} +13 -6
- data/vendor/faiss/faiss/impl/{LookupTableScaler.h → fast_scan/LookupTableScaler.h} +16 -5
- data/vendor/faiss/faiss/impl/fast_scan/accumulate_loops.h +237 -0
- data/vendor/faiss/faiss/impl/fast_scan/accumulate_loops_512.h +185 -0
- data/vendor/faiss/faiss/impl/fast_scan/decompose_qbs.h +229 -0
- data/vendor/faiss/faiss/impl/fast_scan/dispatching.h +268 -0
- data/vendor/faiss/faiss/impl/{pq4_fast_scan.cpp → fast_scan/fast_scan.cpp} +169 -2
- data/vendor/faiss/faiss/impl/fast_scan/fast_scan.h +341 -0
- data/vendor/faiss/faiss/impl/fast_scan/impl-avx2.cpp +36 -0
- data/vendor/faiss/faiss/impl/fast_scan/impl-avx512.cpp +40 -0
- data/vendor/faiss/faiss/impl/fast_scan/impl-neon.cpp +120 -0
- data/vendor/faiss/faiss/impl/fast_scan/impl-riscv.cpp +104 -0
- data/vendor/faiss/faiss/impl/fast_scan/kernels_simd256.h +213 -0
- data/vendor/faiss/faiss/impl/{pq4_fast_scan_search_qbs.cpp → fast_scan/kernels_simd512.h} +26 -356
- data/vendor/faiss/faiss/impl/fast_scan/rabitq_dispatching.h +90 -0
- data/vendor/faiss/faiss/impl/fast_scan/rabitq_result_handler.h +108 -0
- data/vendor/faiss/faiss/impl/{simd_result_handlers.h → fast_scan/simd_result_handlers.h} +282 -134
- data/vendor/faiss/faiss/impl/hnsw/LockVector.cpp +54 -0
- data/vendor/faiss/faiss/impl/hnsw/LockVector.h +64 -0
- data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.cpp +91 -0
- data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.h +64 -0
- data/vendor/faiss/faiss/impl/hnsw/avx2.cpp +104 -0
- data/vendor/faiss/faiss/impl/hnsw/avx512.cpp +111 -0
- data/vendor/faiss/faiss/impl/index_read.cpp +1132 -45
- data/vendor/faiss/faiss/impl/index_read_utils.h +1 -1
- data/vendor/faiss/faiss/impl/index_write.cpp +95 -13
- data/vendor/faiss/faiss/impl/io.cpp +6 -6
- data/vendor/faiss/faiss/impl/io_macros.h +33 -16
- data/vendor/faiss/faiss/impl/kmeans1d.cpp +10 -10
- data/vendor/faiss/faiss/impl/lattice_Zn.cpp +37 -23
- data/vendor/faiss/faiss/impl/lattice_Zn.h +6 -6
- data/vendor/faiss/faiss/impl/mapped_io.cpp +6 -6
- data/vendor/faiss/faiss/impl/platform_macros.h +11 -4
- data/vendor/faiss/faiss/impl/pq_code_distance/IVFPQScanner_impl.h +549 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/IVFPQ_QueryTables.cpp +245 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/IVFPQ_QueryTables.h +105 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/PQDistanceComputer_impl.h +106 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/avx2.cpp +21 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/avx512.cpp +21 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/neon.cpp +21 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/{pq_code_distance-avx2.cpp → pq_code_distance-avx2.h} +9 -13
- data/vendor/faiss/faiss/impl/pq_code_distance/{pq_code_distance-avx512.cpp → pq_code_distance-avx512.h} +9 -57
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-generic.cpp +29 -111
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-generic.h +96 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-inl.h +238 -5
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-sve.cpp +5 -7
- data/vendor/faiss/faiss/impl/pq_code_distance/rvv.cpp +68 -0
- data/vendor/faiss/faiss/impl/residual_quantizer_encode_steps.cpp +311 -477
- data/vendor/faiss/faiss/impl/residual_quantizer_encode_steps.h +1 -1
- data/vendor/faiss/faiss/impl/scalar_quantizer/codecs.h +1 -1
- data/vendor/faiss/faiss/impl/scalar_quantizer/distance_computers.h +3 -2
- data/vendor/faiss/faiss/impl/scalar_quantizer/quantizers.h +102 -11
- data/vendor/faiss/faiss/impl/scalar_quantizer/scanners.h +27 -1
- data/vendor/faiss/faiss/impl/scalar_quantizer/similarities.h +3 -3
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx2.cpp +148 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512.cpp +167 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-dispatch.h +59 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-neon.cpp +163 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-rvv.cpp +311 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/training.cpp +192 -8
- data/vendor/faiss/faiss/impl/scalar_quantizer/training.h +12 -0
- data/vendor/faiss/faiss/impl/simd_dispatch.h +100 -66
- data/vendor/faiss/faiss/impl/simdlib/simdlib.h +57 -0
- data/vendor/faiss/faiss/{utils → impl/simdlib}/simdlib_avx2.h +264 -172
- data/vendor/faiss/faiss/impl/simdlib/simdlib_avx512.h +414 -0
- data/vendor/faiss/faiss/impl/simdlib/simdlib_dispatch.h +44 -0
- data/vendor/faiss/faiss/{utils → impl/simdlib}/simdlib_emulated.h +231 -166
- data/vendor/faiss/faiss/{utils → impl/simdlib}/simdlib_neon.h +270 -218
- data/vendor/faiss/faiss/{utils → impl/simdlib}/simdlib_ppc64.h +201 -160
- data/vendor/faiss/faiss/impl/svs_io.cpp +12 -3
- data/vendor/faiss/faiss/impl/svs_io.h +8 -2
- data/vendor/faiss/faiss/index_factory.cpp +86 -18
- data/vendor/faiss/faiss/index_io.h +24 -0
- data/vendor/faiss/faiss/invlists/BlockInvertedLists.cpp +66 -16
- data/vendor/faiss/faiss/invlists/DirectMap.cpp +24 -14
- data/vendor/faiss/faiss/invlists/DirectMap.h +4 -3
- data/vendor/faiss/faiss/invlists/InvertedLists.cpp +157 -73
- data/vendor/faiss/faiss/invlists/InvertedLists.h +86 -23
- data/vendor/faiss/faiss/invlists/InvertedListsIOHook.cpp +4 -4
- data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.cpp +13 -13
- data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.h +1 -1
- data/vendor/faiss/faiss/svs/IndexSVSFaissUtils.h +1 -1
- data/vendor/faiss/faiss/svs/IndexSVSFlat.cpp +2 -2
- data/vendor/faiss/faiss/svs/IndexSVSIVF.cpp +350 -0
- data/vendor/faiss/faiss/svs/IndexSVSIVF.h +128 -0
- data/vendor/faiss/faiss/svs/IndexSVSIVFLVQ.cpp +40 -0
- data/vendor/faiss/faiss/svs/IndexSVSIVFLVQ.h +43 -0
- data/vendor/faiss/faiss/svs/IndexSVSIVFLeanVec.cpp +225 -0
- data/vendor/faiss/faiss/svs/IndexSVSIVFLeanVec.h +71 -0
- data/vendor/faiss/faiss/svs/IndexSVSVamana.cpp +25 -1
- data/vendor/faiss/faiss/svs/IndexSVSVamana.h +18 -2
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.h +1 -1
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.cpp +12 -3
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.h +7 -2
- data/vendor/faiss/faiss/utils/Heap.cpp +10 -10
- data/vendor/faiss/faiss/utils/NeuralNet.cpp +47 -36
- data/vendor/faiss/faiss/utils/NeuralNet.h +1 -1
- data/vendor/faiss/faiss/utils/approx_topk_hamming/approx_topk_hamming.h +10 -4
- data/vendor/faiss/faiss/utils/distances.cpp +390 -560
- data/vendor/faiss/faiss/utils/distances.h +20 -1
- data/vendor/faiss/faiss/utils/distances_dispatch.h +117 -37
- data/vendor/faiss/faiss/utils/distances_fused/avx512.cpp +8 -7
- data/vendor/faiss/faiss/utils/distances_fused/distances_fused.cpp +33 -14
- data/vendor/faiss/faiss/utils/distances_fused/distances_fused.h +12 -1
- data/vendor/faiss/faiss/utils/distances_fused/simdlib_based.cpp +16 -293
- data/vendor/faiss/faiss/utils/distances_fused/simdlib_based_neon.cpp +57 -0
- data/vendor/faiss/faiss/utils/distances_fused/simdlib_kernel-inl.h +290 -0
- data/vendor/faiss/faiss/utils/distances_simd.cpp +5 -177
- data/vendor/faiss/faiss/utils/extra_distances.cpp +9 -8
- data/vendor/faiss/faiss/utils/extra_distances.h +32 -6
- data/vendor/faiss/faiss/utils/hamming-inl.h +13 -11
- data/vendor/faiss/faiss/utils/hamming.cpp +66 -517
- data/vendor/faiss/faiss/utils/hamming.h +92 -2
- data/vendor/faiss/faiss/utils/hamming_distance/common.h +287 -10
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx2.cpp +15 -0
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx512.cpp +15 -0
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx2.h +142 -0
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512.h +234 -0
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-generic.h +368 -0
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-neon.h +322 -0
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-rvv.h +39 -0
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer.h +146 -0
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_impl.h +481 -0
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_neon.cpp +15 -0
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_rvv.cpp +15 -0
- data/vendor/faiss/faiss/utils/partitioning.cpp +66 -987
- data/vendor/faiss/faiss/utils/partitioning.h +31 -0
- data/vendor/faiss/faiss/utils/popcount.h +29 -0
- data/vendor/faiss/faiss/utils/pq_code_distance.h +2 -2
- data/vendor/faiss/faiss/utils/prefetch.h +2 -2
- data/vendor/faiss/faiss/utils/quantize_lut.cpp +30 -30
- data/vendor/faiss/faiss/utils/quantize_lut.h +1 -1
- data/vendor/faiss/faiss/utils/rabitq_simd.h +57 -536
- data/vendor/faiss/faiss/utils/random.cpp +6 -6
- data/vendor/faiss/faiss/utils/simd_impl/IVFFlatScanner-inl.h +51 -0
- data/vendor/faiss/faiss/utils/simd_impl/distances_aarch64.cpp +5 -1
- data/vendor/faiss/faiss/utils/simd_impl/distances_arm_sve.cpp +213 -4
- data/vendor/faiss/faiss/utils/simd_impl/distances_autovec-inl.h +163 -10
- data/vendor/faiss/faiss/utils/simd_impl/distances_avx2.cpp +250 -4
- data/vendor/faiss/faiss/utils/simd_impl/distances_avx512.cpp +7 -4
- data/vendor/faiss/faiss/utils/simd_impl/distances_rvv.cpp +189 -0
- data/vendor/faiss/faiss/utils/simd_impl/distances_simdlib256.h +195 -0
- data/vendor/faiss/faiss/utils/simd_impl/distances_sse-inl.h +2 -1
- data/vendor/faiss/faiss/utils/{distances_fused/simdlib_based.h → simd_impl/exhaustive_L2sqr_blas_cmax.h} +5 -10
- data/vendor/faiss/faiss/utils/simd_impl/hamming_impl.h +481 -0
- data/vendor/faiss/faiss/utils/simd_impl/partitioning_avx2.cpp +14 -0
- data/vendor/faiss/faiss/utils/simd_impl/partitioning_neon.cpp +14 -0
- data/vendor/faiss/faiss/utils/simd_impl/partitioning_simdlib256.h +1085 -0
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx2.cpp +355 -0
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512.cpp +477 -0
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_neon.cpp +55 -0
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_rvv.cpp +55 -0
- data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_dispatch.h +32 -0
- data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels.h +43 -0
- data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels_avx2.cpp +57 -0
- data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels_avx512.cpp +45 -0
- data/vendor/faiss/faiss/utils/simd_levels.cpp +17 -5
- data/vendor/faiss/faiss/utils/simd_levels.h +93 -1
- data/vendor/faiss/faiss/utils/sorting.cpp +48 -36
- data/vendor/faiss/faiss/utils/utils.cpp +5 -5
- data/vendor/faiss/faiss/utils/utils.h +3 -3
- metadata +119 -34
- data/vendor/faiss/faiss/impl/RaBitQStats.cpp +0 -29
- data/vendor/faiss/faiss/impl/RaBitQStats.h +0 -56
- data/vendor/faiss/faiss/impl/pq4_fast_scan.h +0 -224
- data/vendor/faiss/faiss/impl/pq4_fast_scan_search_1.cpp +0 -230
- data/vendor/faiss/faiss/utils/approx_topk/approx_topk.h +0 -84
- data/vendor/faiss/faiss/utils/approx_topk/avx2-inl.h +0 -196
- data/vendor/faiss/faiss/utils/approx_topk/mode.h +0 -34
- data/vendor/faiss/faiss/utils/distances_fused/avx512.h +0 -36
- data/vendor/faiss/faiss/utils/extra_distances-inl.h +0 -235
- data/vendor/faiss/faiss/utils/hamming_distance/avx2-inl.h +0 -462
- data/vendor/faiss/faiss/utils/hamming_distance/avx512-inl.h +0 -490
- data/vendor/faiss/faiss/utils/hamming_distance/generic-inl.h +0 -449
- data/vendor/faiss/faiss/utils/hamming_distance/hamdis-inl.h +0 -87
- data/vendor/faiss/faiss/utils/hamming_distance/neon-inl.h +0 -524
- data/vendor/faiss/faiss/utils/simdlib.h +0 -42
- data/vendor/faiss/faiss/utils/simdlib_avx512.h +0 -365
- /data/ext/faiss/{utils_rb.h → utils.h} +0 -0
|
@@ -25,22 +25,33 @@
|
|
|
25
25
|
#include <faiss/utils/sorting.h>
|
|
26
26
|
#include <faiss/utils/utils.h>
|
|
27
27
|
|
|
28
|
+
#include <faiss/impl/simd_dispatch.h>
|
|
29
|
+
|
|
30
|
+
// Scalar (NONE) fallback for dynamic dispatch
|
|
31
|
+
#define THE_SIMD_LEVEL SIMDLevel::NONE
|
|
32
|
+
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
33
|
+
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
34
|
+
#include <faiss/impl/binary_hamming/IndexBinaryIVF_impl.h>
|
|
35
|
+
#include <faiss/utils/hamming_distance/hamming_computer-generic.h>
|
|
36
|
+
#undef THE_SIMD_LEVEL
|
|
37
|
+
|
|
28
38
|
namespace faiss {
|
|
29
39
|
|
|
30
40
|
IndexBinaryIVF::IndexBinaryIVF(
|
|
31
|
-
IndexBinary*
|
|
32
|
-
size_t
|
|
33
|
-
size_t
|
|
34
|
-
bool
|
|
35
|
-
: IndexBinary(
|
|
41
|
+
IndexBinary* quantizer_,
|
|
42
|
+
size_t d_,
|
|
43
|
+
size_t nlist_,
|
|
44
|
+
bool own_invlists_)
|
|
45
|
+
: IndexBinary(d_),
|
|
36
46
|
invlists(
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
own_invlists(
|
|
40
|
-
quantizer(
|
|
41
|
-
nlist(
|
|
42
|
-
FAISS_THROW_IF_NOT(
|
|
43
|
-
is_trained =
|
|
47
|
+
own_invlists_ ? new ArrayInvertedLists(nlist_, code_size)
|
|
48
|
+
: nullptr),
|
|
49
|
+
own_invlists(own_invlists_),
|
|
50
|
+
quantizer(quantizer_),
|
|
51
|
+
nlist(nlist_) {
|
|
52
|
+
FAISS_THROW_IF_NOT(d_ == static_cast<size_t>(quantizer_->d));
|
|
53
|
+
is_trained = quantizer_->is_trained &&
|
|
54
|
+
(static_cast<size_t>(quantizer_->ntotal) == nlist_);
|
|
44
55
|
cp.niter = 10;
|
|
45
56
|
}
|
|
46
57
|
|
|
@@ -63,7 +74,7 @@ void IndexBinaryIVF::add_core(
|
|
|
63
74
|
const idx_t* xids,
|
|
64
75
|
const idx_t* precomputed_idx) {
|
|
65
76
|
FAISS_THROW_IF_NOT(is_trained);
|
|
66
|
-
|
|
77
|
+
FAISS_THROW_IF_NOT_MSG(invlists, "invlists not initialized");
|
|
67
78
|
direct_map.check_can_add(xids);
|
|
68
79
|
|
|
69
80
|
const idx_t* idx;
|
|
@@ -79,7 +90,7 @@ void IndexBinaryIVF::add_core(
|
|
|
79
90
|
}
|
|
80
91
|
|
|
81
92
|
idx_t n_add = 0;
|
|
82
|
-
for (
|
|
93
|
+
for (idx_t i = 0; i < n; i++) {
|
|
83
94
|
idx_t id = xids ? xids[i] : ntotal + i;
|
|
84
95
|
idx_t list_no = idx[i];
|
|
85
96
|
|
|
@@ -123,6 +134,7 @@ void IndexBinaryIVF::search(
|
|
|
123
134
|
idx_t* labels,
|
|
124
135
|
const SearchParameters* params_in) const {
|
|
125
136
|
FAISS_THROW_IF_NOT(k > 0);
|
|
137
|
+
FAISS_THROW_IF_NOT_MSG(invlists, "IVF index has no inverted lists");
|
|
126
138
|
const IVFSearchParameters* params = nullptr;
|
|
127
139
|
if (params_in) {
|
|
128
140
|
params = dynamic_cast<const IVFSearchParameters*>(params_in);
|
|
@@ -131,25 +143,25 @@ void IndexBinaryIVF::search(
|
|
|
131
143
|
FAISS_THROW_IF_MSG(
|
|
132
144
|
params->sel, "IDSelector is not supported for IndexBinaryIVF");
|
|
133
145
|
}
|
|
134
|
-
const size_t
|
|
146
|
+
const size_t nprobe_ =
|
|
135
147
|
std::min(nlist, params ? params->nprobe : this->nprobe);
|
|
136
|
-
FAISS_THROW_IF_NOT(
|
|
148
|
+
FAISS_THROW_IF_NOT(nprobe_ > 0);
|
|
137
149
|
|
|
138
|
-
std::unique_ptr<idx_t[]> idx(new idx_t[n *
|
|
139
|
-
std::unique_ptr<int32_t[]> coarse_dis(new int32_t[n *
|
|
150
|
+
std::unique_ptr<idx_t[]> idx(new idx_t[n * nprobe_]);
|
|
151
|
+
std::unique_ptr<int32_t[]> coarse_dis(new int32_t[n * nprobe_]);
|
|
140
152
|
|
|
141
153
|
double t0 = getmillisecs();
|
|
142
154
|
quantizer->search(
|
|
143
155
|
n,
|
|
144
156
|
x,
|
|
145
|
-
|
|
157
|
+
nprobe_,
|
|
146
158
|
coarse_dis.get(),
|
|
147
159
|
idx.get(),
|
|
148
160
|
params ? params->quantizer_params : nullptr);
|
|
149
161
|
indexIVF_stats.quantization_time += getmillisecs() - t0;
|
|
150
162
|
|
|
151
163
|
t0 = getmillisecs();
|
|
152
|
-
invlists->prefetch_lists(idx.get(), n *
|
|
164
|
+
invlists->prefetch_lists(idx.get(), static_cast<int>(n * nprobe_));
|
|
153
165
|
|
|
154
166
|
search_preassigned(
|
|
155
167
|
n,
|
|
@@ -172,11 +184,11 @@ void IndexBinaryIVF::reconstruct(idx_t key, uint8_t* recons) const {
|
|
|
172
184
|
void IndexBinaryIVF::reconstruct_n(idx_t i0, idx_t ni, uint8_t* recons) const {
|
|
173
185
|
FAISS_THROW_IF_NOT(ni == 0 || (i0 >= 0 && i0 + ni <= ntotal));
|
|
174
186
|
|
|
175
|
-
for (
|
|
187
|
+
for (size_t list_no = 0; list_no < nlist; list_no++) {
|
|
176
188
|
size_t list_size = invlists->list_size(list_no);
|
|
177
189
|
const idx_t* idlist = invlists->get_ids(list_no);
|
|
178
190
|
|
|
179
|
-
for (
|
|
191
|
+
for (size_t offset = 0; offset < list_size; offset++) {
|
|
180
192
|
idx_t id = idlist[offset];
|
|
181
193
|
if (!(id >= i0 && id < i0 + ni)) {
|
|
182
194
|
continue;
|
|
@@ -207,7 +219,7 @@ void IndexBinaryIVF::search_and_reconstruct(
|
|
|
207
219
|
|
|
208
220
|
quantizer->search(n, x, nprobe_2, coarse_dis.get(), idx.get());
|
|
209
221
|
|
|
210
|
-
invlists->prefetch_lists(idx.get(), n * nprobe_2);
|
|
222
|
+
invlists->prefetch_lists(idx.get(), static_cast<int>(n * nprobe_2));
|
|
211
223
|
|
|
212
224
|
// search_preassigned() with `store_pairs` enabled to obtain the list_no
|
|
213
225
|
// and offset into `codes` for reconstruction
|
|
@@ -265,7 +277,8 @@ void IndexBinaryIVF::train(idx_t n, const uint8_t* x) {
|
|
|
265
277
|
printf("Training quantizer\n");
|
|
266
278
|
}
|
|
267
279
|
|
|
268
|
-
if (quantizer->is_trained &&
|
|
280
|
+
if (quantizer->is_trained &&
|
|
281
|
+
(static_cast<size_t>(quantizer->ntotal) == nlist)) {
|
|
269
282
|
if (verbose) {
|
|
270
283
|
printf("IVF quantizer does not need training.\n");
|
|
271
284
|
}
|
|
@@ -274,7 +287,7 @@ void IndexBinaryIVF::train(idx_t n, const uint8_t* x) {
|
|
|
274
287
|
printf("Training quantizer on %" PRId64 " vectors in %dD\n", n, d);
|
|
275
288
|
}
|
|
276
289
|
|
|
277
|
-
Clustering clus(d, nlist, cp);
|
|
290
|
+
Clustering clus(d, static_cast<int>(nlist), cp);
|
|
278
291
|
quantizer->reset();
|
|
279
292
|
|
|
280
293
|
IndexFlatL2 index_tmp(d);
|
|
@@ -326,7 +339,9 @@ void IndexBinaryIVF::merge_from(IndexBinary& otherIndex, idx_t add_id) {
|
|
|
326
339
|
}
|
|
327
340
|
|
|
328
341
|
void IndexBinaryIVF::replace_invlists(InvertedLists* il, bool own) {
|
|
329
|
-
FAISS_THROW_IF_NOT(
|
|
342
|
+
FAISS_THROW_IF_NOT(
|
|
343
|
+
il->nlist == nlist &&
|
|
344
|
+
il->code_size == static_cast<size_t>(code_size));
|
|
330
345
|
if (own_invlists) {
|
|
331
346
|
delete invlists;
|
|
332
347
|
}
|
|
@@ -334,68 +349,11 @@ void IndexBinaryIVF::replace_invlists(InvertedLists* il, bool own) {
|
|
|
334
349
|
own_invlists = own;
|
|
335
350
|
}
|
|
336
351
|
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
struct IVFBinaryScannerL2 : BinaryInvertedListScanner {
|
|
341
|
-
HammingComputer hc;
|
|
342
|
-
size_t code_size;
|
|
343
|
-
bool store_pairs;
|
|
344
|
-
|
|
345
|
-
IVFBinaryScannerL2(size_t code_size, bool store_pairs)
|
|
346
|
-
: code_size(code_size), store_pairs(store_pairs) {}
|
|
347
|
-
|
|
348
|
-
void set_query(const uint8_t* query_vector) override {
|
|
349
|
-
hc.set(query_vector, code_size);
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
idx_t list_no;
|
|
353
|
-
void set_list(idx_t list_no_2, uint8_t /* coarse_dis */) override {
|
|
354
|
-
this->list_no = list_no_2;
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
uint32_t distance_to_code(const uint8_t* code) const override {
|
|
358
|
-
return hc.hamming(code);
|
|
359
|
-
}
|
|
360
|
-
|
|
361
|
-
size_t scan_codes(
|
|
362
|
-
size_t n,
|
|
363
|
-
const uint8_t* __restrict codes,
|
|
364
|
-
const idx_t* __restrict ids,
|
|
365
|
-
int32_t* __restrict simi,
|
|
366
|
-
idx_t* __restrict idxi,
|
|
367
|
-
size_t k) const override {
|
|
368
|
-
using C = CMax<int32_t, idx_t>;
|
|
369
|
-
|
|
370
|
-
size_t nup = 0;
|
|
371
|
-
for (size_t j = 0; j < n; j++) {
|
|
372
|
-
uint32_t dis = hc.hamming(codes);
|
|
373
|
-
if (dis < simi[0]) {
|
|
374
|
-
idx_t id = store_pairs ? lo_build(list_no, j) : ids[j];
|
|
375
|
-
heap_replace_top<C>(k, simi, idxi, dis, id);
|
|
376
|
-
nup++;
|
|
377
|
-
}
|
|
378
|
-
codes += code_size;
|
|
379
|
-
}
|
|
380
|
-
return nup;
|
|
381
|
-
}
|
|
352
|
+
// IVFBinaryScannerL2, search_knn_hamming_count, BlockSearch,
|
|
353
|
+
// BlockSearchVariableK, search_knn_hamming_per_invlist are now in
|
|
354
|
+
// impl/binary_hamming/IndexBinaryIVF_impl.h (compiled per-ISA)
|
|
382
355
|
|
|
383
|
-
|
|
384
|
-
size_t n,
|
|
385
|
-
const uint8_t* __restrict codes,
|
|
386
|
-
const idx_t* __restrict ids,
|
|
387
|
-
int radius,
|
|
388
|
-
RangeQueryResult& result) const override {
|
|
389
|
-
for (size_t j = 0; j < n; j++) {
|
|
390
|
-
uint32_t dis = hc.hamming(codes);
|
|
391
|
-
if (dis < radius) {
|
|
392
|
-
int64_t id = store_pairs ? lo_build(list_no, j) : ids[j];
|
|
393
|
-
result.add(dis, id);
|
|
394
|
-
}
|
|
395
|
-
codes += code_size;
|
|
396
|
-
}
|
|
397
|
-
}
|
|
398
|
-
};
|
|
356
|
+
namespace {
|
|
399
357
|
|
|
400
358
|
void search_knn_hamming_heap(
|
|
401
359
|
const IndexBinaryIVF* ivf,
|
|
@@ -429,7 +387,7 @@ void search_knn_hamming_heap(
|
|
|
429
387
|
ivf->get_InvertedListScanner(store_pairs));
|
|
430
388
|
|
|
431
389
|
#pragma omp for
|
|
432
|
-
for (idx_t i = 0; i < n; i++) {
|
|
390
|
+
for (idx_t i = 0; i < static_cast<idx_t>(n); i++) {
|
|
433
391
|
const uint8_t* xi = x + i * ivf->code_size;
|
|
434
392
|
scanner->set_query(xi);
|
|
435
393
|
|
|
@@ -445,7 +403,7 @@ void search_knn_hamming_heap(
|
|
|
445
403
|
|
|
446
404
|
size_t nscan = 0;
|
|
447
405
|
|
|
448
|
-
for (
|
|
406
|
+
for (idx_t ik = 0; ik < nprobe; ik++) {
|
|
449
407
|
idx_t key = keysi[ik]; /* select the list */
|
|
450
408
|
if (key < 0) {
|
|
451
409
|
// not enough centroids for multiprobe
|
|
@@ -455,7 +413,7 @@ void search_knn_hamming_heap(
|
|
|
455
413
|
key < (idx_t)ivf->nlist,
|
|
456
414
|
"Invalid key=%" PRId64 " at ik=%zd nlist=%zd\n",
|
|
457
415
|
key,
|
|
458
|
-
ik,
|
|
416
|
+
static_cast<size_t>(ik),
|
|
459
417
|
ivf->nlist);
|
|
460
418
|
|
|
461
419
|
scanner->set_list(key, coarse_dis[i * nprobe + ik]);
|
|
@@ -463,7 +421,7 @@ void search_knn_hamming_heap(
|
|
|
463
421
|
nlistv++;
|
|
464
422
|
|
|
465
423
|
size_t list_size = ivf->invlists->list_size(key);
|
|
466
|
-
size_t list_size_max = max_codes - nscan;
|
|
424
|
+
size_t list_size_max = static_cast<size_t>(max_codes) - nscan;
|
|
467
425
|
if (list_size > list_size_max) {
|
|
468
426
|
list_size = list_size_max;
|
|
469
427
|
}
|
|
@@ -481,7 +439,7 @@ void search_knn_hamming_heap(
|
|
|
481
439
|
list_size, scodes.get(), ids, simi, idxi, k);
|
|
482
440
|
|
|
483
441
|
nscan += list_size;
|
|
484
|
-
if (nscan >= max_codes) {
|
|
442
|
+
if (nscan >= static_cast<size_t>(max_codes)) {
|
|
485
443
|
break;
|
|
486
444
|
}
|
|
487
445
|
}
|
|
@@ -502,317 +460,17 @@ void search_knn_hamming_heap(
|
|
|
502
460
|
indexIVF_stats.nheap_updates += nheap;
|
|
503
461
|
}
|
|
504
462
|
|
|
505
|
-
template <class HammingComputer, bool store_pairs>
|
|
506
|
-
void search_knn_hamming_count(
|
|
507
|
-
const IndexBinaryIVF* ivf,
|
|
508
|
-
size_t nx,
|
|
509
|
-
const uint8_t* __restrict x,
|
|
510
|
-
const idx_t* __restrict keys,
|
|
511
|
-
int k,
|
|
512
|
-
int32_t* __restrict distances,
|
|
513
|
-
idx_t* __restrict labels,
|
|
514
|
-
const IVFSearchParameters* params) {
|
|
515
|
-
const int nBuckets = ivf->d + 1;
|
|
516
|
-
std::vector<int> all_counters(nx * nBuckets, 0);
|
|
517
|
-
std::unique_ptr<idx_t[]> all_ids_per_dis(new idx_t[nx * nBuckets * k]);
|
|
518
|
-
|
|
519
|
-
idx_t nprobe = params ? params->nprobe : ivf->nprobe;
|
|
520
|
-
nprobe = std::min((idx_t)ivf->nlist, nprobe);
|
|
521
|
-
idx_t max_codes = params ? params->max_codes : ivf->max_codes;
|
|
522
|
-
|
|
523
|
-
std::vector<HCounterState<HammingComputer>> cs;
|
|
524
|
-
for (size_t i = 0; i < nx; ++i) {
|
|
525
|
-
cs.push_back(
|
|
526
|
-
HCounterState<HammingComputer>(
|
|
527
|
-
all_counters.data() + i * nBuckets,
|
|
528
|
-
all_ids_per_dis.get() + i * nBuckets * k,
|
|
529
|
-
x + i * ivf->code_size,
|
|
530
|
-
ivf->d,
|
|
531
|
-
k));
|
|
532
|
-
}
|
|
533
|
-
|
|
534
|
-
size_t nlistv = 0, ndis = 0;
|
|
535
|
-
|
|
536
|
-
#pragma omp parallel for reduction(+ : nlistv, ndis)
|
|
537
|
-
for (int64_t i = 0; i < nx; i++) {
|
|
538
|
-
const idx_t* keysi = keys + i * nprobe;
|
|
539
|
-
HCounterState<HammingComputer>& csi = cs[i];
|
|
540
|
-
|
|
541
|
-
size_t nscan = 0;
|
|
542
|
-
|
|
543
|
-
for (size_t ik = 0; ik < nprobe; ik++) {
|
|
544
|
-
idx_t key = keysi[ik]; /* select the list */
|
|
545
|
-
if (key < 0) {
|
|
546
|
-
// not enough centroids for multiprobe
|
|
547
|
-
continue;
|
|
548
|
-
}
|
|
549
|
-
FAISS_THROW_IF_NOT_FMT(
|
|
550
|
-
key < (idx_t)ivf->nlist,
|
|
551
|
-
"Invalid key=%" PRId64 " at ik=%zd nlist=%zd\n",
|
|
552
|
-
key,
|
|
553
|
-
ik,
|
|
554
|
-
ivf->nlist);
|
|
555
|
-
|
|
556
|
-
nlistv++;
|
|
557
|
-
size_t list_size = ivf->invlists->list_size(key);
|
|
558
|
-
size_t list_size_max = max_codes - nscan;
|
|
559
|
-
if (list_size > list_size_max) {
|
|
560
|
-
list_size = list_size_max;
|
|
561
|
-
}
|
|
562
|
-
InvertedLists::ScopedCodes scodes(ivf->invlists, key);
|
|
563
|
-
const uint8_t* list_vecs = scodes.get();
|
|
564
|
-
const idx_t* ids =
|
|
565
|
-
store_pairs ? nullptr : ivf->invlists->get_ids(key);
|
|
566
|
-
|
|
567
|
-
for (size_t j = 0; j < list_size; j++) {
|
|
568
|
-
const uint8_t* yj = list_vecs + ivf->code_size * j;
|
|
569
|
-
|
|
570
|
-
idx_t id = store_pairs ? (key << 32 | j) : ids[j];
|
|
571
|
-
csi.update_counter(yj, id);
|
|
572
|
-
}
|
|
573
|
-
if (ids) {
|
|
574
|
-
ivf->invlists->release_ids(key, ids);
|
|
575
|
-
}
|
|
576
|
-
|
|
577
|
-
nscan += list_size;
|
|
578
|
-
if (nscan >= max_codes) {
|
|
579
|
-
break;
|
|
580
|
-
}
|
|
581
|
-
}
|
|
582
|
-
ndis += nscan;
|
|
583
|
-
|
|
584
|
-
int nres = 0;
|
|
585
|
-
for (int b = 0; b < nBuckets && nres < k; b++) {
|
|
586
|
-
for (int l = 0; l < csi.counters[b] && nres < k; l++) {
|
|
587
|
-
labels[i * k + nres] = csi.ids_per_dis[b * k + l];
|
|
588
|
-
distances[i * k + nres] = b;
|
|
589
|
-
nres++;
|
|
590
|
-
}
|
|
591
|
-
}
|
|
592
|
-
while (nres < k) {
|
|
593
|
-
labels[i * k + nres] = -1;
|
|
594
|
-
distances[i * k + nres] = std::numeric_limits<int32_t>::max();
|
|
595
|
-
++nres;
|
|
596
|
-
}
|
|
597
|
-
}
|
|
598
|
-
|
|
599
|
-
indexIVF_stats.nq += nx;
|
|
600
|
-
indexIVF_stats.nlist += nlistv;
|
|
601
|
-
indexIVF_stats.ndis += ndis;
|
|
602
|
-
}
|
|
603
|
-
|
|
604
|
-
/* Manages NQ queries at a time, stores results */
|
|
605
|
-
template <class HammingComputer, int NQ, int K>
|
|
606
|
-
struct BlockSearch {
|
|
607
|
-
HammingComputer hcs[NQ];
|
|
608
|
-
// heaps to update for each query
|
|
609
|
-
int32_t* distances[NQ];
|
|
610
|
-
idx_t* labels[NQ];
|
|
611
|
-
// curent top of heap
|
|
612
|
-
int32_t heap_tops[NQ];
|
|
613
|
-
|
|
614
|
-
BlockSearch(
|
|
615
|
-
size_t code_size,
|
|
616
|
-
const uint8_t* __restrict x,
|
|
617
|
-
const int32_t* __restrict keys,
|
|
618
|
-
int32_t* __restrict all_distances,
|
|
619
|
-
idx_t* __restrict all_labels) {
|
|
620
|
-
for (idx_t q = 0; q < NQ; q++) {
|
|
621
|
-
idx_t qno = keys[q];
|
|
622
|
-
hcs[q] = HammingComputer(x + qno * code_size, code_size);
|
|
623
|
-
distances[q] = all_distances + qno * K;
|
|
624
|
-
labels[q] = all_labels + qno * K;
|
|
625
|
-
heap_tops[q] = distances[q][0];
|
|
626
|
-
}
|
|
627
|
-
}
|
|
628
|
-
|
|
629
|
-
void add_bcode(const uint8_t* bcode, idx_t id) {
|
|
630
|
-
using C = CMax<int32_t, idx_t>;
|
|
631
|
-
for (int q = 0; q < NQ; q++) {
|
|
632
|
-
int dis = hcs[q].hamming(bcode);
|
|
633
|
-
if (dis < heap_tops[q]) {
|
|
634
|
-
heap_replace_top<C>(K, distances[q], labels[q], dis, id);
|
|
635
|
-
heap_tops[q] = distances[q][0];
|
|
636
|
-
}
|
|
637
|
-
}
|
|
638
|
-
}
|
|
639
|
-
};
|
|
640
|
-
|
|
641
|
-
template <class HammingComputer, int NQ>
|
|
642
|
-
struct BlockSearchVariableK {
|
|
643
|
-
int k;
|
|
644
|
-
HammingComputer hcs[NQ];
|
|
645
|
-
// heaps to update for each query
|
|
646
|
-
int32_t* distances[NQ];
|
|
647
|
-
idx_t* labels[NQ];
|
|
648
|
-
// curent top of heap
|
|
649
|
-
int32_t heap_tops[NQ];
|
|
650
|
-
|
|
651
|
-
BlockSearchVariableK(
|
|
652
|
-
size_t code_size,
|
|
653
|
-
int k,
|
|
654
|
-
const uint8_t* __restrict x,
|
|
655
|
-
const int32_t* __restrict keys,
|
|
656
|
-
int32_t* __restrict all_distances,
|
|
657
|
-
idx_t* __restrict all_labels)
|
|
658
|
-
: k(k) {
|
|
659
|
-
for (idx_t q = 0; q < NQ; q++) {
|
|
660
|
-
idx_t qno = keys[q];
|
|
661
|
-
hcs[q] = HammingComputer(x + qno * code_size, code_size);
|
|
662
|
-
distances[q] = all_distances + qno * k;
|
|
663
|
-
labels[q] = all_labels + qno * k;
|
|
664
|
-
heap_tops[q] = distances[q][0];
|
|
665
|
-
}
|
|
666
|
-
}
|
|
667
|
-
|
|
668
|
-
void add_bcode(const uint8_t* bcode, idx_t id) {
|
|
669
|
-
using C = CMax<int32_t, idx_t>;
|
|
670
|
-
for (int q = 0; q < NQ; q++) {
|
|
671
|
-
int dis = hcs[q].hamming(bcode);
|
|
672
|
-
if (dis < heap_tops[q]) {
|
|
673
|
-
heap_replace_top<C>(k, distances[q], labels[q], dis, id);
|
|
674
|
-
heap_tops[q] = distances[q][0];
|
|
675
|
-
}
|
|
676
|
-
}
|
|
677
|
-
}
|
|
678
|
-
};
|
|
679
|
-
|
|
680
|
-
template <class HammingComputer>
|
|
681
|
-
void search_knn_hamming_per_invlist(
|
|
682
|
-
const IndexBinaryIVF* ivf,
|
|
683
|
-
size_t n,
|
|
684
|
-
const uint8_t* __restrict x,
|
|
685
|
-
idx_t k,
|
|
686
|
-
const idx_t* __restrict keys_in,
|
|
687
|
-
const int32_t* __restrict coarse_dis,
|
|
688
|
-
int32_t* __restrict distances,
|
|
689
|
-
idx_t* __restrict labels,
|
|
690
|
-
bool store_pairs,
|
|
691
|
-
const IVFSearchParameters* params) {
|
|
692
|
-
idx_t nprobe = params ? params->nprobe : ivf->nprobe;
|
|
693
|
-
nprobe = std::min((idx_t)ivf->nlist, nprobe);
|
|
694
|
-
idx_t max_codes = params ? params->max_codes : ivf->max_codes;
|
|
695
|
-
FAISS_THROW_IF_NOT(max_codes == 0);
|
|
696
|
-
FAISS_THROW_IF_NOT(!store_pairs);
|
|
697
|
-
|
|
698
|
-
// reorder buckets
|
|
699
|
-
std::vector<int64_t> lims(n + 1);
|
|
700
|
-
int32_t* keys = new int32_t[n * nprobe];
|
|
701
|
-
std::unique_ptr<int32_t[]> delete_keys(keys);
|
|
702
|
-
for (idx_t i = 0; i < n * nprobe; i++) {
|
|
703
|
-
keys[i] = keys_in[i];
|
|
704
|
-
}
|
|
705
|
-
matrix_bucket_sort_inplace(n, nprobe, keys, ivf->nlist, lims.data(), 0);
|
|
706
|
-
|
|
707
|
-
using C = CMax<int32_t, idx_t>;
|
|
708
|
-
heap_heapify<C>(n * k, distances, labels);
|
|
709
|
-
const size_t code_size = ivf->code_size;
|
|
710
|
-
|
|
711
|
-
for (idx_t l = 0; l < ivf->nlist; l++) {
|
|
712
|
-
idx_t l0 = lims[l], nq = lims[l + 1] - l0;
|
|
713
|
-
|
|
714
|
-
InvertedLists::ScopedCodes scodes(ivf->invlists, l);
|
|
715
|
-
InvertedLists::ScopedIds sidx(ivf->invlists, l);
|
|
716
|
-
idx_t nb = ivf->invlists->list_size(l);
|
|
717
|
-
const uint8_t* bcodes = scodes.get();
|
|
718
|
-
const idx_t* ids = sidx.get();
|
|
719
|
-
|
|
720
|
-
idx_t i = 0;
|
|
721
|
-
|
|
722
|
-
// process as much as possible by blocks
|
|
723
|
-
constexpr int BS = 4;
|
|
724
|
-
|
|
725
|
-
if (k == 1) {
|
|
726
|
-
for (; i + BS <= nq; i += BS) {
|
|
727
|
-
BlockSearch<HammingComputer, BS, 1> bc(
|
|
728
|
-
code_size, x, keys + l0 + i, distances, labels);
|
|
729
|
-
for (idx_t j = 0; j < nb; j++) {
|
|
730
|
-
bc.add_bcode(bcodes + j * code_size, ids[j]);
|
|
731
|
-
}
|
|
732
|
-
}
|
|
733
|
-
} else if (k == 2) {
|
|
734
|
-
for (; i + BS <= nq; i += BS) {
|
|
735
|
-
BlockSearch<HammingComputer, BS, 2> bc(
|
|
736
|
-
code_size, x, keys + l0 + i, distances, labels);
|
|
737
|
-
for (idx_t j = 0; j < nb; j++) {
|
|
738
|
-
bc.add_bcode(bcodes + j * code_size, ids[j]);
|
|
739
|
-
}
|
|
740
|
-
}
|
|
741
|
-
} else if (k == 4) {
|
|
742
|
-
for (; i + BS <= nq; i += BS) {
|
|
743
|
-
BlockSearch<HammingComputer, BS, 4> bc(
|
|
744
|
-
code_size, x, keys + l0 + i, distances, labels);
|
|
745
|
-
for (idx_t j = 0; j < nb; j++) {
|
|
746
|
-
bc.add_bcode(bcodes + j * code_size, ids[j]);
|
|
747
|
-
}
|
|
748
|
-
}
|
|
749
|
-
} else {
|
|
750
|
-
for (; i + BS <= nq; i += BS) {
|
|
751
|
-
BlockSearchVariableK<HammingComputer, BS> bc(
|
|
752
|
-
code_size, k, x, keys + l0 + i, distances, labels);
|
|
753
|
-
for (idx_t j = 0; j < nb; j++) {
|
|
754
|
-
bc.add_bcode(bcodes + j * code_size, ids[j]);
|
|
755
|
-
}
|
|
756
|
-
}
|
|
757
|
-
}
|
|
758
|
-
|
|
759
|
-
// leftovers
|
|
760
|
-
for (; i < nq; i++) {
|
|
761
|
-
idx_t qno = keys[l0 + i];
|
|
762
|
-
HammingComputer hc(x + qno * code_size, code_size);
|
|
763
|
-
idx_t* __restrict idxi = labels + qno * k;
|
|
764
|
-
int32_t* __restrict simi = distances + qno * k;
|
|
765
|
-
int32_t simi0 = simi[0];
|
|
766
|
-
for (idx_t j = 0; j < nb; j++) {
|
|
767
|
-
int dis = hc.hamming(bcodes + j * code_size);
|
|
768
|
-
|
|
769
|
-
if (dis < simi0) {
|
|
770
|
-
idx_t id = store_pairs ? lo_build(l, j) : ids[j];
|
|
771
|
-
heap_replace_top<C>(k, simi, idxi, dis, id);
|
|
772
|
-
simi0 = simi[0];
|
|
773
|
-
}
|
|
774
|
-
}
|
|
775
|
-
}
|
|
776
|
-
}
|
|
777
|
-
for (idx_t i = 0; i < n; i++) {
|
|
778
|
-
heap_reorder<C>(k, distances + i * k, labels + i * k);
|
|
779
|
-
}
|
|
780
|
-
}
|
|
781
|
-
|
|
782
|
-
struct Run_search_knn_hamming_per_invlist {
|
|
783
|
-
using T = void;
|
|
784
|
-
|
|
785
|
-
template <class HammingComputer, class... Types>
|
|
786
|
-
void f(Types... args) {
|
|
787
|
-
search_knn_hamming_per_invlist<HammingComputer>(args...);
|
|
788
|
-
}
|
|
789
|
-
};
|
|
790
|
-
|
|
791
|
-
template <bool store_pairs>
|
|
792
|
-
struct Run_search_knn_hamming_count {
|
|
793
|
-
using T = void;
|
|
794
|
-
|
|
795
|
-
template <class HammingComputer, class... Types>
|
|
796
|
-
void f(Types... args) {
|
|
797
|
-
search_knn_hamming_count<HammingComputer, store_pairs>(args...);
|
|
798
|
-
}
|
|
799
|
-
};
|
|
800
|
-
|
|
801
|
-
struct BuildScanner {
|
|
802
|
-
using T = BinaryInvertedListScanner*;
|
|
803
|
-
|
|
804
|
-
template <class HammingComputer>
|
|
805
|
-
T f(size_t code_size, bool store_pairs) {
|
|
806
|
-
return new IVFBinaryScannerL2<HammingComputer>(code_size, store_pairs);
|
|
807
|
-
}
|
|
808
|
-
};
|
|
809
|
-
|
|
810
463
|
} // anonymous namespace
|
|
811
464
|
|
|
465
|
+
// The remaining template code (search_knn_hamming_count,
|
|
466
|
+
// search_knn_hamming_per_invlist, etc.) has been moved to
|
|
467
|
+
// impl/binary_hamming/IndexBinaryIVF_impl.h
|
|
468
|
+
|
|
812
469
|
BinaryInvertedListScanner* IndexBinaryIVF::get_InvertedListScanner(
|
|
813
470
|
bool store_pairs) const {
|
|
814
|
-
|
|
815
|
-
|
|
471
|
+
return with_simd_level([&]<SIMDLevel SL>() {
|
|
472
|
+
return make_binary_ivf_scanner_fixSL<SL>(code_size, store_pairs);
|
|
473
|
+
});
|
|
816
474
|
}
|
|
817
475
|
|
|
818
476
|
void IndexBinaryIVF::search_preassigned(
|
|
@@ -826,23 +484,37 @@ void IndexBinaryIVF::search_preassigned(
|
|
|
826
484
|
bool store_pairs,
|
|
827
485
|
const IVFSearchParameters* params) const {
|
|
828
486
|
if (per_invlist_search) {
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
487
|
+
with_simd_level([&]<SIMDLevel SL>() {
|
|
488
|
+
search_knn_hamming_per_invlist_fixSL<SL>(
|
|
489
|
+
code_size,
|
|
490
|
+
this,
|
|
491
|
+
n,
|
|
492
|
+
x,
|
|
493
|
+
k,
|
|
494
|
+
cidx,
|
|
495
|
+
cdis,
|
|
496
|
+
dis,
|
|
497
|
+
idx,
|
|
498
|
+
store_pairs,
|
|
499
|
+
params);
|
|
500
|
+
});
|
|
835
501
|
} else if (use_heap) {
|
|
836
502
|
search_knn_hamming_heap(
|
|
837
503
|
this, n, x, k, cidx, cdis, dis, idx, store_pairs, params);
|
|
838
|
-
} else
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
504
|
+
} else {
|
|
505
|
+
with_simd_level([&]<SIMDLevel SL>() {
|
|
506
|
+
search_knn_hamming_count_fixSL<SL>(
|
|
507
|
+
code_size,
|
|
508
|
+
store_pairs,
|
|
509
|
+
this,
|
|
510
|
+
n,
|
|
511
|
+
x,
|
|
512
|
+
cidx,
|
|
513
|
+
k,
|
|
514
|
+
dis,
|
|
515
|
+
idx,
|
|
516
|
+
params);
|
|
517
|
+
});
|
|
846
518
|
}
|
|
847
519
|
}
|
|
848
520
|
|
|
@@ -863,7 +535,7 @@ void IndexBinaryIVF::range_search(
|
|
|
863
535
|
indexIVF_stats.quantization_time += getmillisecs() - t0;
|
|
864
536
|
|
|
865
537
|
t0 = getmillisecs();
|
|
866
|
-
invlists->prefetch_lists(idx.get(), n * nprobe_2);
|
|
538
|
+
invlists->prefetch_lists(idx.get(), static_cast<int>(n * nprobe_2));
|
|
867
539
|
|
|
868
540
|
range_search_preassigned(n, x, radius, idx.get(), coarse_dis.get(), res);
|
|
869
541
|
|
|
@@ -875,7 +547,7 @@ void IndexBinaryIVF::range_search_preassigned(
|
|
|
875
547
|
const uint8_t* __restrict x,
|
|
876
548
|
int radius,
|
|
877
549
|
const idx_t* __restrict assign,
|
|
878
|
-
const int32_t* __restrict centroid_dis
|
|
550
|
+
const int32_t* __restrict /* centroid_dis */,
|
|
879
551
|
RangeSearchResult* __restrict res) const {
|
|
880
552
|
const size_t nprobe_2 = std::min(nlist, this->nprobe);
|
|
881
553
|
bool store_pairs = false;
|