faiss 0.4.2 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/ext/faiss/index.cpp +36 -10
- data/ext/faiss/index_binary.cpp +19 -6
- data/ext/faiss/kmeans.cpp +6 -6
- data/ext/faiss/numo.hpp +273 -123
- data/lib/faiss/version.rb +1 -1
- data/vendor/faiss/faiss/AutoTune.cpp +2 -3
- data/vendor/faiss/faiss/AutoTune.h +1 -1
- data/vendor/faiss/faiss/Clustering.cpp +2 -2
- data/vendor/faiss/faiss/Clustering.h +2 -2
- data/vendor/faiss/faiss/IVFlib.cpp +1 -2
- data/vendor/faiss/faiss/IVFlib.h +1 -1
- data/vendor/faiss/faiss/Index.h +10 -10
- data/vendor/faiss/faiss/Index2Layer.cpp +1 -1
- data/vendor/faiss/faiss/Index2Layer.h +2 -2
- data/vendor/faiss/faiss/IndexAdditiveQuantizerFastScan.cpp +9 -4
- data/vendor/faiss/faiss/IndexAdditiveQuantizerFastScan.h +5 -1
- data/vendor/faiss/faiss/IndexBinary.h +7 -7
- data/vendor/faiss/faiss/IndexBinaryFromFloat.h +1 -1
- data/vendor/faiss/faiss/IndexBinaryHNSW.cpp +3 -1
- data/vendor/faiss/faiss/IndexBinaryHNSW.h +1 -1
- data/vendor/faiss/faiss/IndexBinaryHash.cpp +3 -3
- data/vendor/faiss/faiss/IndexBinaryHash.h +5 -5
- data/vendor/faiss/faiss/IndexBinaryIVF.cpp +7 -6
- data/vendor/faiss/faiss/IndexFastScan.cpp +125 -49
- data/vendor/faiss/faiss/IndexFastScan.h +107 -7
- data/vendor/faiss/faiss/IndexFlat.h +1 -1
- data/vendor/faiss/faiss/IndexHNSW.cpp +3 -1
- data/vendor/faiss/faiss/IndexHNSW.h +1 -1
- data/vendor/faiss/faiss/IndexIDMap.cpp +14 -13
- data/vendor/faiss/faiss/IndexIDMap.h +6 -6
- data/vendor/faiss/faiss/IndexIVF.cpp +1 -1
- data/vendor/faiss/faiss/IndexIVF.h +5 -5
- data/vendor/faiss/faiss/IndexIVFAdditiveQuantizer.cpp +1 -1
- data/vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.cpp +9 -3
- data/vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.h +3 -1
- data/vendor/faiss/faiss/IndexIVFFastScan.cpp +176 -90
- data/vendor/faiss/faiss/IndexIVFFastScan.h +173 -18
- data/vendor/faiss/faiss/IndexIVFFlat.cpp +1 -0
- data/vendor/faiss/faiss/IndexIVFFlatPanorama.cpp +366 -0
- data/vendor/faiss/faiss/IndexIVFFlatPanorama.h +64 -0
- data/vendor/faiss/faiss/IndexIVFPQ.cpp +3 -1
- data/vendor/faiss/faiss/IndexIVFPQ.h +1 -1
- data/vendor/faiss/faiss/IndexIVFPQFastScan.cpp +134 -2
- data/vendor/faiss/faiss/IndexIVFPQFastScan.h +7 -1
- data/vendor/faiss/faiss/IndexIVFRaBitQ.cpp +13 -6
- data/vendor/faiss/faiss/IndexIVFRaBitQ.h +1 -0
- data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.cpp +650 -0
- data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.h +216 -0
- data/vendor/faiss/faiss/IndexIVFSpectralHash.cpp +1 -1
- data/vendor/faiss/faiss/IndexIVFSpectralHash.h +1 -1
- data/vendor/faiss/faiss/IndexNNDescent.cpp +1 -1
- data/vendor/faiss/faiss/IndexNSG.cpp +1 -1
- data/vendor/faiss/faiss/IndexNeuralNetCodec.h +1 -1
- data/vendor/faiss/faiss/IndexPQ.h +1 -1
- data/vendor/faiss/faiss/IndexPQFastScan.cpp +6 -2
- data/vendor/faiss/faiss/IndexPQFastScan.h +5 -1
- data/vendor/faiss/faiss/IndexRaBitQ.cpp +13 -10
- data/vendor/faiss/faiss/IndexRaBitQ.h +7 -2
- data/vendor/faiss/faiss/IndexRaBitQFastScan.cpp +586 -0
- data/vendor/faiss/faiss/IndexRaBitQFastScan.h +149 -0
- data/vendor/faiss/faiss/IndexShards.cpp +1 -1
- data/vendor/faiss/faiss/MatrixStats.cpp +3 -3
- data/vendor/faiss/faiss/MetricType.h +1 -1
- data/vendor/faiss/faiss/VectorTransform.h +2 -2
- data/vendor/faiss/faiss/clone_index.cpp +3 -1
- data/vendor/faiss/faiss/gpu/GpuCloner.cpp +1 -1
- data/vendor/faiss/faiss/gpu/GpuIndex.h +11 -11
- data/vendor/faiss/faiss/gpu/GpuIndexBinaryCagra.h +1 -1
- data/vendor/faiss/faiss/gpu/GpuIndexBinaryFlat.h +1 -1
- data/vendor/faiss/faiss/gpu/GpuIndexCagra.h +10 -6
- data/vendor/faiss/faiss/gpu/perf/IndexWrapper-inl.h +2 -0
- data/vendor/faiss/faiss/gpu/test/TestGpuIcmEncoder.cpp +7 -0
- data/vendor/faiss/faiss/gpu/test/TestGpuIndexIVFFlat.cpp +1 -1
- data/vendor/faiss/faiss/impl/AdditiveQuantizer.cpp +1 -1
- data/vendor/faiss/faiss/impl/AdditiveQuantizer.h +1 -1
- data/vendor/faiss/faiss/impl/AuxIndexStructures.cpp +2 -2
- data/vendor/faiss/faiss/impl/AuxIndexStructures.h +1 -1
- data/vendor/faiss/faiss/impl/CodePacker.h +2 -2
- data/vendor/faiss/faiss/impl/DistanceComputer.h +3 -3
- data/vendor/faiss/faiss/impl/FastScanDistancePostProcessing.h +53 -0
- data/vendor/faiss/faiss/impl/HNSW.cpp +1 -1
- data/vendor/faiss/faiss/impl/HNSW.h +4 -4
- data/vendor/faiss/faiss/impl/IDSelector.cpp +2 -2
- data/vendor/faiss/faiss/impl/IDSelector.h +1 -1
- data/vendor/faiss/faiss/impl/LocalSearchQuantizer.cpp +4 -4
- data/vendor/faiss/faiss/impl/LocalSearchQuantizer.h +1 -1
- data/vendor/faiss/faiss/impl/LookupTableScaler.h +1 -1
- data/vendor/faiss/faiss/impl/NNDescent.cpp +1 -1
- data/vendor/faiss/faiss/impl/NNDescent.h +2 -2
- data/vendor/faiss/faiss/impl/NSG.cpp +1 -1
- data/vendor/faiss/faiss/impl/PanoramaStats.cpp +33 -0
- data/vendor/faiss/faiss/impl/PanoramaStats.h +38 -0
- data/vendor/faiss/faiss/impl/PolysemousTraining.cpp +5 -5
- data/vendor/faiss/faiss/impl/ProductAdditiveQuantizer.cpp +1 -1
- data/vendor/faiss/faiss/impl/ProductAdditiveQuantizer.h +1 -1
- data/vendor/faiss/faiss/impl/ProductQuantizer-inl.h +2 -0
- data/vendor/faiss/faiss/impl/ProductQuantizer.h +1 -1
- data/vendor/faiss/faiss/impl/RaBitQUtils.cpp +246 -0
- data/vendor/faiss/faiss/impl/RaBitQUtils.h +153 -0
- data/vendor/faiss/faiss/impl/RaBitQuantizer.cpp +54 -158
- data/vendor/faiss/faiss/impl/RaBitQuantizer.h +2 -1
- data/vendor/faiss/faiss/impl/ResidualQuantizer.h +1 -1
- data/vendor/faiss/faiss/impl/ResultHandler.h +4 -4
- data/vendor/faiss/faiss/impl/ScalarQuantizer.cpp +1 -1
- data/vendor/faiss/faiss/impl/ScalarQuantizer.h +1 -1
- data/vendor/faiss/faiss/impl/ThreadedIndex-inl.h +7 -4
- data/vendor/faiss/faiss/impl/index_read.cpp +87 -3
- data/vendor/faiss/faiss/impl/index_write.cpp +73 -3
- data/vendor/faiss/faiss/impl/io.cpp +2 -2
- data/vendor/faiss/faiss/impl/io.h +4 -4
- data/vendor/faiss/faiss/impl/kmeans1d.cpp +1 -1
- data/vendor/faiss/faiss/impl/kmeans1d.h +1 -1
- data/vendor/faiss/faiss/impl/lattice_Zn.h +2 -2
- data/vendor/faiss/faiss/impl/mapped_io.cpp +2 -2
- data/vendor/faiss/faiss/impl/mapped_io.h +4 -3
- data/vendor/faiss/faiss/impl/maybe_owned_vector.h +8 -1
- data/vendor/faiss/faiss/impl/pq4_fast_scan.cpp +30 -4
- data/vendor/faiss/faiss/impl/pq4_fast_scan.h +14 -8
- data/vendor/faiss/faiss/impl/pq4_fast_scan_search_qbs.cpp +5 -6
- data/vendor/faiss/faiss/impl/simd_result_handlers.h +55 -11
- data/vendor/faiss/faiss/impl/zerocopy_io.h +1 -1
- data/vendor/faiss/faiss/index_factory.cpp +43 -1
- data/vendor/faiss/faiss/index_factory.h +1 -1
- data/vendor/faiss/faiss/index_io.h +1 -1
- data/vendor/faiss/faiss/invlists/InvertedLists.cpp +205 -0
- data/vendor/faiss/faiss/invlists/InvertedLists.h +62 -0
- data/vendor/faiss/faiss/utils/AlignedTable.h +1 -1
- data/vendor/faiss/faiss/utils/Heap.cpp +2 -2
- data/vendor/faiss/faiss/utils/Heap.h +3 -3
- data/vendor/faiss/faiss/utils/NeuralNet.cpp +1 -1
- data/vendor/faiss/faiss/utils/NeuralNet.h +3 -3
- data/vendor/faiss/faiss/utils/approx_topk/approx_topk.h +2 -2
- data/vendor/faiss/faiss/utils/approx_topk/avx2-inl.h +2 -2
- data/vendor/faiss/faiss/utils/approx_topk/mode.h +1 -1
- data/vendor/faiss/faiss/utils/distances.h +2 -2
- data/vendor/faiss/faiss/utils/extra_distances-inl.h +3 -1
- data/vendor/faiss/faiss/utils/hamming-inl.h +2 -0
- data/vendor/faiss/faiss/utils/hamming.cpp +7 -6
- data/vendor/faiss/faiss/utils/hamming.h +1 -1
- data/vendor/faiss/faiss/utils/hamming_distance/common.h +1 -2
- data/vendor/faiss/faiss/utils/partitioning.cpp +5 -5
- data/vendor/faiss/faiss/utils/partitioning.h +2 -2
- data/vendor/faiss/faiss/utils/rabitq_simd.h +222 -336
- data/vendor/faiss/faiss/utils/random.cpp +1 -1
- data/vendor/faiss/faiss/utils/simdlib_avx2.h +1 -1
- data/vendor/faiss/faiss/utils/simdlib_avx512.h +1 -1
- data/vendor/faiss/faiss/utils/simdlib_neon.h +2 -2
- data/vendor/faiss/faiss/utils/transpose/transpose-avx512-inl.h +1 -1
- data/vendor/faiss/faiss/utils/utils.cpp +5 -2
- data/vendor/faiss/faiss/utils/utils.h +2 -2
- metadata +14 -3
|
@@ -27,11 +27,13 @@
|
|
|
27
27
|
#include <faiss/IndexIVFAdditiveQuantizer.h>
|
|
28
28
|
#include <faiss/IndexIVFAdditiveQuantizerFastScan.h>
|
|
29
29
|
#include <faiss/IndexIVFFlat.h>
|
|
30
|
+
#include <faiss/IndexIVFFlatPanorama.h>
|
|
30
31
|
#include <faiss/IndexIVFIndependentQuantizer.h>
|
|
31
32
|
#include <faiss/IndexIVFPQ.h>
|
|
32
33
|
#include <faiss/IndexIVFPQFastScan.h>
|
|
33
34
|
#include <faiss/IndexIVFPQR.h>
|
|
34
35
|
#include <faiss/IndexIVFRaBitQ.h>
|
|
36
|
+
#include <faiss/IndexIVFRaBitQFastScan.h>
|
|
35
37
|
#include <faiss/IndexIVFSpectralHash.h>
|
|
36
38
|
#include <faiss/IndexLSH.h>
|
|
37
39
|
#include <faiss/IndexLattice.h>
|
|
@@ -41,6 +43,7 @@
|
|
|
41
43
|
#include <faiss/IndexPQFastScan.h>
|
|
42
44
|
#include <faiss/IndexPreTransform.h>
|
|
43
45
|
#include <faiss/IndexRaBitQ.h>
|
|
46
|
+
#include <faiss/IndexRaBitQFastScan.h>
|
|
44
47
|
#include <faiss/IndexRefine.h>
|
|
45
48
|
#include <faiss/IndexRowwiseMinMax.h>
|
|
46
49
|
#include <faiss/IndexScalarQuantizer.h>
|
|
@@ -68,7 +71,7 @@
|
|
|
68
71
|
* or deprecated fields), the fourcc can be replaced. New code should
|
|
69
72
|
* be able to read the old fourcc and fill in new classes.
|
|
70
73
|
*
|
|
71
|
-
* TODO: in this file, the read functions that
|
|
74
|
+
* TODO: in this file, the read functions that encounter errors may
|
|
72
75
|
* leak memory.
|
|
73
76
|
**************************************************************/
|
|
74
77
|
|
|
@@ -245,6 +248,33 @@ void write_InvertedLists(const InvertedLists* ils, IOWriter* f) {
|
|
|
245
248
|
if (ils == nullptr) {
|
|
246
249
|
uint32_t h = fourcc("il00");
|
|
247
250
|
WRITE1(h);
|
|
251
|
+
} else if (
|
|
252
|
+
const auto& ailp =
|
|
253
|
+
dynamic_cast<const ArrayInvertedListsPanorama*>(ils)) {
|
|
254
|
+
uint32_t h = fourcc("ilpn");
|
|
255
|
+
WRITE1(h);
|
|
256
|
+
WRITE1(ailp->nlist);
|
|
257
|
+
WRITE1(ailp->code_size);
|
|
258
|
+
WRITE1(ailp->n_levels);
|
|
259
|
+
uint32_t list_type = fourcc("full");
|
|
260
|
+
WRITE1(list_type);
|
|
261
|
+
std::vector<size_t> sizes;
|
|
262
|
+
sizes.reserve(ailp->nlist);
|
|
263
|
+
for (size_t i = 0; i < ailp->nlist; i++) {
|
|
264
|
+
sizes.push_back(ailp->ids[i].size());
|
|
265
|
+
}
|
|
266
|
+
WRITEVECTOR(sizes);
|
|
267
|
+
|
|
268
|
+
// Write codes, ids, and cum_sums
|
|
269
|
+
for (size_t i = 0; i < ailp->nlist; i++) {
|
|
270
|
+
size_t n = ailp->ids[i].size();
|
|
271
|
+
if (n > 0) {
|
|
272
|
+
WRITEANDCHECK(ailp->codes[i].data(), ailp->codes[i].size());
|
|
273
|
+
WRITEANDCHECK(ailp->ids[i].data(), n);
|
|
274
|
+
WRITEANDCHECK(
|
|
275
|
+
ailp->cum_sums[i].data(), ailp->cum_sums[i].size());
|
|
276
|
+
}
|
|
277
|
+
}
|
|
248
278
|
} else if (
|
|
249
279
|
const auto& ails = dynamic_cast<const ArrayInvertedLists*>(ils)) {
|
|
250
280
|
uint32_t h = fourcc("ilar");
|
|
@@ -638,6 +668,14 @@ void write_index(const Index* idx, IOWriter* f, int io_flags) {
|
|
|
638
668
|
WRITEVECTOR(tab);
|
|
639
669
|
}
|
|
640
670
|
write_InvertedLists(ivfl->invlists, f);
|
|
671
|
+
} else if (
|
|
672
|
+
const IndexIVFFlatPanorama* ivfp =
|
|
673
|
+
dynamic_cast<const IndexIVFFlatPanorama*>(idx)) {
|
|
674
|
+
uint32_t h = fourcc("IwPn");
|
|
675
|
+
WRITE1(h);
|
|
676
|
+
write_ivf_header(ivfp, f);
|
|
677
|
+
WRITE1(ivfp->n_levels);
|
|
678
|
+
write_InvertedLists(ivfp->invlists, f);
|
|
641
679
|
} else if (
|
|
642
680
|
const IndexIVFFlat* ivfl_2 =
|
|
643
681
|
dynamic_cast<const IndexIVFFlat*>(idx)) {
|
|
@@ -848,7 +886,7 @@ void write_index(const Index* idx, IOWriter* f, int io_flags) {
|
|
|
848
886
|
} else if (
|
|
849
887
|
const IndexRowwiseMinMax* imm =
|
|
850
888
|
dynamic_cast<const IndexRowwiseMinMax*>(idx)) {
|
|
851
|
-
//
|
|
889
|
+
// IndexRowwiseMinMaxFloat
|
|
852
890
|
uint32_t h = fourcc("IRMf");
|
|
853
891
|
WRITE1(h);
|
|
854
892
|
write_index_header(imm, f);
|
|
@@ -856,11 +894,26 @@ void write_index(const Index* idx, IOWriter* f, int io_flags) {
|
|
|
856
894
|
} else if (
|
|
857
895
|
const IndexRowwiseMinMaxFP16* imm_2 =
|
|
858
896
|
dynamic_cast<const IndexRowwiseMinMaxFP16*>(idx)) {
|
|
859
|
-
//
|
|
897
|
+
// IndexRowwiseMinMaxHalf
|
|
860
898
|
uint32_t h = fourcc("IRMh");
|
|
861
899
|
WRITE1(h);
|
|
862
900
|
write_index_header(imm_2, f);
|
|
863
901
|
write_index(imm_2->index, f);
|
|
902
|
+
} else if (
|
|
903
|
+
const IndexRaBitQFastScan* idxqfs =
|
|
904
|
+
dynamic_cast<const IndexRaBitQFastScan*>(idx)) {
|
|
905
|
+
uint32_t h = fourcc("Irfs");
|
|
906
|
+
WRITE1(h);
|
|
907
|
+
write_index_header(idx, f);
|
|
908
|
+
write_RaBitQuantizer(&idxqfs->rabitq, f);
|
|
909
|
+
WRITEVECTOR(idxqfs->center);
|
|
910
|
+
WRITE1(idxqfs->qb);
|
|
911
|
+
WRITEVECTOR(idxqfs->factors_storage);
|
|
912
|
+
WRITE1(idxqfs->bbs);
|
|
913
|
+
WRITE1(idxqfs->ntotal2);
|
|
914
|
+
WRITE1(idxqfs->M2);
|
|
915
|
+
WRITE1(idxqfs->code_size);
|
|
916
|
+
WRITEVECTOR(idxqfs->codes);
|
|
864
917
|
} else if (
|
|
865
918
|
const IndexRaBitQ* idxq = dynamic_cast<const IndexRaBitQ*>(idx)) {
|
|
866
919
|
uint32_t h = fourcc("Ixrq");
|
|
@@ -881,6 +934,23 @@ void write_index(const Index* idx, IOWriter* f, int io_flags) {
|
|
|
881
934
|
WRITE1(ivrq->by_residual);
|
|
882
935
|
WRITE1(ivrq->qb);
|
|
883
936
|
write_InvertedLists(ivrq->invlists, f);
|
|
937
|
+
} else if (
|
|
938
|
+
const IndexIVFRaBitQFastScan* ivrqfs =
|
|
939
|
+
dynamic_cast<const IndexIVFRaBitQFastScan*>(idx)) {
|
|
940
|
+
uint32_t h = fourcc("Iwrf");
|
|
941
|
+
WRITE1(h);
|
|
942
|
+
write_ivf_header(ivrqfs, f);
|
|
943
|
+
write_RaBitQuantizer(&ivrqfs->rabitq, f);
|
|
944
|
+
WRITE1(ivrqfs->by_residual);
|
|
945
|
+
WRITE1(ivrqfs->code_size);
|
|
946
|
+
WRITE1(ivrqfs->bbs);
|
|
947
|
+
WRITE1(ivrqfs->qbs2);
|
|
948
|
+
WRITE1(ivrqfs->M2);
|
|
949
|
+
WRITE1(ivrqfs->implem);
|
|
950
|
+
WRITE1(ivrqfs->qb);
|
|
951
|
+
WRITE1(ivrqfs->centered);
|
|
952
|
+
WRITEVECTOR(ivrqfs->factors_storage);
|
|
953
|
+
write_InvertedLists(ivrqfs->invlists, f);
|
|
884
954
|
} else {
|
|
885
955
|
FAISS_THROW_MSG("don't know how to serialize this type of index");
|
|
886
956
|
}
|
|
@@ -74,7 +74,7 @@ FileIOReader::FileIOReader(const char* fname) {
|
|
|
74
74
|
FileIOReader::~FileIOReader() {
|
|
75
75
|
if (need_close) {
|
|
76
76
|
int ret = fclose(f);
|
|
77
|
-
if (ret != 0) { // we cannot raise
|
|
77
|
+
if (ret != 0) { // we cannot raise an exception in the destructor
|
|
78
78
|
fprintf(stderr,
|
|
79
79
|
"file %s close error: %s",
|
|
80
80
|
name.c_str(),
|
|
@@ -109,7 +109,7 @@ FileIOWriter::~FileIOWriter() {
|
|
|
109
109
|
if (need_close) {
|
|
110
110
|
int ret = fclose(f);
|
|
111
111
|
if (ret != 0) {
|
|
112
|
-
// we cannot raise
|
|
112
|
+
// we cannot raise an exception in the destructor
|
|
113
113
|
fprintf(stderr,
|
|
114
114
|
"file %s close error: %s",
|
|
115
115
|
name.c_str(),
|
|
@@ -65,9 +65,9 @@ struct FileIOReader : IOReader {
|
|
|
65
65
|
FILE* f = nullptr;
|
|
66
66
|
bool need_close = false;
|
|
67
67
|
|
|
68
|
-
FileIOReader(FILE* rf);
|
|
68
|
+
explicit FileIOReader(FILE* rf);
|
|
69
69
|
|
|
70
|
-
FileIOReader(const char* fname);
|
|
70
|
+
explicit FileIOReader(const char* fname);
|
|
71
71
|
|
|
72
72
|
~FileIOReader() override;
|
|
73
73
|
|
|
@@ -80,9 +80,9 @@ struct FileIOWriter : IOWriter {
|
|
|
80
80
|
FILE* f = nullptr;
|
|
81
81
|
bool need_close = false;
|
|
82
82
|
|
|
83
|
-
FileIOWriter(FILE* wf);
|
|
83
|
+
explicit FileIOWriter(FILE* wf);
|
|
84
84
|
|
|
85
|
-
FileIOWriter(const char* fname);
|
|
85
|
+
explicit FileIOWriter(const char* fname);
|
|
86
86
|
|
|
87
87
|
~FileIOWriter() override;
|
|
88
88
|
|
|
@@ -41,7 +41,7 @@ void smawk(
|
|
|
41
41
|
* @param n input array length
|
|
42
42
|
* @param nclusters number of clusters
|
|
43
43
|
* @param centroids output centroids, size nclusters
|
|
44
|
-
* @return
|
|
44
|
+
* @return imbalance factor
|
|
45
45
|
*/
|
|
46
46
|
double kmeans1d(const float* x, size_t n, size_t nclusters, float* centroids);
|
|
47
47
|
|
|
@@ -26,7 +26,7 @@ struct ZnSphereSearch {
|
|
|
26
26
|
int dimS, r2;
|
|
27
27
|
int natom;
|
|
28
28
|
|
|
29
|
-
/// size dim *
|
|
29
|
+
/// size dim * natom
|
|
30
30
|
std::vector<float> voc;
|
|
31
31
|
|
|
32
32
|
ZnSphereSearch(int dim, int r2);
|
|
@@ -138,7 +138,7 @@ struct ZnSphereCodec : ZnSphereSearch, EnumeratedVectors {
|
|
|
138
138
|
*
|
|
139
139
|
* Uses a recursive decomposition on the dimensions to encode
|
|
140
140
|
* centroids found by the ZnSphereSearch. The codes are *not*
|
|
141
|
-
* compatible with the ones of
|
|
141
|
+
* compatible with the ones of ZnSphereCodec
|
|
142
142
|
*/
|
|
143
143
|
struct ZnSphereCodecRec : EnumeratedVectors {
|
|
144
144
|
int r2;
|
|
@@ -33,7 +33,7 @@ struct MmappedFileMappingOwner::PImpl {
|
|
|
33
33
|
void* ptr = nullptr;
|
|
34
34
|
size_t ptr_size = 0;
|
|
35
35
|
|
|
36
|
-
PImpl(const std::string& filename) {
|
|
36
|
+
explicit PImpl(const std::string& filename) {
|
|
37
37
|
auto f = std::unique_ptr<FILE, decltype(&fclose)>(
|
|
38
38
|
fopen(filename.c_str(), "r"), &fclose);
|
|
39
39
|
FAISS_THROW_IF_NOT_FMT(
|
|
@@ -64,7 +64,7 @@ struct MmappedFileMappingOwner::PImpl {
|
|
|
64
64
|
ptr_size = filesize;
|
|
65
65
|
}
|
|
66
66
|
|
|
67
|
-
PImpl(FILE* f) {
|
|
67
|
+
explicit PImpl(FILE* f) {
|
|
68
68
|
// get the size
|
|
69
69
|
struct stat s;
|
|
70
70
|
int status = fstat(fileno(f), &s);
|
|
@@ -18,8 +18,8 @@ namespace faiss {
|
|
|
18
18
|
|
|
19
19
|
// holds a memory-mapped region over a file
|
|
20
20
|
struct MmappedFileMappingOwner : public MaybeOwnedVectorOwner {
|
|
21
|
-
MmappedFileMappingOwner(const std::string& filename);
|
|
22
|
-
MmappedFileMappingOwner(FILE* f);
|
|
21
|
+
explicit MmappedFileMappingOwner(const std::string& filename);
|
|
22
|
+
explicit MmappedFileMappingOwner(FILE* f);
|
|
23
23
|
~MmappedFileMappingOwner();
|
|
24
24
|
|
|
25
25
|
void* data() const;
|
|
@@ -37,7 +37,8 @@ struct MappedFileIOReader : IOReader {
|
|
|
37
37
|
|
|
38
38
|
size_t pos = 0;
|
|
39
39
|
|
|
40
|
-
MappedFileIOReader(
|
|
40
|
+
explicit MappedFileIOReader(
|
|
41
|
+
const std::shared_ptr<MmappedFileMappingOwner>& owner);
|
|
41
42
|
|
|
42
43
|
// perform a copy
|
|
43
44
|
size_t operator()(void* ptr, size_t size, size_t nitems) override;
|
|
@@ -51,7 +51,7 @@ struct MaybeOwnedVector {
|
|
|
51
51
|
size_t c_size = 0;
|
|
52
52
|
|
|
53
53
|
MaybeOwnedVector() = default;
|
|
54
|
-
MaybeOwnedVector(const size_t initial_size) {
|
|
54
|
+
explicit MaybeOwnedVector(const size_t initial_size) {
|
|
55
55
|
is_owned = true;
|
|
56
56
|
|
|
57
57
|
owned_data.resize(initial_size);
|
|
@@ -295,9 +295,16 @@ struct is_maybe_owned_vector : std::false_type {};
|
|
|
295
295
|
template <typename T>
|
|
296
296
|
struct is_maybe_owned_vector<MaybeOwnedVector<T>> : std::true_type {};
|
|
297
297
|
|
|
298
|
+
// guard with c++-17 (maybe, it is available somewhere in
|
|
299
|
+
// faiss/impl/platform.h?).
|
|
300
|
+
// This allows headers to be included in c++11 code.
|
|
301
|
+
#if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L)
|
|
302
|
+
|
|
298
303
|
template <typename T>
|
|
299
304
|
inline constexpr bool is_maybe_owned_vector_v = is_maybe_owned_vector<T>::value;
|
|
300
305
|
|
|
306
|
+
#endif
|
|
307
|
+
|
|
301
308
|
template <typename T>
|
|
302
309
|
bool operator==(
|
|
303
310
|
const MaybeOwnedVector<T>& lhs,
|
|
@@ -49,7 +49,19 @@ void pq4_pack_codes(
|
|
|
49
49
|
size_t nb,
|
|
50
50
|
size_t bbs,
|
|
51
51
|
size_t nsq,
|
|
52
|
-
uint8_t* blocks
|
|
52
|
+
uint8_t* blocks,
|
|
53
|
+
size_t code_stride) {
|
|
54
|
+
// Determine stride: use custom if provided, otherwise use legacy
|
|
55
|
+
// calculation
|
|
56
|
+
size_t actual_stride = (code_stride == 0) ? (M + 1) / 2 : code_stride;
|
|
57
|
+
|
|
58
|
+
// Input validation for custom stride
|
|
59
|
+
if (code_stride != 0) {
|
|
60
|
+
FAISS_THROW_IF_NOT_MSG(
|
|
61
|
+
code_stride >= (M + 1) / 2,
|
|
62
|
+
"Custom stride must be >= minimum code size");
|
|
63
|
+
}
|
|
64
|
+
|
|
53
65
|
FAISS_THROW_IF_NOT(bbs % 32 == 0);
|
|
54
66
|
FAISS_THROW_IF_NOT(nb % bbs == 0);
|
|
55
67
|
FAISS_THROW_IF_NOT(nsq % 2 == 0);
|
|
@@ -72,7 +84,8 @@ void pq4_pack_codes(
|
|
|
72
84
|
for (size_t i = 0; i < bbs; i += 32) {
|
|
73
85
|
std::array<uint8_t, 32> c, c0, c1;
|
|
74
86
|
get_matrix_column(
|
|
75
|
-
codes, ntotal,
|
|
87
|
+
codes, ntotal, actual_stride, i0 + i, sq / 2, c);
|
|
88
|
+
|
|
76
89
|
for (int j = 0; j < 32; j++) {
|
|
77
90
|
c0[j] = c[j] & 15;
|
|
78
91
|
c1[j] = c[j] >> 4;
|
|
@@ -97,7 +110,19 @@ void pq4_pack_codes_range(
|
|
|
97
110
|
size_t i1,
|
|
98
111
|
size_t bbs,
|
|
99
112
|
size_t nsq,
|
|
100
|
-
uint8_t* blocks
|
|
113
|
+
uint8_t* blocks,
|
|
114
|
+
size_t code_stride) {
|
|
115
|
+
// Determine stride: use custom if provided, otherwise use legacy
|
|
116
|
+
// calculation
|
|
117
|
+
size_t actual_stride = (code_stride == 0) ? (M + 1) / 2 : code_stride;
|
|
118
|
+
|
|
119
|
+
// Input validation for custom stride
|
|
120
|
+
if (code_stride != 0) {
|
|
121
|
+
FAISS_THROW_IF_NOT_MSG(
|
|
122
|
+
code_stride >= (M + 1) / 2,
|
|
123
|
+
"Custom stride must be >= minimum code size");
|
|
124
|
+
}
|
|
125
|
+
|
|
101
126
|
#ifdef FAISS_BIG_ENDIAN
|
|
102
127
|
const uint8_t perm0[16] = {
|
|
103
128
|
8, 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7};
|
|
@@ -117,7 +142,8 @@ void pq4_pack_codes_range(
|
|
|
117
142
|
for (size_t i = 0; i < bbs; i += 32) {
|
|
118
143
|
std::array<uint8_t, 32> c, c0, c1;
|
|
119
144
|
get_matrix_column(
|
|
120
|
-
codes, i1 - i0,
|
|
145
|
+
codes, i1 - i0, actual_stride, i_base + i, sq / 2, c);
|
|
146
|
+
|
|
121
147
|
for (int j = 0; j < 32; j++) {
|
|
122
148
|
c0[j] = c[j] & 15;
|
|
123
149
|
c1[j] = c[j] >> 4;
|
|
@@ -34,9 +34,11 @@ struct SIMDResultHandler;
|
|
|
34
34
|
* @param ntotal number of input codes
|
|
35
35
|
* @param nb output number of codes (ntotal rounded up to a multiple of
|
|
36
36
|
* bbs)
|
|
37
|
-
* @param nsq number of sub-quantizers (=M rounded up to a
|
|
37
|
+
* @param nsq number of sub-quantizers (=M rounded up to a multiple of 2)
|
|
38
38
|
* @param bbs size of database blocks (multiple of 32)
|
|
39
39
|
* @param blocks output array, size nb * nsq / 2.
|
|
40
|
+
* @param code_stride optional stride between consecutive codes (0 = use
|
|
41
|
+
default (M + 1) / 2)
|
|
40
42
|
*/
|
|
41
43
|
void pq4_pack_codes(
|
|
42
44
|
const uint8_t* codes,
|
|
@@ -45,7 +47,8 @@ void pq4_pack_codes(
|
|
|
45
47
|
size_t nb,
|
|
46
48
|
size_t bbs,
|
|
47
49
|
size_t nsq,
|
|
48
|
-
uint8_t* blocks
|
|
50
|
+
uint8_t* blocks,
|
|
51
|
+
size_t code_stride = 0);
|
|
49
52
|
|
|
50
53
|
/** Same as pack_codes but write in a given range of the output,
|
|
51
54
|
* leaving the rest untouched. Assumes allocated entries are 0 on input.
|
|
@@ -54,6 +57,8 @@ void pq4_pack_codes(
|
|
|
54
57
|
* @param i0 first output code to write
|
|
55
58
|
* @param i1 last output code to write
|
|
56
59
|
* @param blocks output array, size at least ceil(i1 / bbs) * bbs * nsq / 2
|
|
60
|
+
* @param code_stride optional stride between consecutive codes (0 = use
|
|
61
|
+
* default (M + 1) / 2)
|
|
57
62
|
*/
|
|
58
63
|
void pq4_pack_codes_range(
|
|
59
64
|
const uint8_t* codes,
|
|
@@ -62,7 +67,8 @@ void pq4_pack_codes_range(
|
|
|
62
67
|
size_t i1,
|
|
63
68
|
size_t bbs,
|
|
64
69
|
size_t nsq,
|
|
65
|
-
uint8_t* blocks
|
|
70
|
+
uint8_t* blocks,
|
|
71
|
+
size_t code_stride = 0);
|
|
66
72
|
|
|
67
73
|
/** get a single element from a packed codes table
|
|
68
74
|
*
|
|
@@ -104,7 +110,7 @@ struct CodePackerPQ4 : CodePacker {
|
|
|
104
110
|
/** Pack Look-up table for consumption by the kernel.
|
|
105
111
|
*
|
|
106
112
|
* @param nq number of queries
|
|
107
|
-
* @param nsq number of sub-quantizers (
|
|
113
|
+
* @param nsq number of sub-quantizers (multiple of 2)
|
|
108
114
|
* @param src input array, size (nq, 16)
|
|
109
115
|
* @param dest output array, size (nq, 16)
|
|
110
116
|
*/
|
|
@@ -115,7 +121,7 @@ void pq4_pack_LUT(int nq, int nsq, const uint8_t* src, uint8_t* dest);
|
|
|
115
121
|
* @param nq number of queries
|
|
116
122
|
* @param nb number of database elements
|
|
117
123
|
* @param bbs size of database blocks (multiple of 32)
|
|
118
|
-
* @param nsq number of sub-quantizers (
|
|
124
|
+
* @param nsq number of sub-quantizers (multiple of 2)
|
|
119
125
|
* @param codes packed codes array
|
|
120
126
|
* @param LUT packed look-up table
|
|
121
127
|
* @param scaler scaler to scale the encoded norm
|
|
@@ -154,7 +160,7 @@ int pq4_preferred_qbs(int nq);
|
|
|
154
160
|
*
|
|
155
161
|
* @param qbs 4-bit encoded number of query blocks, the total number of
|
|
156
162
|
* queries handled (nq) is deduced from it
|
|
157
|
-
* @param nsq number of sub-quantizers (
|
|
163
|
+
* @param nsq number of sub-quantizers (multiple of 2)
|
|
158
164
|
* @param src input array, size (nq, 16)
|
|
159
165
|
* @param dest output array, size (nq, 16)
|
|
160
166
|
* @return nq
|
|
@@ -173,11 +179,11 @@ int pq4_pack_LUT_qbs_q_map(
|
|
|
173
179
|
/** Run accumulation loop.
|
|
174
180
|
*
|
|
175
181
|
* @param qbs 4-bit encoded number of queries
|
|
176
|
-
* @param nb number of database codes (
|
|
182
|
+
* @param nb number of database codes (multiple of bbs)
|
|
177
183
|
* @param nsq number of sub-quantizers
|
|
178
184
|
* @param codes encoded database vectors (packed)
|
|
179
185
|
* @param LUT look-up table (packed)
|
|
180
|
-
* @param res call-back for the
|
|
186
|
+
* @param res call-back for the results
|
|
181
187
|
* @param scaler scaler to scale the encoded norm
|
|
182
188
|
*/
|
|
183
189
|
void pq4_accumulate_loop_qbs(
|
|
@@ -53,8 +53,8 @@ void kernel_accumulate_block(
|
|
|
53
53
|
|
|
54
54
|
// _mm_prefetch(codes + 768, 0);
|
|
55
55
|
for (int sq = 0; sq < nsq - scaler.nscale; sq += 2) {
|
|
56
|
-
|
|
57
|
-
|
|
56
|
+
simd32uint8 c;
|
|
57
|
+
c.loadu(codes);
|
|
58
58
|
codes += 32;
|
|
59
59
|
|
|
60
60
|
simd32uint8 mask(0xf);
|
|
@@ -79,8 +79,8 @@ void kernel_accumulate_block(
|
|
|
79
79
|
}
|
|
80
80
|
|
|
81
81
|
for (int sq = 0; sq < scaler.nscale; sq += 2) {
|
|
82
|
-
|
|
83
|
-
|
|
82
|
+
simd32uint8 c;
|
|
83
|
+
c.loadu(codes);
|
|
84
84
|
codes += 32;
|
|
85
85
|
|
|
86
86
|
simd32uint8 mask(0xf);
|
|
@@ -623,7 +623,6 @@ void accumulate(
|
|
|
623
623
|
ResultHandler& res,
|
|
624
624
|
const Scaler& scaler) {
|
|
625
625
|
assert(nsq % 2 == 0);
|
|
626
|
-
assert(is_aligned_pointer(codes));
|
|
627
626
|
assert(is_aligned_pointer(LUT));
|
|
628
627
|
|
|
629
628
|
#define DISPATCH(NQ) \
|
|
@@ -782,7 +781,7 @@ void accumulate_to_mem(
|
|
|
782
781
|
}
|
|
783
782
|
|
|
784
783
|
int pq4_preferred_qbs(int n) {
|
|
785
|
-
// from
|
|
784
|
+
// from timings in P141901742, P141902828
|
|
786
785
|
static int map[12] = {
|
|
787
786
|
0, 1, 2, 3, 0x13, 0x23, 0x33, 0x223, 0x233, 0x333, 0x2233, 0x2333};
|
|
788
787
|
if (n <= 11) {
|
|
@@ -46,7 +46,7 @@ struct SIMDResultHandler {
|
|
|
46
46
|
virtual ~SIMDResultHandler() {}
|
|
47
47
|
};
|
|
48
48
|
|
|
49
|
-
/* Result handler that will return float
|
|
49
|
+
/* Result handler that will return float results eventually */
|
|
50
50
|
struct SIMDResultHandlerToFloat : SIMDResultHandler {
|
|
51
51
|
size_t nq; // number of queries
|
|
52
52
|
size_t ntotal; // ignore excess elements after ntotal
|
|
@@ -70,13 +70,35 @@ struct SIMDResultHandlerToFloat : SIMDResultHandler {
|
|
|
70
70
|
virtual void end() {
|
|
71
71
|
normalizers = nullptr;
|
|
72
72
|
}
|
|
73
|
+
|
|
74
|
+
// Number of updates made to the underlying data structure.
|
|
75
|
+
// For example: number of heap updates.
|
|
76
|
+
virtual size_t num_updates() {
|
|
77
|
+
return 0;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Set context information for handlers that need additional data
|
|
81
|
+
*
|
|
82
|
+
* This method can be overridden by handlers that need list numbers
|
|
83
|
+
* and probe mappings (e.g., RaBitQ handlers). Base implementation
|
|
84
|
+
* does nothing since most handlers don't need this context.
|
|
85
|
+
*
|
|
86
|
+
* @param list_no current inverted list number being processed
|
|
87
|
+
* @param probe_map mapping from local query index to probe index
|
|
88
|
+
*/
|
|
89
|
+
virtual void set_list_context(
|
|
90
|
+
size_t /* list_no */,
|
|
91
|
+
const std::vector<int>& /* probe_map */) {
|
|
92
|
+
// Default implementation does nothing
|
|
93
|
+
// Derived handlers can override if they need this context
|
|
94
|
+
}
|
|
73
95
|
};
|
|
74
96
|
|
|
75
97
|
FAISS_API extern bool simd_result_handlers_accept_virtual;
|
|
76
98
|
|
|
77
99
|
namespace simd_result_handlers {
|
|
78
100
|
|
|
79
|
-
/** Dummy structure that just computes a
|
|
101
|
+
/** Dummy structure that just computes a checksum on results
|
|
80
102
|
* (to avoid the computation to be optimized away) */
|
|
81
103
|
struct DummyResultHandler : SIMDResultHandler {
|
|
82
104
|
size_t cs = 0;
|
|
@@ -318,8 +340,8 @@ struct HeapHandler : ResultHandlerCompare<C, with_id_map> {
|
|
|
318
340
|
std::vector<TI> iids;
|
|
319
341
|
float* dis;
|
|
320
342
|
int64_t* ids;
|
|
321
|
-
|
|
322
|
-
|
|
343
|
+
size_t k; // number of results to keep
|
|
344
|
+
size_t nup = 0; // number of heap updates
|
|
323
345
|
|
|
324
346
|
HeapHandler(
|
|
325
347
|
size_t nq,
|
|
@@ -327,14 +349,30 @@ struct HeapHandler : ResultHandlerCompare<C, with_id_map> {
|
|
|
327
349
|
int64_t k,
|
|
328
350
|
float* dis,
|
|
329
351
|
int64_t* ids,
|
|
330
|
-
const IDSelector* sel_in
|
|
352
|
+
const IDSelector* sel_in,
|
|
353
|
+
const float* normalizers = nullptr)
|
|
331
354
|
: RHC(nq, ntotal, sel_in),
|
|
332
|
-
idis(nq * k),
|
|
333
|
-
iids(nq * k),
|
|
355
|
+
idis(nq * k, threshold_idis(dis, normalizers)),
|
|
356
|
+
iids(nq * k, -1),
|
|
334
357
|
dis(dis),
|
|
335
358
|
ids(ids),
|
|
336
|
-
k(k) {
|
|
337
|
-
|
|
359
|
+
k(k) {}
|
|
360
|
+
|
|
361
|
+
static uint16_t threshold_idis(float* dis_in, const float* normalizers) {
|
|
362
|
+
if (dis_in[0] == std::numeric_limits<float>::max()) {
|
|
363
|
+
return std::numeric_limits<uint16_t>::max();
|
|
364
|
+
}
|
|
365
|
+
if (dis_in[0] == std::numeric_limits<float>::lowest()) {
|
|
366
|
+
return 0;
|
|
367
|
+
}
|
|
368
|
+
if (normalizers) {
|
|
369
|
+
float one_a = 1 / normalizers[0], b = normalizers[1];
|
|
370
|
+
float f = (dis_in[0] - b) / one_a;
|
|
371
|
+
f = C::is_max ? std::ceil(f) : std::floor(f);
|
|
372
|
+
return std::clamp<float>(
|
|
373
|
+
f, 0, std::numeric_limits<uint16_t>::max());
|
|
374
|
+
}
|
|
375
|
+
return C::neutral();
|
|
338
376
|
}
|
|
339
377
|
|
|
340
378
|
void handle(size_t q, size_t b, simd16uint16 d0, simd16uint16 d1) final {
|
|
@@ -372,6 +410,7 @@ struct HeapHandler : ResultHandlerCompare<C, with_id_map> {
|
|
|
372
410
|
if (C::cmp(heap_dis[0], dis_2)) {
|
|
373
411
|
heap_replace_top<C>(
|
|
374
412
|
k, heap_dis, heap_ids, dis_2, real_idx);
|
|
413
|
+
nup++;
|
|
375
414
|
}
|
|
376
415
|
}
|
|
377
416
|
}
|
|
@@ -384,6 +423,7 @@ struct HeapHandler : ResultHandlerCompare<C, with_id_map> {
|
|
|
384
423
|
if (C::cmp(heap_dis[0], dis_2)) {
|
|
385
424
|
int64_t idx = this->adjust_id(b, j);
|
|
386
425
|
heap_replace_top<C>(k, heap_dis, heap_ids, dis_2, idx);
|
|
426
|
+
nup++;
|
|
387
427
|
}
|
|
388
428
|
}
|
|
389
429
|
}
|
|
@@ -408,6 +448,10 @@ struct HeapHandler : ResultHandlerCompare<C, with_id_map> {
|
|
|
408
448
|
}
|
|
409
449
|
}
|
|
410
450
|
}
|
|
451
|
+
|
|
452
|
+
size_t num_updates() override {
|
|
453
|
+
return nup;
|
|
454
|
+
}
|
|
411
455
|
};
|
|
412
456
|
|
|
413
457
|
/** Simple top-N implementation using a reservoir.
|
|
@@ -726,7 +770,7 @@ void dispatch_SIMDResultHandler_fixedCW(
|
|
|
726
770
|
} else { // generic handler -- will not be inlined
|
|
727
771
|
FAISS_THROW_IF_NOT_FMT(
|
|
728
772
|
simd_result_handlers_accept_virtual,
|
|
729
|
-
"Running
|
|
773
|
+
"Running virtual handler for %s",
|
|
730
774
|
typeid(res).name());
|
|
731
775
|
consumer.template f<SIMDResultHandler>(res, args...);
|
|
732
776
|
}
|
|
@@ -757,7 +801,7 @@ void dispatch_SIMDResultHandler(
|
|
|
757
801
|
} else { // generic path
|
|
758
802
|
FAISS_THROW_IF_NOT_FMT(
|
|
759
803
|
simd_result_handlers_accept_virtual,
|
|
760
|
-
"Running
|
|
804
|
+
"Running virtual handler for %s",
|
|
761
805
|
typeid(res).name());
|
|
762
806
|
consumer.template f<SIMDResultHandler>(res, args...);
|
|
763
807
|
}
|
|
@@ -20,7 +20,7 @@ struct ZeroCopyIOReader : public faiss::IOReader {
|
|
|
20
20
|
size_t total_ = 0;
|
|
21
21
|
|
|
22
22
|
ZeroCopyIOReader(const uint8_t* data, size_t size);
|
|
23
|
-
~ZeroCopyIOReader();
|
|
23
|
+
~ZeroCopyIOReader() override;
|
|
24
24
|
|
|
25
25
|
void reset();
|
|
26
26
|
size_t get_data_view(void** ptr, size_t size, size_t nitems);
|