faiss 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/faiss/version.rb +1 -1
- data/vendor/faiss/faiss/AutoTune.cpp +292 -291
- data/vendor/faiss/faiss/AutoTune.h +55 -56
- data/vendor/faiss/faiss/Clustering.cpp +334 -195
- data/vendor/faiss/faiss/Clustering.h +88 -35
- data/vendor/faiss/faiss/IVFlib.cpp +171 -195
- data/vendor/faiss/faiss/IVFlib.h +48 -51
- data/vendor/faiss/faiss/Index.cpp +85 -103
- data/vendor/faiss/faiss/Index.h +54 -48
- data/vendor/faiss/faiss/Index2Layer.cpp +139 -164
- data/vendor/faiss/faiss/Index2Layer.h +22 -22
- data/vendor/faiss/faiss/IndexBinary.cpp +45 -37
- data/vendor/faiss/faiss/IndexBinary.h +140 -132
- data/vendor/faiss/faiss/IndexBinaryFlat.cpp +73 -53
- data/vendor/faiss/faiss/IndexBinaryFlat.h +29 -24
- data/vendor/faiss/faiss/IndexBinaryFromFloat.cpp +46 -43
- data/vendor/faiss/faiss/IndexBinaryFromFloat.h +16 -15
- data/vendor/faiss/faiss/IndexBinaryHNSW.cpp +215 -232
- data/vendor/faiss/faiss/IndexBinaryHNSW.h +25 -24
- data/vendor/faiss/faiss/IndexBinaryHash.cpp +182 -177
- data/vendor/faiss/faiss/IndexBinaryHash.h +41 -34
- data/vendor/faiss/faiss/IndexBinaryIVF.cpp +489 -461
- data/vendor/faiss/faiss/IndexBinaryIVF.h +97 -68
- data/vendor/faiss/faiss/IndexFlat.cpp +116 -147
- data/vendor/faiss/faiss/IndexFlat.h +35 -46
- data/vendor/faiss/faiss/IndexHNSW.cpp +372 -348
- data/vendor/faiss/faiss/IndexHNSW.h +57 -41
- data/vendor/faiss/faiss/IndexIVF.cpp +474 -454
- data/vendor/faiss/faiss/IndexIVF.h +146 -113
- data/vendor/faiss/faiss/IndexIVFFlat.cpp +248 -250
- data/vendor/faiss/faiss/IndexIVFFlat.h +48 -51
- data/vendor/faiss/faiss/IndexIVFPQ.cpp +457 -516
- data/vendor/faiss/faiss/IndexIVFPQ.h +74 -66
- data/vendor/faiss/faiss/IndexIVFPQFastScan.cpp +406 -372
- data/vendor/faiss/faiss/IndexIVFPQFastScan.h +82 -57
- data/vendor/faiss/faiss/IndexIVFPQR.cpp +104 -102
- data/vendor/faiss/faiss/IndexIVFPQR.h +33 -28
- data/vendor/faiss/faiss/IndexIVFSpectralHash.cpp +125 -133
- data/vendor/faiss/faiss/IndexIVFSpectralHash.h +19 -21
- data/vendor/faiss/faiss/IndexLSH.cpp +75 -96
- data/vendor/faiss/faiss/IndexLSH.h +21 -26
- data/vendor/faiss/faiss/IndexLattice.cpp +42 -56
- data/vendor/faiss/faiss/IndexLattice.h +11 -16
- data/vendor/faiss/faiss/IndexNNDescent.cpp +231 -0
- data/vendor/faiss/faiss/IndexNNDescent.h +72 -0
- data/vendor/faiss/faiss/IndexNSG.cpp +303 -0
- data/vendor/faiss/faiss/IndexNSG.h +85 -0
- data/vendor/faiss/faiss/IndexPQ.cpp +405 -464
- data/vendor/faiss/faiss/IndexPQ.h +64 -67
- data/vendor/faiss/faiss/IndexPQFastScan.cpp +143 -170
- data/vendor/faiss/faiss/IndexPQFastScan.h +46 -32
- data/vendor/faiss/faiss/IndexPreTransform.cpp +120 -150
- data/vendor/faiss/faiss/IndexPreTransform.h +33 -36
- data/vendor/faiss/faiss/IndexRefine.cpp +115 -131
- data/vendor/faiss/faiss/IndexRefine.h +22 -23
- data/vendor/faiss/faiss/IndexReplicas.cpp +147 -153
- data/vendor/faiss/faiss/IndexReplicas.h +62 -56
- data/vendor/faiss/faiss/IndexResidual.cpp +291 -0
- data/vendor/faiss/faiss/IndexResidual.h +152 -0
- data/vendor/faiss/faiss/IndexScalarQuantizer.cpp +120 -155
- data/vendor/faiss/faiss/IndexScalarQuantizer.h +41 -45
- data/vendor/faiss/faiss/IndexShards.cpp +256 -240
- data/vendor/faiss/faiss/IndexShards.h +85 -73
- data/vendor/faiss/faiss/MatrixStats.cpp +112 -97
- data/vendor/faiss/faiss/MatrixStats.h +7 -10
- data/vendor/faiss/faiss/MetaIndexes.cpp +135 -157
- data/vendor/faiss/faiss/MetaIndexes.h +40 -34
- data/vendor/faiss/faiss/MetricType.h +7 -7
- data/vendor/faiss/faiss/VectorTransform.cpp +652 -474
- data/vendor/faiss/faiss/VectorTransform.h +61 -89
- data/vendor/faiss/faiss/clone_index.cpp +77 -73
- data/vendor/faiss/faiss/clone_index.h +4 -9
- data/vendor/faiss/faiss/gpu/GpuAutoTune.cpp +33 -38
- data/vendor/faiss/faiss/gpu/GpuAutoTune.h +11 -9
- data/vendor/faiss/faiss/gpu/GpuCloner.cpp +197 -170
- data/vendor/faiss/faiss/gpu/GpuCloner.h +53 -35
- data/vendor/faiss/faiss/gpu/GpuClonerOptions.cpp +12 -14
- data/vendor/faiss/faiss/gpu/GpuClonerOptions.h +27 -25
- data/vendor/faiss/faiss/gpu/GpuDistance.h +116 -112
- data/vendor/faiss/faiss/gpu/GpuFaissAssert.h +1 -2
- data/vendor/faiss/faiss/gpu/GpuIndex.h +134 -137
- data/vendor/faiss/faiss/gpu/GpuIndexBinaryFlat.h +76 -73
- data/vendor/faiss/faiss/gpu/GpuIndexFlat.h +173 -162
- data/vendor/faiss/faiss/gpu/GpuIndexIVF.h +67 -64
- data/vendor/faiss/faiss/gpu/GpuIndexIVFFlat.h +89 -86
- data/vendor/faiss/faiss/gpu/GpuIndexIVFPQ.h +150 -141
- data/vendor/faiss/faiss/gpu/GpuIndexIVFScalarQuantizer.h +101 -103
- data/vendor/faiss/faiss/gpu/GpuIndicesOptions.h +17 -16
- data/vendor/faiss/faiss/gpu/GpuResources.cpp +116 -128
- data/vendor/faiss/faiss/gpu/GpuResources.h +182 -186
- data/vendor/faiss/faiss/gpu/StandardGpuResources.cpp +433 -422
- data/vendor/faiss/faiss/gpu/StandardGpuResources.h +131 -130
- data/vendor/faiss/faiss/gpu/impl/InterleavedCodes.cpp +468 -456
- data/vendor/faiss/faiss/gpu/impl/InterleavedCodes.h +25 -19
- data/vendor/faiss/faiss/gpu/impl/RemapIndices.cpp +22 -20
- data/vendor/faiss/faiss/gpu/impl/RemapIndices.h +9 -8
- data/vendor/faiss/faiss/gpu/perf/IndexWrapper-inl.h +39 -44
- data/vendor/faiss/faiss/gpu/perf/IndexWrapper.h +16 -14
- data/vendor/faiss/faiss/gpu/perf/PerfClustering.cpp +77 -71
- data/vendor/faiss/faiss/gpu/perf/PerfIVFPQAdd.cpp +109 -88
- data/vendor/faiss/faiss/gpu/perf/WriteIndex.cpp +75 -64
- data/vendor/faiss/faiss/gpu/test/TestCodePacking.cpp +230 -215
- data/vendor/faiss/faiss/gpu/test/TestGpuIndexBinaryFlat.cpp +80 -86
- data/vendor/faiss/faiss/gpu/test/TestGpuIndexFlat.cpp +284 -277
- data/vendor/faiss/faiss/gpu/test/TestGpuIndexIVFFlat.cpp +416 -416
- data/vendor/faiss/faiss/gpu/test/TestGpuIndexIVFPQ.cpp +611 -517
- data/vendor/faiss/faiss/gpu/test/TestGpuIndexIVFScalarQuantizer.cpp +166 -164
- data/vendor/faiss/faiss/gpu/test/TestGpuMemoryException.cpp +61 -53
- data/vendor/faiss/faiss/gpu/test/TestUtils.cpp +274 -238
- data/vendor/faiss/faiss/gpu/test/TestUtils.h +73 -57
- data/vendor/faiss/faiss/gpu/test/demo_ivfpq_indexing_gpu.cpp +47 -50
- data/vendor/faiss/faiss/gpu/utils/DeviceUtils.h +79 -72
- data/vendor/faiss/faiss/gpu/utils/StackDeviceMemory.cpp +140 -146
- data/vendor/faiss/faiss/gpu/utils/StackDeviceMemory.h +69 -71
- data/vendor/faiss/faiss/gpu/utils/StaticUtils.h +21 -16
- data/vendor/faiss/faiss/gpu/utils/Timer.cpp +25 -29
- data/vendor/faiss/faiss/gpu/utils/Timer.h +30 -29
- data/vendor/faiss/faiss/impl/AdditiveQuantizer.cpp +270 -0
- data/vendor/faiss/faiss/impl/AdditiveQuantizer.h +115 -0
- data/vendor/faiss/faiss/impl/AuxIndexStructures.cpp +90 -120
- data/vendor/faiss/faiss/impl/AuxIndexStructures.h +81 -65
- data/vendor/faiss/faiss/impl/FaissAssert.h +73 -58
- data/vendor/faiss/faiss/impl/FaissException.cpp +56 -48
- data/vendor/faiss/faiss/impl/FaissException.h +41 -29
- data/vendor/faiss/faiss/impl/HNSW.cpp +595 -611
- data/vendor/faiss/faiss/impl/HNSW.h +179 -200
- data/vendor/faiss/faiss/impl/LocalSearchQuantizer.cpp +672 -0
- data/vendor/faiss/faiss/impl/LocalSearchQuantizer.h +172 -0
- data/vendor/faiss/faiss/impl/NNDescent.cpp +487 -0
- data/vendor/faiss/faiss/impl/NNDescent.h +154 -0
- data/vendor/faiss/faiss/impl/NSG.cpp +682 -0
- data/vendor/faiss/faiss/impl/NSG.h +199 -0
- data/vendor/faiss/faiss/impl/PolysemousTraining.cpp +484 -454
- data/vendor/faiss/faiss/impl/PolysemousTraining.h +52 -55
- data/vendor/faiss/faiss/impl/ProductQuantizer-inl.h +26 -47
- data/vendor/faiss/faiss/impl/ProductQuantizer.cpp +469 -459
- data/vendor/faiss/faiss/impl/ProductQuantizer.h +76 -87
- data/vendor/faiss/faiss/impl/ResidualQuantizer.cpp +448 -0
- data/vendor/faiss/faiss/impl/ResidualQuantizer.h +130 -0
- data/vendor/faiss/faiss/impl/ResultHandler.h +96 -132
- data/vendor/faiss/faiss/impl/ScalarQuantizer.cpp +648 -701
- data/vendor/faiss/faiss/impl/ScalarQuantizer.h +48 -46
- data/vendor/faiss/faiss/impl/ThreadedIndex-inl.h +129 -131
- data/vendor/faiss/faiss/impl/ThreadedIndex.h +61 -55
- data/vendor/faiss/faiss/impl/index_read.cpp +547 -479
- data/vendor/faiss/faiss/impl/index_write.cpp +497 -407
- data/vendor/faiss/faiss/impl/io.cpp +75 -94
- data/vendor/faiss/faiss/impl/io.h +31 -41
- data/vendor/faiss/faiss/impl/io_macros.h +40 -29
- data/vendor/faiss/faiss/impl/lattice_Zn.cpp +137 -186
- data/vendor/faiss/faiss/impl/lattice_Zn.h +40 -51
- data/vendor/faiss/faiss/impl/platform_macros.h +29 -8
- data/vendor/faiss/faiss/impl/pq4_fast_scan.cpp +77 -124
- data/vendor/faiss/faiss/impl/pq4_fast_scan.h +39 -48
- data/vendor/faiss/faiss/impl/pq4_fast_scan_search_1.cpp +41 -52
- data/vendor/faiss/faiss/impl/pq4_fast_scan_search_qbs.cpp +80 -117
- data/vendor/faiss/faiss/impl/simd_result_handlers.h +109 -137
- data/vendor/faiss/faiss/index_factory.cpp +269 -218
- data/vendor/faiss/faiss/index_factory.h +6 -7
- data/vendor/faiss/faiss/index_io.h +23 -26
- data/vendor/faiss/faiss/invlists/BlockInvertedLists.cpp +67 -75
- data/vendor/faiss/faiss/invlists/BlockInvertedLists.h +22 -24
- data/vendor/faiss/faiss/invlists/DirectMap.cpp +96 -112
- data/vendor/faiss/faiss/invlists/DirectMap.h +29 -33
- data/vendor/faiss/faiss/invlists/InvertedLists.cpp +307 -364
- data/vendor/faiss/faiss/invlists/InvertedLists.h +151 -151
- data/vendor/faiss/faiss/invlists/InvertedListsIOHook.cpp +29 -34
- data/vendor/faiss/faiss/invlists/InvertedListsIOHook.h +17 -18
- data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.cpp +257 -293
- data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.h +50 -45
- data/vendor/faiss/faiss/python/python_callbacks.cpp +23 -26
- data/vendor/faiss/faiss/python/python_callbacks.h +9 -16
- data/vendor/faiss/faiss/utils/AlignedTable.h +79 -44
- data/vendor/faiss/faiss/utils/Heap.cpp +40 -48
- data/vendor/faiss/faiss/utils/Heap.h +186 -209
- data/vendor/faiss/faiss/utils/WorkerThread.cpp +67 -76
- data/vendor/faiss/faiss/utils/WorkerThread.h +32 -33
- data/vendor/faiss/faiss/utils/distances.cpp +301 -310
- data/vendor/faiss/faiss/utils/distances.h +133 -118
- data/vendor/faiss/faiss/utils/distances_simd.cpp +456 -516
- data/vendor/faiss/faiss/utils/extra_distances-inl.h +117 -0
- data/vendor/faiss/faiss/utils/extra_distances.cpp +113 -232
- data/vendor/faiss/faiss/utils/extra_distances.h +30 -29
- data/vendor/faiss/faiss/utils/hamming-inl.h +260 -209
- data/vendor/faiss/faiss/utils/hamming.cpp +375 -469
- data/vendor/faiss/faiss/utils/hamming.h +62 -85
- data/vendor/faiss/faiss/utils/ordered_key_value.h +16 -18
- data/vendor/faiss/faiss/utils/partitioning.cpp +393 -318
- data/vendor/faiss/faiss/utils/partitioning.h +26 -21
- data/vendor/faiss/faiss/utils/quantize_lut.cpp +78 -66
- data/vendor/faiss/faiss/utils/quantize_lut.h +22 -20
- data/vendor/faiss/faiss/utils/random.cpp +39 -63
- data/vendor/faiss/faiss/utils/random.h +13 -16
- data/vendor/faiss/faiss/utils/simdlib.h +4 -2
- data/vendor/faiss/faiss/utils/simdlib_avx2.h +88 -85
- data/vendor/faiss/faiss/utils/simdlib_emulated.h +226 -165
- data/vendor/faiss/faiss/utils/simdlib_neon.h +832 -0
- data/vendor/faiss/faiss/utils/utils.cpp +304 -287
- data/vendor/faiss/faiss/utils/utils.h +53 -48
- metadata +20 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6ee0d4a001e68d64f406740bab3bb62f8f2813f8f58e6c9ea93261514cfaec47
|
|
4
|
+
data.tar.gz: a5ddc48f022b0d9b72499766af6f3eaa52c4a36cb2a21d72b333654795539799
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 242636e52ae8fb025cf212fd41e3d27c06d17e05eb59469dd400ab8cf5412e61a1b88d582e0ea188985f3281a90eac0dd2b0ecbefb957864600cda22d9b3900a
|
|
7
|
+
data.tar.gz: 503c4da4a25fa905ef108e564ac715030aa9d10f2df5b1b80a35da99020c11eda3b8ed4ab067324f0d2029b0b90e7ce65d330343ee49d6761c4d98b0fb27655e
|
data/CHANGELOG.md
CHANGED
data/lib/faiss/version.rb
CHANGED
|
@@ -18,22 +18,22 @@
|
|
|
18
18
|
#include <typeinfo>
|
|
19
19
|
|
|
20
20
|
#include <faiss/impl/FaissAssert.h>
|
|
21
|
-
#include <faiss/utils/utils.h>
|
|
22
21
|
#include <faiss/utils/random.h>
|
|
22
|
+
#include <faiss/utils/utils.h>
|
|
23
23
|
|
|
24
24
|
#include <faiss/IndexFlat.h>
|
|
25
|
-
#include <faiss/
|
|
26
|
-
#include <faiss/IndexPreTransform.h>
|
|
27
|
-
#include <faiss/IndexLSH.h>
|
|
28
|
-
#include <faiss/IndexPQ.h>
|
|
25
|
+
#include <faiss/IndexHNSW.h>
|
|
29
26
|
#include <faiss/IndexIVF.h>
|
|
27
|
+
#include <faiss/IndexIVFFlat.h>
|
|
30
28
|
#include <faiss/IndexIVFPQ.h>
|
|
31
29
|
#include <faiss/IndexIVFPQR.h>
|
|
32
|
-
#include <faiss/
|
|
33
|
-
#include <faiss/
|
|
34
|
-
#include <faiss/
|
|
35
|
-
#include <faiss/IndexHNSW.h>
|
|
30
|
+
#include <faiss/IndexLSH.h>
|
|
31
|
+
#include <faiss/IndexPQ.h>
|
|
32
|
+
#include <faiss/IndexPreTransform.h>
|
|
36
33
|
#include <faiss/IndexRefine.h>
|
|
34
|
+
#include <faiss/IndexScalarQuantizer.h>
|
|
35
|
+
#include <faiss/MetaIndexes.h>
|
|
36
|
+
#include <faiss/VectorTransform.h>
|
|
37
37
|
|
|
38
38
|
#include <faiss/IndexBinaryFlat.h>
|
|
39
39
|
#include <faiss/IndexBinaryHNSW.h>
|
|
@@ -41,80 +41,70 @@
|
|
|
41
41
|
|
|
42
42
|
namespace faiss {
|
|
43
43
|
|
|
44
|
+
AutoTuneCriterion::AutoTuneCriterion(idx_t nq, idx_t nnn)
|
|
45
|
+
: nq(nq), nnn(nnn), gt_nnn(0) {}
|
|
44
46
|
|
|
45
|
-
AutoTuneCriterion::
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
void AutoTuneCriterion::set_groundtruth (
|
|
51
|
-
int gt_nnn, const float *gt_D_in, const idx_t *gt_I_in)
|
|
52
|
-
{
|
|
47
|
+
void AutoTuneCriterion::set_groundtruth(
|
|
48
|
+
int gt_nnn,
|
|
49
|
+
const float* gt_D_in,
|
|
50
|
+
const idx_t* gt_I_in) {
|
|
53
51
|
this->gt_nnn = gt_nnn;
|
|
54
52
|
if (gt_D_in) { // allow null for this, as it is often not used
|
|
55
|
-
gt_D.resize
|
|
56
|
-
memcpy
|
|
53
|
+
gt_D.resize(nq * gt_nnn);
|
|
54
|
+
memcpy(gt_D.data(), gt_D_in, sizeof(gt_D[0]) * nq * gt_nnn);
|
|
57
55
|
}
|
|
58
|
-
gt_I.resize
|
|
59
|
-
memcpy
|
|
56
|
+
gt_I.resize(nq * gt_nnn);
|
|
57
|
+
memcpy(gt_I.data(), gt_I_in, sizeof(gt_I[0]) * nq * gt_nnn);
|
|
60
58
|
}
|
|
61
59
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
OneRecallAtRCriterion::OneRecallAtRCriterion (idx_t nq, idx_t R):
|
|
65
|
-
AutoTuneCriterion(nq, R), R(R)
|
|
66
|
-
{}
|
|
60
|
+
OneRecallAtRCriterion::OneRecallAtRCriterion(idx_t nq, idx_t R)
|
|
61
|
+
: AutoTuneCriterion(nq, R), R(R) {}
|
|
67
62
|
|
|
68
63
|
double OneRecallAtRCriterion::evaluate(const float* /*D*/, const idx_t* I)
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
64
|
+
const {
|
|
65
|
+
FAISS_THROW_IF_NOT_MSG(
|
|
66
|
+
(gt_I.size() == gt_nnn * nq && gt_nnn >= 1 && nnn >= R),
|
|
67
|
+
"ground truth not initialized");
|
|
68
|
+
idx_t n_ok = 0;
|
|
69
|
+
for (idx_t q = 0; q < nq; q++) {
|
|
70
|
+
idx_t gt_nn = gt_I[q * gt_nnn];
|
|
71
|
+
const idx_t* I_line = I + q * nnn;
|
|
72
|
+
for (int i = 0; i < R; i++) {
|
|
73
|
+
if (I_line[i] == gt_nn) {
|
|
74
|
+
n_ok++;
|
|
75
|
+
break;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return n_ok / double(nq);
|
|
85
80
|
}
|
|
86
81
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
AutoTuneCriterion(nq, R), R(R)
|
|
90
|
-
{}
|
|
82
|
+
IntersectionCriterion::IntersectionCriterion(idx_t nq, idx_t R)
|
|
83
|
+
: AutoTuneCriterion(nq, R), R(R) {}
|
|
91
84
|
|
|
92
85
|
double IntersectionCriterion::evaluate(const float* /*D*/, const idx_t* I)
|
|
93
|
-
|
|
86
|
+
const {
|
|
94
87
|
FAISS_THROW_IF_NOT_MSG(
|
|
95
|
-
|
|
96
|
-
|
|
88
|
+
(gt_I.size() == gt_nnn * nq && gt_nnn >= R && nnn >= R),
|
|
89
|
+
"ground truth not initialized");
|
|
97
90
|
int64_t n_ok = 0;
|
|
98
|
-
#pragma omp parallel for reduction(
|
|
91
|
+
#pragma omp parallel for reduction(+ : n_ok)
|
|
99
92
|
for (idx_t q = 0; q < nq; q++) {
|
|
100
|
-
n_ok += ranklist_intersection_size
|
|
101
|
-
|
|
102
|
-
R, I + q * nnn);
|
|
93
|
+
n_ok += ranklist_intersection_size(
|
|
94
|
+
R, >_I[q * gt_nnn], R, I + q * nnn);
|
|
103
95
|
}
|
|
104
|
-
return n_ok / double
|
|
96
|
+
return n_ok / double(nq * R);
|
|
105
97
|
}
|
|
106
98
|
|
|
107
99
|
/***************************************************************
|
|
108
100
|
* OperatingPoints
|
|
109
101
|
***************************************************************/
|
|
110
102
|
|
|
111
|
-
OperatingPoints::OperatingPoints
|
|
112
|
-
{
|
|
103
|
+
OperatingPoints::OperatingPoints() {
|
|
113
104
|
clear();
|
|
114
105
|
}
|
|
115
106
|
|
|
116
|
-
void OperatingPoints::clear
|
|
117
|
-
{
|
|
107
|
+
void OperatingPoints::clear() {
|
|
118
108
|
all_pts.clear();
|
|
119
109
|
optimal_pts.clear();
|
|
120
110
|
/// default point: doing nothing gives 0 performance and takes 0 time
|
|
@@ -123,20 +113,22 @@ void OperatingPoints::clear ()
|
|
|
123
113
|
}
|
|
124
114
|
|
|
125
115
|
/// add a performance measure
|
|
126
|
-
bool OperatingPoints::add
|
|
127
|
-
|
|
128
|
-
|
|
116
|
+
bool OperatingPoints::add(
|
|
117
|
+
double perf,
|
|
118
|
+
double t,
|
|
119
|
+
const std::string& key,
|
|
120
|
+
size_t cno) {
|
|
129
121
|
OperatingPoint op = {perf, t, key, int64_t(cno)};
|
|
130
|
-
all_pts.push_back
|
|
122
|
+
all_pts.push_back(op);
|
|
131
123
|
if (perf == 0) {
|
|
132
|
-
return false;
|
|
124
|
+
return false; // no method for 0 accuracy is faster than doing nothing
|
|
133
125
|
}
|
|
134
|
-
std::vector<OperatingPoint
|
|
126
|
+
std::vector<OperatingPoint>& a = optimal_pts;
|
|
135
127
|
if (perf > a.back().perf) {
|
|
136
128
|
// keep unconditionally
|
|
137
|
-
a.push_back
|
|
129
|
+
a.push_back(op);
|
|
138
130
|
} else if (perf == a.back().perf) {
|
|
139
|
-
if (t < a.back
|
|
131
|
+
if (t < a.back().t) {
|
|
140
132
|
a.back() = op;
|
|
141
133
|
} else {
|
|
142
134
|
return false;
|
|
@@ -145,14 +137,15 @@ bool OperatingPoints::add (double perf, double t, const std::string & key,
|
|
|
145
137
|
int i;
|
|
146
138
|
// stricto sensu this should be a bissection
|
|
147
139
|
for (i = 0; i < a.size(); i++) {
|
|
148
|
-
if (a[i].perf >= perf)
|
|
140
|
+
if (a[i].perf >= perf)
|
|
141
|
+
break;
|
|
149
142
|
}
|
|
150
|
-
assert
|
|
143
|
+
assert(i < a.size());
|
|
151
144
|
if (t < a[i].t) {
|
|
152
145
|
if (a[i].perf == perf) {
|
|
153
146
|
a[i] = op;
|
|
154
147
|
} else {
|
|
155
|
-
a.insert
|
|
148
|
+
a.insert(a.begin() + i, op);
|
|
156
149
|
}
|
|
157
150
|
} else {
|
|
158
151
|
return false;
|
|
@@ -162,86 +155,82 @@ bool OperatingPoints::add (double perf, double t, const std::string & key,
|
|
|
162
155
|
int i = a.size() - 1;
|
|
163
156
|
while (i > 0) {
|
|
164
157
|
if (a[i].t < a[i - 1].t)
|
|
165
|
-
a.erase
|
|
158
|
+
a.erase(a.begin() + (i - 1));
|
|
166
159
|
i--;
|
|
167
160
|
}
|
|
168
161
|
}
|
|
169
162
|
return true;
|
|
170
163
|
}
|
|
171
164
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
{
|
|
165
|
+
int OperatingPoints::merge_with(
|
|
166
|
+
const OperatingPoints& other,
|
|
167
|
+
const std::string& prefix) {
|
|
176
168
|
int n_add = 0;
|
|
177
169
|
for (int i = 0; i < other.all_pts.size(); i++) {
|
|
178
|
-
const OperatingPoint
|
|
179
|
-
if (add
|
|
170
|
+
const OperatingPoint& op = other.all_pts[i];
|
|
171
|
+
if (add(op.perf, op.t, prefix + op.key, op.cno))
|
|
180
172
|
n_add++;
|
|
181
173
|
}
|
|
182
174
|
return n_add;
|
|
183
175
|
}
|
|
184
176
|
|
|
185
|
-
|
|
186
|
-
|
|
187
177
|
/// get time required to obtain a given performance measure
|
|
188
|
-
double OperatingPoints::t_for_perf
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
178
|
+
double OperatingPoints::t_for_perf(double perf) const {
|
|
179
|
+
const std::vector<OperatingPoint>& a = optimal_pts;
|
|
180
|
+
if (perf > a.back().perf)
|
|
181
|
+
return 1e50;
|
|
192
182
|
int i0 = -1, i1 = a.size() - 1;
|
|
193
183
|
while (i0 + 1 < i1) {
|
|
194
184
|
int imed = (i0 + i1 + 1) / 2;
|
|
195
|
-
if (a[imed].perf < perf)
|
|
196
|
-
|
|
185
|
+
if (a[imed].perf < perf)
|
|
186
|
+
i0 = imed;
|
|
187
|
+
else
|
|
188
|
+
i1 = imed;
|
|
197
189
|
}
|
|
198
190
|
return a[i1].t;
|
|
199
191
|
}
|
|
200
192
|
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
{
|
|
204
|
-
FILE *f = fopen(fname, "w");
|
|
193
|
+
void OperatingPoints::all_to_gnuplot(const char* fname) const {
|
|
194
|
+
FILE* f = fopen(fname, "w");
|
|
205
195
|
if (!f) {
|
|
206
|
-
fprintf
|
|
196
|
+
fprintf(stderr, "cannot open %s", fname);
|
|
207
197
|
perror("");
|
|
208
198
|
abort();
|
|
209
199
|
}
|
|
210
200
|
for (int i = 0; i < all_pts.size(); i++) {
|
|
211
|
-
const OperatingPoint
|
|
212
|
-
fprintf
|
|
201
|
+
const OperatingPoint& op = all_pts[i];
|
|
202
|
+
fprintf(f, "%g %g %s\n", op.perf, op.t, op.key.c_str());
|
|
213
203
|
}
|
|
214
204
|
fclose(f);
|
|
215
205
|
}
|
|
216
206
|
|
|
217
|
-
void OperatingPoints::optimal_to_gnuplot
|
|
218
|
-
|
|
219
|
-
FILE *f = fopen(fname, "w");
|
|
207
|
+
void OperatingPoints::optimal_to_gnuplot(const char* fname) const {
|
|
208
|
+
FILE* f = fopen(fname, "w");
|
|
220
209
|
if (!f) {
|
|
221
|
-
fprintf
|
|
210
|
+
fprintf(stderr, "cannot open %s", fname);
|
|
222
211
|
perror("");
|
|
223
212
|
abort();
|
|
224
213
|
}
|
|
225
214
|
double prev_perf = 0.0;
|
|
226
215
|
for (int i = 0; i < optimal_pts.size(); i++) {
|
|
227
|
-
const OperatingPoint
|
|
228
|
-
fprintf
|
|
229
|
-
fprintf
|
|
216
|
+
const OperatingPoint& op = optimal_pts[i];
|
|
217
|
+
fprintf(f, "%g %g\n", prev_perf, op.t);
|
|
218
|
+
fprintf(f, "%g %g %s\n", op.perf, op.t, op.key.c_str());
|
|
230
219
|
prev_perf = op.perf;
|
|
231
220
|
}
|
|
232
221
|
fclose(f);
|
|
233
222
|
}
|
|
234
223
|
|
|
235
|
-
void OperatingPoints::display
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
only_optimal ? optimal_pts : all_pts;
|
|
224
|
+
void OperatingPoints::display(bool only_optimal) const {
|
|
225
|
+
const std::vector<OperatingPoint>& pts =
|
|
226
|
+
only_optimal ? optimal_pts : all_pts;
|
|
239
227
|
printf("Tested %zd operating points, %zd ones are Pareto-optimal:\n",
|
|
240
|
-
all_pts.size(),
|
|
228
|
+
all_pts.size(),
|
|
229
|
+
optimal_pts.size());
|
|
241
230
|
|
|
242
231
|
for (int i = 0; i < pts.size(); i++) {
|
|
243
|
-
const OperatingPoint
|
|
244
|
-
const char
|
|
232
|
+
const OperatingPoint& op = pts[i];
|
|
233
|
+
const char* star = "";
|
|
245
234
|
if (!only_optimal) {
|
|
246
235
|
for (int j = 0; j < optimal_pts.size(); j++) {
|
|
247
236
|
if (op.cno == optimal_pts[j].cno) {
|
|
@@ -250,22 +239,25 @@ void OperatingPoints::display (bool only_optimal) const
|
|
|
250
239
|
}
|
|
251
240
|
}
|
|
252
241
|
}
|
|
253
|
-
printf
|
|
254
|
-
|
|
242
|
+
printf("cno=%" PRId64 " key=%s perf=%.4f t=%.3f %s\n",
|
|
243
|
+
op.cno,
|
|
244
|
+
op.key.c_str(),
|
|
245
|
+
op.perf,
|
|
246
|
+
op.t,
|
|
247
|
+
star);
|
|
255
248
|
}
|
|
256
|
-
|
|
257
249
|
}
|
|
258
250
|
|
|
259
251
|
/***************************************************************
|
|
260
252
|
* ParameterSpace
|
|
261
253
|
***************************************************************/
|
|
262
254
|
|
|
263
|
-
ParameterSpace::ParameterSpace
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
}
|
|
255
|
+
ParameterSpace::ParameterSpace()
|
|
256
|
+
: verbose(1),
|
|
257
|
+
n_experiments(500),
|
|
258
|
+
batchsize(1 << 30),
|
|
259
|
+
thread_over_batches(false),
|
|
260
|
+
min_test_duration(0) {}
|
|
269
261
|
|
|
270
262
|
/* not keeping this constructor as inheritors will call the parent
|
|
271
263
|
initialize()
|
|
@@ -281,8 +273,7 @@ ParameterSpace::ParameterSpace (Index *index):
|
|
|
281
273
|
}
|
|
282
274
|
#endif
|
|
283
275
|
|
|
284
|
-
size_t ParameterSpace::n_combinations
|
|
285
|
-
{
|
|
276
|
+
size_t ParameterSpace::n_combinations() const {
|
|
286
277
|
size_t n = 1;
|
|
287
278
|
for (int i = 0; i < parameter_ranges.size(); i++)
|
|
288
279
|
n *= parameter_ranges[i].values.size();
|
|
@@ -290,131 +281,128 @@ size_t ParameterSpace::n_combinations () const
|
|
|
290
281
|
}
|
|
291
282
|
|
|
292
283
|
/// get string representation of the combination
|
|
293
|
-
std::string ParameterSpace::combination_name
|
|
284
|
+
std::string ParameterSpace::combination_name(size_t cno) const {
|
|
294
285
|
char buf[1000], *wp = buf;
|
|
295
286
|
*wp = 0;
|
|
296
287
|
for (int i = 0; i < parameter_ranges.size(); i++) {
|
|
297
|
-
const ParameterRange
|
|
288
|
+
const ParameterRange& pr = parameter_ranges[i];
|
|
298
289
|
size_t j = cno % pr.values.size();
|
|
299
290
|
cno /= pr.values.size();
|
|
300
|
-
wp += snprintf
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
291
|
+
wp += snprintf(
|
|
292
|
+
wp,
|
|
293
|
+
buf + 1000 - wp,
|
|
294
|
+
"%s%s=%g",
|
|
295
|
+
i == 0 ? "" : ",",
|
|
296
|
+
pr.name.c_str(),
|
|
297
|
+
pr.values[j]);
|
|
298
|
+
}
|
|
299
|
+
return std::string(buf);
|
|
305
300
|
}
|
|
306
301
|
|
|
307
|
-
|
|
308
|
-
bool ParameterSpace::combination_ge (size_t c1, size_t c2) const
|
|
309
|
-
{
|
|
302
|
+
bool ParameterSpace::combination_ge(size_t c1, size_t c2) const {
|
|
310
303
|
for (int i = 0; i < parameter_ranges.size(); i++) {
|
|
311
304
|
int nval = parameter_ranges[i].values.size();
|
|
312
305
|
size_t j1 = c1 % nval;
|
|
313
306
|
size_t j2 = c2 % nval;
|
|
314
|
-
if (!(j1 >= j2))
|
|
307
|
+
if (!(j1 >= j2))
|
|
308
|
+
return false;
|
|
315
309
|
c1 /= nval;
|
|
316
310
|
c2 /= nval;
|
|
317
311
|
}
|
|
318
312
|
return true;
|
|
319
313
|
}
|
|
320
314
|
|
|
321
|
-
|
|
322
|
-
|
|
323
315
|
#define DC(classname) \
|
|
324
|
-
const classname
|
|
316
|
+
const classname* ix = dynamic_cast<const classname*>(index)
|
|
325
317
|
|
|
326
|
-
static void init_pq_ParameterRange
|
|
327
|
-
|
|
328
|
-
{
|
|
318
|
+
static void init_pq_ParameterRange(
|
|
319
|
+
const ProductQuantizer& pq,
|
|
320
|
+
ParameterRange& pr) {
|
|
329
321
|
if (pq.code_size % 4 == 0) {
|
|
330
322
|
// Polysemous not supported for code sizes that are not a
|
|
331
323
|
// multiple of 4
|
|
332
|
-
for (int i = 2; i <= pq.code_size * 8 / 2; i+= 2)
|
|
324
|
+
for (int i = 2; i <= pq.code_size * 8 / 2; i += 2)
|
|
333
325
|
pr.values.push_back(i);
|
|
334
326
|
}
|
|
335
|
-
pr.values.push_back
|
|
327
|
+
pr.values.push_back(pq.code_size * 8);
|
|
336
328
|
}
|
|
337
329
|
|
|
338
|
-
ParameterRange
|
|
339
|
-
{
|
|
340
|
-
for (auto & pr : parameter_ranges) {
|
|
330
|
+
ParameterRange& ParameterSpace::add_range(const std::string& name) {
|
|
331
|
+
for (auto& pr : parameter_ranges) {
|
|
341
332
|
if (pr.name == name) {
|
|
342
333
|
return pr;
|
|
343
334
|
}
|
|
344
335
|
}
|
|
345
|
-
parameter_ranges.push_back
|
|
346
|
-
parameter_ranges.back
|
|
347
|
-
return parameter_ranges.back
|
|
336
|
+
parameter_ranges.push_back(ParameterRange());
|
|
337
|
+
parameter_ranges.back().name = name;
|
|
338
|
+
return parameter_ranges.back();
|
|
348
339
|
}
|
|
349
340
|
|
|
350
|
-
|
|
351
341
|
/// initialize with reasonable parameters for this type of index
|
|
352
|
-
void ParameterSpace::initialize
|
|
353
|
-
{
|
|
354
|
-
if (DC (IndexPreTransform)) {
|
|
342
|
+
void ParameterSpace::initialize(const Index* index) {
|
|
343
|
+
if (DC(IndexPreTransform)) {
|
|
355
344
|
index = ix->index;
|
|
356
345
|
}
|
|
357
|
-
if (DC
|
|
358
|
-
ParameterRange
|
|
346
|
+
if (DC(IndexRefine)) {
|
|
347
|
+
ParameterRange& pr = add_range("k_factor_rf");
|
|
359
348
|
for (int i = 0; i <= 6; i++) {
|
|
360
|
-
pr.values.push_back
|
|
349
|
+
pr.values.push_back(1 << i);
|
|
361
350
|
}
|
|
362
351
|
index = ix->base_index;
|
|
363
352
|
}
|
|
364
|
-
if (DC
|
|
353
|
+
if (DC(IndexPreTransform)) {
|
|
365
354
|
index = ix->index;
|
|
366
355
|
}
|
|
367
356
|
|
|
368
|
-
if (DC
|
|
357
|
+
if (DC(IndexIVF)) {
|
|
369
358
|
{
|
|
370
|
-
ParameterRange
|
|
359
|
+
ParameterRange& pr = add_range("nprobe");
|
|
371
360
|
for (int i = 0; i < 13; i++) {
|
|
372
361
|
size_t nprobe = 1 << i;
|
|
373
|
-
if (nprobe >= ix->nlist)
|
|
374
|
-
|
|
362
|
+
if (nprobe >= ix->nlist)
|
|
363
|
+
break;
|
|
364
|
+
pr.values.push_back(nprobe);
|
|
375
365
|
}
|
|
376
366
|
}
|
|
377
367
|
ParameterSpace ivf_pspace;
|
|
378
368
|
ivf_pspace.initialize(ix->quantizer);
|
|
379
369
|
|
|
380
|
-
for (const ParameterRange
|
|
381
|
-
ParameterRange
|
|
370
|
+
for (const ParameterRange& p : ivf_pspace.parameter_ranges) {
|
|
371
|
+
ParameterRange& pr = add_range("quantizer_" + p.name);
|
|
382
372
|
pr.values = p.values;
|
|
383
373
|
}
|
|
384
|
-
|
|
385
374
|
}
|
|
386
|
-
if (DC
|
|
387
|
-
ParameterRange
|
|
388
|
-
init_pq_ParameterRange
|
|
375
|
+
if (DC(IndexPQ)) {
|
|
376
|
+
ParameterRange& pr = add_range("ht");
|
|
377
|
+
init_pq_ParameterRange(ix->pq, pr);
|
|
389
378
|
}
|
|
390
|
-
if (DC
|
|
391
|
-
ParameterRange
|
|
392
|
-
init_pq_ParameterRange
|
|
379
|
+
if (DC(IndexIVFPQ)) {
|
|
380
|
+
ParameterRange& pr = add_range("ht");
|
|
381
|
+
init_pq_ParameterRange(ix->pq, pr);
|
|
393
382
|
}
|
|
394
383
|
|
|
395
|
-
if (DC
|
|
396
|
-
const MultiIndexQuantizer
|
|
397
|
-
|
|
384
|
+
if (DC(IndexIVF)) {
|
|
385
|
+
const MultiIndexQuantizer* miq =
|
|
386
|
+
dynamic_cast<const MultiIndexQuantizer*>(ix->quantizer);
|
|
398
387
|
if (miq) {
|
|
399
|
-
ParameterRange
|
|
388
|
+
ParameterRange& pr_max_codes = add_range("max_codes");
|
|
400
389
|
for (int i = 8; i < 20; i++) {
|
|
401
|
-
pr_max_codes.values.push_back
|
|
390
|
+
pr_max_codes.values.push_back(1 << i);
|
|
402
391
|
}
|
|
403
|
-
pr_max_codes.values.push_back
|
|
404
|
-
|
|
405
|
-
);
|
|
392
|
+
pr_max_codes.values.push_back(
|
|
393
|
+
std::numeric_limits<double>::infinity());
|
|
406
394
|
}
|
|
407
395
|
}
|
|
408
|
-
if (DC
|
|
409
|
-
ParameterRange
|
|
396
|
+
if (DC(IndexIVFPQR)) {
|
|
397
|
+
ParameterRange& pr = add_range("k_factor");
|
|
410
398
|
for (int i = 0; i <= 6; i++) {
|
|
411
|
-
pr.values.push_back
|
|
399
|
+
pr.values.push_back(1 << i);
|
|
412
400
|
}
|
|
413
401
|
}
|
|
414
402
|
if (dynamic_cast<const IndexHNSW*>(index)) {
|
|
415
|
-
ParameterRange
|
|
403
|
+
ParameterRange& pr = add_range("efSearch");
|
|
416
404
|
for (int i = 2; i <= 9; i++) {
|
|
417
|
-
pr.values.push_back
|
|
405
|
+
pr.values.push_back(1 << i);
|
|
418
406
|
}
|
|
419
407
|
}
|
|
420
408
|
}
|
|
@@ -422,45 +410,41 @@ void ParameterSpace::initialize (const Index * index)
|
|
|
422
410
|
#undef DC
|
|
423
411
|
|
|
424
412
|
// non-const version
|
|
425
|
-
#define DC(classname) classname
|
|
426
|
-
|
|
413
|
+
#define DC(classname) classname* ix = dynamic_cast<classname*>(index)
|
|
427
414
|
|
|
428
415
|
/// set a combination of parameters on an index
|
|
429
|
-
void ParameterSpace::set_index_parameters
|
|
430
|
-
{
|
|
431
|
-
|
|
416
|
+
void ParameterSpace::set_index_parameters(Index* index, size_t cno) const {
|
|
432
417
|
for (int i = 0; i < parameter_ranges.size(); i++) {
|
|
433
|
-
const ParameterRange
|
|
418
|
+
const ParameterRange& pr = parameter_ranges[i];
|
|
434
419
|
size_t j = cno % pr.values.size();
|
|
435
420
|
cno /= pr.values.size();
|
|
436
|
-
double val = pr.values
|
|
437
|
-
set_index_parameter
|
|
421
|
+
double val = pr.values[j];
|
|
422
|
+
set_index_parameter(index, pr.name, val);
|
|
438
423
|
}
|
|
439
424
|
}
|
|
440
425
|
|
|
441
426
|
/// set a combination of parameters on an index
|
|
442
|
-
void ParameterSpace::set_index_parameters
|
|
443
|
-
|
|
444
|
-
{
|
|
427
|
+
void ParameterSpace::set_index_parameters(
|
|
428
|
+
Index* index,
|
|
429
|
+
const char* description_in) const {
|
|
445
430
|
std::string description(description_in);
|
|
446
|
-
char
|
|
431
|
+
char* ptr;
|
|
447
432
|
|
|
448
|
-
for (char
|
|
449
|
-
tok
|
|
450
|
-
tok = strtok_r (nullptr, " ,", &ptr)) {
|
|
433
|
+
for (char* tok = strtok_r(&description[0], " ,", &ptr); tok;
|
|
434
|
+
tok = strtok_r(nullptr, " ,", &ptr)) {
|
|
451
435
|
char name[100];
|
|
452
436
|
double val;
|
|
453
|
-
int ret = sscanf
|
|
454
|
-
FAISS_THROW_IF_NOT_FMT
|
|
455
|
-
|
|
456
|
-
set_index_parameter
|
|
437
|
+
int ret = sscanf(tok, "%99[^=]=%lf", name, &val);
|
|
438
|
+
FAISS_THROW_IF_NOT_FMT(
|
|
439
|
+
ret == 2, "could not interpret parameters %s", tok);
|
|
440
|
+
set_index_parameter(index, name, val);
|
|
457
441
|
}
|
|
458
|
-
|
|
459
442
|
}
|
|
460
443
|
|
|
461
|
-
void ParameterSpace::set_index_parameter
|
|
462
|
-
Index
|
|
463
|
-
|
|
444
|
+
void ParameterSpace::set_index_parameter(
|
|
445
|
+
Index* index,
|
|
446
|
+
const std::string& name,
|
|
447
|
+
double val) const {
|
|
464
448
|
if (verbose > 1) {
|
|
465
449
|
printf(" set_index_parameter %s=%g\n", name.c_str(), val);
|
|
466
450
|
}
|
|
@@ -469,30 +453,29 @@ void ParameterSpace::set_index_parameter (
|
|
|
469
453
|
index->verbose = int(val);
|
|
470
454
|
// and fall through to also enable it on sub-indexes
|
|
471
455
|
}
|
|
472
|
-
if (DC
|
|
473
|
-
set_index_parameter
|
|
456
|
+
if (DC(IndexIDMap)) {
|
|
457
|
+
set_index_parameter(ix->index, name, val);
|
|
474
458
|
return;
|
|
475
459
|
}
|
|
476
|
-
if (DC
|
|
477
|
-
set_index_parameter
|
|
460
|
+
if (DC(IndexPreTransform)) {
|
|
461
|
+
set_index_parameter(ix->index, name, val);
|
|
478
462
|
return;
|
|
479
463
|
}
|
|
480
|
-
if (DC
|
|
464
|
+
if (DC(ThreadedIndex<Index>)) {
|
|
481
465
|
// call on all sub-indexes
|
|
482
|
-
auto fn =
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
};
|
|
466
|
+
auto fn = [this, name, val](int /* no */, Index* subIndex) {
|
|
467
|
+
set_index_parameter(subIndex, name, val);
|
|
468
|
+
};
|
|
486
469
|
ix->runOnIndex(fn);
|
|
487
470
|
return;
|
|
488
471
|
}
|
|
489
|
-
if (DC
|
|
472
|
+
if (DC(IndexRefine)) {
|
|
490
473
|
if (name == "k_factor_rf") {
|
|
491
474
|
ix->k_factor = int(val);
|
|
492
475
|
return;
|
|
493
476
|
}
|
|
494
477
|
// otherwise it is for the sub-index
|
|
495
|
-
set_index_parameter
|
|
478
|
+
set_index_parameter(ix->base_index, name, val);
|
|
496
479
|
return;
|
|
497
480
|
}
|
|
498
481
|
|
|
@@ -502,14 +485,14 @@ void ParameterSpace::set_index_parameter (
|
|
|
502
485
|
}
|
|
503
486
|
|
|
504
487
|
if (name == "nprobe") {
|
|
505
|
-
if (DC
|
|
488
|
+
if (DC(IndexIVF)) {
|
|
506
489
|
ix->nprobe = int(val);
|
|
507
490
|
return;
|
|
508
491
|
}
|
|
509
492
|
}
|
|
510
493
|
|
|
511
494
|
if (name == "ht") {
|
|
512
|
-
if (DC
|
|
495
|
+
if (DC(IndexPQ)) {
|
|
513
496
|
if (val >= ix->pq.code_size * 8) {
|
|
514
497
|
ix->search_type = IndexPQ::ST_PQ;
|
|
515
498
|
} else {
|
|
@@ -517,7 +500,7 @@ void ParameterSpace::set_index_parameter (
|
|
|
517
500
|
ix->polysemous_ht = int(val);
|
|
518
501
|
}
|
|
519
502
|
return;
|
|
520
|
-
} else if (DC
|
|
503
|
+
} else if (DC(IndexIVFPQ)) {
|
|
521
504
|
if (val >= ix->pq.code_size * 8) {
|
|
522
505
|
ix->polysemous_ht = 0;
|
|
523
506
|
} else {
|
|
@@ -528,26 +511,25 @@ void ParameterSpace::set_index_parameter (
|
|
|
528
511
|
}
|
|
529
512
|
|
|
530
513
|
if (name == "k_factor") {
|
|
531
|
-
if (DC
|
|
514
|
+
if (DC(IndexIVFPQR)) {
|
|
532
515
|
ix->k_factor = val;
|
|
533
516
|
return;
|
|
534
517
|
}
|
|
535
518
|
}
|
|
536
519
|
if (name == "max_codes") {
|
|
537
|
-
if (DC
|
|
520
|
+
if (DC(IndexIVF)) {
|
|
538
521
|
ix->max_codes = std::isfinite(val) ? size_t(val) : 0;
|
|
539
522
|
return;
|
|
540
523
|
}
|
|
541
524
|
}
|
|
542
525
|
|
|
543
526
|
if (name == "efSearch") {
|
|
544
|
-
if (DC
|
|
527
|
+
if (DC(IndexHNSW)) {
|
|
545
528
|
ix->hnsw.efSearch = int(val);
|
|
546
529
|
return;
|
|
547
530
|
}
|
|
548
|
-
if (DC
|
|
549
|
-
if (IndexHNSW
|
|
550
|
-
dynamic_cast<IndexHNSW *>(ix->quantizer)) {
|
|
531
|
+
if (DC(IndexIVF)) {
|
|
532
|
+
if (IndexHNSW* cq = dynamic_cast<IndexHNSW*>(ix->quantizer)) {
|
|
551
533
|
cq->hnsw.efSearch = int(val);
|
|
552
534
|
return;
|
|
553
535
|
}
|
|
@@ -562,71 +544,75 @@ void ParameterSpace::set_index_parameter (
|
|
|
562
544
|
}
|
|
563
545
|
}
|
|
564
546
|
|
|
565
|
-
FAISS_THROW_FMT
|
|
566
|
-
|
|
567
|
-
|
|
547
|
+
FAISS_THROW_FMT(
|
|
548
|
+
"ParameterSpace::set_index_parameter:"
|
|
549
|
+
"could not set parameter %s",
|
|
550
|
+
name.c_str());
|
|
568
551
|
}
|
|
569
552
|
|
|
570
|
-
void ParameterSpace::display
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
553
|
+
void ParameterSpace::display() const {
|
|
554
|
+
printf("ParameterSpace, %zd parameters, %zd combinations:\n",
|
|
555
|
+
parameter_ranges.size(),
|
|
556
|
+
n_combinations());
|
|
574
557
|
for (int i = 0; i < parameter_ranges.size(); i++) {
|
|
575
|
-
const ParameterRange
|
|
576
|
-
printf
|
|
558
|
+
const ParameterRange& pr = parameter_ranges[i];
|
|
559
|
+
printf(" %s: ", pr.name.c_str());
|
|
577
560
|
char sep = '[';
|
|
578
561
|
for (int j = 0; j < pr.values.size(); j++) {
|
|
579
|
-
printf
|
|
562
|
+
printf("%c %g", sep, pr.values[j]);
|
|
580
563
|
sep = ',';
|
|
581
564
|
}
|
|
582
|
-
printf
|
|
565
|
+
printf("]\n");
|
|
583
566
|
}
|
|
584
567
|
}
|
|
585
568
|
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
{
|
|
592
|
-
|
|
593
|
-
|
|
569
|
+
void ParameterSpace::update_bounds(
|
|
570
|
+
size_t cno,
|
|
571
|
+
const OperatingPoint& op,
|
|
572
|
+
double* upper_bound_perf,
|
|
573
|
+
double* lower_bound_t) const {
|
|
574
|
+
if (combination_ge(cno, op.cno)) {
|
|
575
|
+
if (op.t > *lower_bound_t)
|
|
576
|
+
*lower_bound_t = op.t;
|
|
594
577
|
}
|
|
595
|
-
if (combination_ge
|
|
596
|
-
if (op.perf < *upper_bound_perf)
|
|
578
|
+
if (combination_ge(op.cno, cno)) {
|
|
579
|
+
if (op.perf < *upper_bound_perf)
|
|
580
|
+
*upper_bound_perf = op.perf;
|
|
597
581
|
}
|
|
598
582
|
}
|
|
599
583
|
|
|
584
|
+
void ParameterSpace::explore(
|
|
585
|
+
Index* index,
|
|
586
|
+
size_t nq,
|
|
587
|
+
const float* xq,
|
|
588
|
+
const AutoTuneCriterion& crit,
|
|
589
|
+
OperatingPoints* ops) const {
|
|
590
|
+
FAISS_THROW_IF_NOT_MSG(
|
|
591
|
+
nq == crit.nq, "criterion does not have the same nb of queries");
|
|
600
592
|
|
|
601
|
-
|
|
602
|
-
void ParameterSpace::explore (Index *index,
|
|
603
|
-
size_t nq, const float *xq,
|
|
604
|
-
const AutoTuneCriterion & crit,
|
|
605
|
-
OperatingPoints * ops) const
|
|
606
|
-
{
|
|
607
|
-
FAISS_THROW_IF_NOT_MSG (nq == crit.nq,
|
|
608
|
-
"criterion does not have the same nb of queries");
|
|
609
|
-
|
|
610
|
-
size_t n_comb = n_combinations ();
|
|
593
|
+
size_t n_comb = n_combinations();
|
|
611
594
|
|
|
612
595
|
if (n_experiments == 0) {
|
|
613
|
-
|
|
614
596
|
for (size_t cno = 0; cno < n_comb; cno++) {
|
|
615
|
-
set_index_parameters
|
|
597
|
+
set_index_parameters(index, cno);
|
|
616
598
|
std::vector<Index::idx_t> I(nq * crit.nnn);
|
|
617
599
|
std::vector<float> D(nq * crit.nnn);
|
|
618
600
|
|
|
619
|
-
double t0 = getmillisecs
|
|
620
|
-
index->search
|
|
601
|
+
double t0 = getmillisecs();
|
|
602
|
+
index->search(nq, xq, crit.nnn, D.data(), I.data());
|
|
621
603
|
double t_search = (getmillisecs() - t0) / 1e3;
|
|
622
604
|
|
|
623
|
-
double perf = crit.evaluate
|
|
605
|
+
double perf = crit.evaluate(D.data(), I.data());
|
|
624
606
|
|
|
625
|
-
bool keep = ops->add
|
|
607
|
+
bool keep = ops->add(perf, t_search, combination_name(cno), cno);
|
|
626
608
|
|
|
627
609
|
if (verbose)
|
|
628
|
-
printf(" %zd/%zd: %s perf=%.3f t=%.3f s %s\n",
|
|
629
|
-
|
|
610
|
+
printf(" %zd/%zd: %s perf=%.3f t=%.3f s %s\n",
|
|
611
|
+
cno,
|
|
612
|
+
n_comb,
|
|
613
|
+
combination_name(cno).c_str(),
|
|
614
|
+
perf,
|
|
615
|
+
t_search,
|
|
630
616
|
keep ? "*" : "");
|
|
631
617
|
}
|
|
632
618
|
return;
|
|
@@ -634,89 +620,104 @@ void ParameterSpace::explore (Index *index,
|
|
|
634
620
|
|
|
635
621
|
int n_exp = n_experiments;
|
|
636
622
|
|
|
637
|
-
if (n_exp > n_comb)
|
|
638
|
-
|
|
639
|
-
|
|
623
|
+
if (n_exp > n_comb)
|
|
624
|
+
n_exp = n_comb;
|
|
625
|
+
FAISS_THROW_IF_NOT(n_comb == 1 || n_exp > 2);
|
|
626
|
+
std::vector<int> perm(n_comb);
|
|
640
627
|
// make sure the slowest and fastest experiment are run
|
|
641
628
|
perm[0] = 0;
|
|
642
629
|
if (n_comb > 1) {
|
|
643
630
|
perm[1] = n_comb - 1;
|
|
644
|
-
rand_perm
|
|
645
|
-
for (int i = 2; i < perm.size(); i++)
|
|
631
|
+
rand_perm(&perm[2], n_comb - 2, 1234);
|
|
632
|
+
for (int i = 2; i < perm.size(); i++)
|
|
633
|
+
perm[i]++;
|
|
646
634
|
}
|
|
647
635
|
|
|
648
636
|
for (size_t xp = 0; xp < n_exp; xp++) {
|
|
649
637
|
size_t cno = perm[xp];
|
|
650
638
|
|
|
651
639
|
if (verbose)
|
|
652
|
-
printf(" %zd/%d: cno=%zd %s ",
|
|
653
|
-
|
|
640
|
+
printf(" %zd/%d: cno=%zd %s ",
|
|
641
|
+
xp,
|
|
642
|
+
n_exp,
|
|
643
|
+
cno,
|
|
644
|
+
combination_name(cno).c_str());
|
|
654
645
|
|
|
655
646
|
{
|
|
656
647
|
double lower_bound_t = 0.0;
|
|
657
648
|
double upper_bound_perf = 1.0;
|
|
658
649
|
for (int i = 0; i < ops->all_pts.size(); i++) {
|
|
659
|
-
update_bounds
|
|
660
|
-
|
|
650
|
+
update_bounds(
|
|
651
|
+
cno,
|
|
652
|
+
ops->all_pts[i],
|
|
653
|
+
&upper_bound_perf,
|
|
654
|
+
&lower_bound_t);
|
|
661
655
|
}
|
|
662
|
-
double best_t = ops->t_for_perf
|
|
656
|
+
double best_t = ops->t_for_perf(upper_bound_perf);
|
|
663
657
|
if (verbose)
|
|
664
|
-
printf
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
658
|
+
printf("bounds [perf<=%.3f t>=%.3f] %s",
|
|
659
|
+
upper_bound_perf,
|
|
660
|
+
lower_bound_t,
|
|
661
|
+
best_t <= lower_bound_t ? "skip\n" : "");
|
|
662
|
+
if (best_t <= lower_bound_t)
|
|
663
|
+
continue;
|
|
668
664
|
}
|
|
669
665
|
|
|
670
|
-
set_index_parameters
|
|
666
|
+
set_index_parameters(index, cno);
|
|
671
667
|
std::vector<Index::idx_t> I(nq * crit.nnn);
|
|
672
668
|
std::vector<float> D(nq * crit.nnn);
|
|
673
669
|
|
|
674
|
-
double t0 = getmillisecs
|
|
670
|
+
double t0 = getmillisecs();
|
|
675
671
|
|
|
676
672
|
int nrun = 0;
|
|
677
673
|
double t_search;
|
|
678
674
|
|
|
679
675
|
do {
|
|
680
|
-
|
|
681
676
|
if (thread_over_batches) {
|
|
682
677
|
#pragma omp parallel for
|
|
683
678
|
for (Index::idx_t q0 = 0; q0 < nq; q0 += batchsize) {
|
|
684
679
|
size_t q1 = q0 + batchsize;
|
|
685
|
-
if (q1 > nq)
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
680
|
+
if (q1 > nq)
|
|
681
|
+
q1 = nq;
|
|
682
|
+
index->search(
|
|
683
|
+
q1 - q0,
|
|
684
|
+
xq + q0 * index->d,
|
|
685
|
+
crit.nnn,
|
|
686
|
+
D.data() + q0 * crit.nnn,
|
|
687
|
+
I.data() + q0 * crit.nnn);
|
|
690
688
|
}
|
|
691
689
|
} else {
|
|
692
690
|
for (size_t q0 = 0; q0 < nq; q0 += batchsize) {
|
|
693
691
|
size_t q1 = q0 + batchsize;
|
|
694
|
-
if (q1 > nq)
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
692
|
+
if (q1 > nq)
|
|
693
|
+
q1 = nq;
|
|
694
|
+
index->search(
|
|
695
|
+
q1 - q0,
|
|
696
|
+
xq + q0 * index->d,
|
|
697
|
+
crit.nnn,
|
|
698
|
+
D.data() + q0 * crit.nnn,
|
|
699
|
+
I.data() + q0 * crit.nnn);
|
|
699
700
|
}
|
|
700
701
|
}
|
|
701
|
-
nrun
|
|
702
|
+
nrun++;
|
|
702
703
|
t_search = (getmillisecs() - t0) / 1e3;
|
|
703
704
|
|
|
704
705
|
} while (t_search < min_test_duration);
|
|
705
706
|
|
|
706
707
|
t_search /= nrun;
|
|
707
708
|
|
|
708
|
-
double perf = crit.evaluate
|
|
709
|
+
double perf = crit.evaluate(D.data(), I.data());
|
|
709
710
|
|
|
710
|
-
bool keep = ops->add
|
|
711
|
+
bool keep = ops->add(perf, t_search, combination_name(cno), cno);
|
|
711
712
|
|
|
712
713
|
if (verbose)
|
|
713
714
|
printf(" perf %.3f t %.3f (%d %s) %s\n",
|
|
714
|
-
perf,
|
|
715
|
+
perf,
|
|
716
|
+
t_search,
|
|
717
|
+
nrun,
|
|
718
|
+
nrun >= 2 ? "runs" : "run",
|
|
715
719
|
keep ? "*" : "");
|
|
716
720
|
}
|
|
717
721
|
}
|
|
718
722
|
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
723
|
} // namespace faiss
|