faiss 0.4.3 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/README.md +2 -0
- data/ext/faiss/index.cpp +33 -6
- data/ext/faiss/index_binary.cpp +17 -4
- data/ext/faiss/kmeans.cpp +6 -6
- data/lib/faiss/version.rb +1 -1
- data/vendor/faiss/faiss/AutoTune.cpp +2 -3
- data/vendor/faiss/faiss/AutoTune.h +1 -1
- data/vendor/faiss/faiss/Clustering.cpp +2 -2
- data/vendor/faiss/faiss/Clustering.h +2 -2
- data/vendor/faiss/faiss/IVFlib.cpp +26 -51
- data/vendor/faiss/faiss/IVFlib.h +1 -1
- data/vendor/faiss/faiss/Index.cpp +11 -0
- data/vendor/faiss/faiss/Index.h +34 -11
- data/vendor/faiss/faiss/Index2Layer.cpp +1 -1
- data/vendor/faiss/faiss/Index2Layer.h +2 -2
- data/vendor/faiss/faiss/IndexAdditiveQuantizer.cpp +1 -0
- data/vendor/faiss/faiss/IndexAdditiveQuantizerFastScan.cpp +9 -4
- data/vendor/faiss/faiss/IndexAdditiveQuantizerFastScan.h +5 -1
- data/vendor/faiss/faiss/IndexBinary.h +7 -7
- data/vendor/faiss/faiss/IndexBinaryFromFloat.h +1 -1
- data/vendor/faiss/faiss/IndexBinaryHNSW.cpp +8 -2
- data/vendor/faiss/faiss/IndexBinaryHNSW.h +1 -1
- data/vendor/faiss/faiss/IndexBinaryHash.cpp +3 -3
- data/vendor/faiss/faiss/IndexBinaryHash.h +5 -5
- data/vendor/faiss/faiss/IndexBinaryIVF.cpp +7 -6
- data/vendor/faiss/faiss/IndexFastScan.cpp +125 -49
- data/vendor/faiss/faiss/IndexFastScan.h +102 -7
- data/vendor/faiss/faiss/IndexFlat.cpp +374 -4
- data/vendor/faiss/faiss/IndexFlat.h +81 -1
- data/vendor/faiss/faiss/IndexHNSW.cpp +93 -2
- data/vendor/faiss/faiss/IndexHNSW.h +58 -2
- data/vendor/faiss/faiss/IndexIDMap.cpp +14 -13
- data/vendor/faiss/faiss/IndexIDMap.h +6 -6
- data/vendor/faiss/faiss/IndexIVF.cpp +1 -1
- data/vendor/faiss/faiss/IndexIVF.h +5 -5
- data/vendor/faiss/faiss/IndexIVFAdditiveQuantizer.cpp +1 -1
- data/vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.cpp +9 -3
- data/vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.h +3 -1
- data/vendor/faiss/faiss/IndexIVFFastScan.cpp +176 -90
- data/vendor/faiss/faiss/IndexIVFFastScan.h +173 -18
- data/vendor/faiss/faiss/IndexIVFFlat.cpp +1 -0
- data/vendor/faiss/faiss/IndexIVFFlatPanorama.cpp +251 -0
- data/vendor/faiss/faiss/IndexIVFFlatPanorama.h +64 -0
- data/vendor/faiss/faiss/IndexIVFPQ.cpp +3 -1
- data/vendor/faiss/faiss/IndexIVFPQ.h +1 -1
- data/vendor/faiss/faiss/IndexIVFPQFastScan.cpp +134 -2
- data/vendor/faiss/faiss/IndexIVFPQFastScan.h +7 -1
- data/vendor/faiss/faiss/IndexIVFRaBitQ.cpp +99 -8
- data/vendor/faiss/faiss/IndexIVFRaBitQ.h +4 -1
- data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.cpp +828 -0
- data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.h +252 -0
- data/vendor/faiss/faiss/IndexIVFSpectralHash.cpp +1 -1
- data/vendor/faiss/faiss/IndexIVFSpectralHash.h +1 -1
- data/vendor/faiss/faiss/IndexNNDescent.cpp +1 -1
- data/vendor/faiss/faiss/IndexNSG.cpp +1 -1
- data/vendor/faiss/faiss/IndexNeuralNetCodec.h +1 -1
- data/vendor/faiss/faiss/IndexPQ.cpp +4 -1
- data/vendor/faiss/faiss/IndexPQ.h +1 -1
- data/vendor/faiss/faiss/IndexPQFastScan.cpp +6 -2
- data/vendor/faiss/faiss/IndexPQFastScan.h +5 -1
- data/vendor/faiss/faiss/IndexPreTransform.cpp +14 -0
- data/vendor/faiss/faiss/IndexPreTransform.h +9 -0
- data/vendor/faiss/faiss/IndexRaBitQ.cpp +96 -13
- data/vendor/faiss/faiss/IndexRaBitQ.h +11 -2
- data/vendor/faiss/faiss/IndexRaBitQFastScan.cpp +731 -0
- data/vendor/faiss/faiss/IndexRaBitQFastScan.h +175 -0
- data/vendor/faiss/faiss/IndexRefine.cpp +49 -0
- data/vendor/faiss/faiss/IndexRefine.h +17 -0
- data/vendor/faiss/faiss/IndexShards.cpp +1 -1
- data/vendor/faiss/faiss/MatrixStats.cpp +3 -3
- data/vendor/faiss/faiss/MetricType.h +1 -1
- data/vendor/faiss/faiss/VectorTransform.h +2 -2
- data/vendor/faiss/faiss/clone_index.cpp +5 -1
- data/vendor/faiss/faiss/gpu/GpuCloner.cpp +1 -1
- data/vendor/faiss/faiss/gpu/GpuClonerOptions.h +3 -1
- data/vendor/faiss/faiss/gpu/GpuIndex.h +11 -11
- data/vendor/faiss/faiss/gpu/GpuIndexBinaryCagra.h +1 -1
- data/vendor/faiss/faiss/gpu/GpuIndexBinaryFlat.h +1 -1
- data/vendor/faiss/faiss/gpu/GpuIndexCagra.h +11 -7
- data/vendor/faiss/faiss/gpu/StandardGpuResources.cpp +1 -1
- data/vendor/faiss/faiss/gpu/perf/IndexWrapper-inl.h +2 -0
- data/vendor/faiss/faiss/gpu/test/TestGpuIcmEncoder.cpp +7 -0
- data/vendor/faiss/faiss/gpu/test/TestGpuIndexIVFFlat.cpp +1 -1
- data/vendor/faiss/faiss/impl/AdditiveQuantizer.cpp +1 -1
- data/vendor/faiss/faiss/impl/AdditiveQuantizer.h +1 -1
- data/vendor/faiss/faiss/impl/AuxIndexStructures.cpp +2 -2
- data/vendor/faiss/faiss/impl/AuxIndexStructures.h +1 -1
- data/vendor/faiss/faiss/impl/CodePacker.h +2 -2
- data/vendor/faiss/faiss/impl/DistanceComputer.h +77 -6
- data/vendor/faiss/faiss/impl/FastScanDistancePostProcessing.h +53 -0
- data/vendor/faiss/faiss/impl/HNSW.cpp +295 -16
- data/vendor/faiss/faiss/impl/HNSW.h +35 -6
- data/vendor/faiss/faiss/impl/IDSelector.cpp +2 -2
- data/vendor/faiss/faiss/impl/IDSelector.h +4 -4
- data/vendor/faiss/faiss/impl/LocalSearchQuantizer.cpp +4 -4
- data/vendor/faiss/faiss/impl/LocalSearchQuantizer.h +1 -1
- data/vendor/faiss/faiss/impl/LookupTableScaler.h +1 -1
- data/vendor/faiss/faiss/impl/NNDescent.cpp +1 -1
- data/vendor/faiss/faiss/impl/NNDescent.h +2 -2
- data/vendor/faiss/faiss/impl/NSG.cpp +1 -1
- data/vendor/faiss/faiss/impl/Panorama.cpp +193 -0
- data/vendor/faiss/faiss/impl/Panorama.h +204 -0
- data/vendor/faiss/faiss/impl/PanoramaStats.cpp +33 -0
- data/vendor/faiss/faiss/impl/PanoramaStats.h +38 -0
- data/vendor/faiss/faiss/impl/PolysemousTraining.cpp +5 -5
- data/vendor/faiss/faiss/impl/ProductAdditiveQuantizer.cpp +1 -1
- data/vendor/faiss/faiss/impl/ProductAdditiveQuantizer.h +1 -1
- data/vendor/faiss/faiss/impl/ProductQuantizer-inl.h +2 -0
- data/vendor/faiss/faiss/impl/ProductQuantizer.h +1 -1
- data/vendor/faiss/faiss/impl/RaBitQStats.cpp +29 -0
- data/vendor/faiss/faiss/impl/RaBitQStats.h +56 -0
- data/vendor/faiss/faiss/impl/RaBitQUtils.cpp +294 -0
- data/vendor/faiss/faiss/impl/RaBitQUtils.h +330 -0
- data/vendor/faiss/faiss/impl/RaBitQuantizer.cpp +304 -223
- data/vendor/faiss/faiss/impl/RaBitQuantizer.h +72 -4
- data/vendor/faiss/faiss/impl/RaBitQuantizerMultiBit.cpp +362 -0
- data/vendor/faiss/faiss/impl/RaBitQuantizerMultiBit.h +112 -0
- data/vendor/faiss/faiss/impl/ResidualQuantizer.h +1 -1
- data/vendor/faiss/faiss/impl/ResultHandler.h +4 -4
- data/vendor/faiss/faiss/impl/ScalarQuantizer.cpp +7 -10
- data/vendor/faiss/faiss/impl/ScalarQuantizer.h +2 -4
- data/vendor/faiss/faiss/impl/ThreadedIndex-inl.h +7 -4
- data/vendor/faiss/faiss/impl/index_read.cpp +238 -10
- data/vendor/faiss/faiss/impl/index_write.cpp +212 -19
- data/vendor/faiss/faiss/impl/io.cpp +2 -2
- data/vendor/faiss/faiss/impl/io.h +4 -4
- data/vendor/faiss/faiss/impl/kmeans1d.cpp +1 -1
- data/vendor/faiss/faiss/impl/kmeans1d.h +1 -1
- data/vendor/faiss/faiss/impl/lattice_Zn.h +2 -2
- data/vendor/faiss/faiss/impl/mapped_io.cpp +2 -2
- data/vendor/faiss/faiss/impl/mapped_io.h +4 -3
- data/vendor/faiss/faiss/impl/maybe_owned_vector.h +8 -1
- data/vendor/faiss/faiss/impl/platform_macros.h +12 -0
- data/vendor/faiss/faiss/impl/pq4_fast_scan.cpp +30 -4
- data/vendor/faiss/faiss/impl/pq4_fast_scan.h +14 -8
- data/vendor/faiss/faiss/impl/pq4_fast_scan_search_qbs.cpp +5 -6
- data/vendor/faiss/faiss/impl/simd_result_handlers.h +55 -11
- data/vendor/faiss/faiss/impl/svs_io.cpp +86 -0
- data/vendor/faiss/faiss/impl/svs_io.h +67 -0
- data/vendor/faiss/faiss/impl/zerocopy_io.h +1 -1
- data/vendor/faiss/faiss/index_factory.cpp +217 -8
- data/vendor/faiss/faiss/index_factory.h +1 -1
- data/vendor/faiss/faiss/index_io.h +1 -1
- data/vendor/faiss/faiss/invlists/BlockInvertedLists.h +1 -1
- data/vendor/faiss/faiss/invlists/DirectMap.cpp +1 -1
- data/vendor/faiss/faiss/invlists/InvertedLists.cpp +115 -1
- data/vendor/faiss/faiss/invlists/InvertedLists.h +46 -0
- data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.cpp +1 -1
- data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.h +1 -1
- data/vendor/faiss/faiss/svs/IndexSVSFaissUtils.h +261 -0
- data/vendor/faiss/faiss/svs/IndexSVSFlat.cpp +117 -0
- data/vendor/faiss/faiss/svs/IndexSVSFlat.h +66 -0
- data/vendor/faiss/faiss/svs/IndexSVSVamana.cpp +245 -0
- data/vendor/faiss/faiss/svs/IndexSVSVamana.h +137 -0
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.cpp +39 -0
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.h +42 -0
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.cpp +149 -0
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.h +58 -0
- data/vendor/faiss/faiss/utils/AlignedTable.h +1 -1
- data/vendor/faiss/faiss/utils/Heap.cpp +2 -2
- data/vendor/faiss/faiss/utils/Heap.h +3 -3
- data/vendor/faiss/faiss/utils/NeuralNet.cpp +1 -1
- data/vendor/faiss/faiss/utils/NeuralNet.h +3 -3
- data/vendor/faiss/faiss/utils/approx_topk/approx_topk.h +2 -2
- data/vendor/faiss/faiss/utils/approx_topk/avx2-inl.h +2 -2
- data/vendor/faiss/faiss/utils/approx_topk/mode.h +1 -1
- data/vendor/faiss/faiss/utils/distances.cpp +0 -3
- data/vendor/faiss/faiss/utils/distances.h +2 -2
- data/vendor/faiss/faiss/utils/extra_distances-inl.h +3 -1
- data/vendor/faiss/faiss/utils/hamming-inl.h +2 -0
- data/vendor/faiss/faiss/utils/hamming.cpp +7 -6
- data/vendor/faiss/faiss/utils/hamming.h +1 -1
- data/vendor/faiss/faiss/utils/hamming_distance/common.h +1 -2
- data/vendor/faiss/faiss/utils/partitioning.cpp +5 -5
- data/vendor/faiss/faiss/utils/partitioning.h +2 -2
- data/vendor/faiss/faiss/utils/rabitq_simd.h +222 -336
- data/vendor/faiss/faiss/utils/random.cpp +1 -1
- data/vendor/faiss/faiss/utils/simdlib_avx2.h +1 -1
- data/vendor/faiss/faiss/utils/simdlib_avx512.h +1 -1
- data/vendor/faiss/faiss/utils/simdlib_neon.h +2 -2
- data/vendor/faiss/faiss/utils/transpose/transpose-avx512-inl.h +1 -1
- data/vendor/faiss/faiss/utils/utils.cpp +9 -2
- data/vendor/faiss/faiss/utils/utils.h +2 -2
- metadata +29 -1
|
@@ -116,7 +116,7 @@ struct IDSelectorBitmap : IDSelector {
|
|
|
116
116
|
/** reverts the membership test of another selector */
|
|
117
117
|
struct IDSelectorNot : IDSelector {
|
|
118
118
|
const IDSelector* sel;
|
|
119
|
-
IDSelectorNot(const IDSelector* sel) : sel(sel) {}
|
|
119
|
+
explicit IDSelectorNot(const IDSelector* sel) : sel(sel) {}
|
|
120
120
|
bool is_member(idx_t id) const final {
|
|
121
121
|
return !sel->is_member(id);
|
|
122
122
|
}
|
|
@@ -131,7 +131,7 @@ struct IDSelectorAll : IDSelector {
|
|
|
131
131
|
virtual ~IDSelectorAll() {}
|
|
132
132
|
};
|
|
133
133
|
|
|
134
|
-
/// does an AND operation on the
|
|
134
|
+
/// does an AND operation on the two given IDSelector's is_membership
|
|
135
135
|
/// results.
|
|
136
136
|
struct IDSelectorAnd : IDSelector {
|
|
137
137
|
const IDSelector* lhs;
|
|
@@ -144,7 +144,7 @@ struct IDSelectorAnd : IDSelector {
|
|
|
144
144
|
virtual ~IDSelectorAnd() {}
|
|
145
145
|
};
|
|
146
146
|
|
|
147
|
-
/// does an OR operation on the
|
|
147
|
+
/// does an OR operation on the two given IDSelector's is_membership
|
|
148
148
|
/// results.
|
|
149
149
|
struct IDSelectorOr : IDSelector {
|
|
150
150
|
const IDSelector* lhs;
|
|
@@ -157,7 +157,7 @@ struct IDSelectorOr : IDSelector {
|
|
|
157
157
|
virtual ~IDSelectorOr() {}
|
|
158
158
|
};
|
|
159
159
|
|
|
160
|
-
/// does an XOR operation on the
|
|
160
|
+
/// does an XOR operation on the two given IDSelector's is_membership
|
|
161
161
|
/// results.
|
|
162
162
|
struct IDSelectorXOr : IDSelector {
|
|
163
163
|
const IDSelector* lhs;
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
#endif
|
|
31
31
|
|
|
32
32
|
extern "C" {
|
|
33
|
-
// LU
|
|
33
|
+
// LU decomposition of a general matrix
|
|
34
34
|
void sgetrf_(
|
|
35
35
|
FINTEGER* m,
|
|
36
36
|
FINTEGER* n,
|
|
@@ -65,7 +65,7 @@ int sgemm_(
|
|
|
65
65
|
float* c,
|
|
66
66
|
FINTEGER* ldc);
|
|
67
67
|
|
|
68
|
-
// LU
|
|
68
|
+
// LU decomposition of a general matrix
|
|
69
69
|
void dgetrf_(
|
|
70
70
|
FINTEGER* m,
|
|
71
71
|
FINTEGER* n,
|
|
@@ -189,7 +189,7 @@ void LocalSearchQuantizer::train(size_t n, const float* x) {
|
|
|
189
189
|
std::vector<int32_t> codes(n * M); // [n, M]
|
|
190
190
|
random_int32(codes, 0, K - 1, gen);
|
|
191
191
|
|
|
192
|
-
// compute standard
|
|
192
|
+
// compute standard deviations of each dimension
|
|
193
193
|
std::vector<float> stddev(d, 0);
|
|
194
194
|
|
|
195
195
|
#pragma omp parallel for
|
|
@@ -487,7 +487,7 @@ void LocalSearchQuantizer::update_codebooks(
|
|
|
487
487
|
* L = (X - \sum cj)^2, j = 1, ..., M
|
|
488
488
|
* L = X^2 - 2X * \sum cj + (\sum cj)^2
|
|
489
489
|
*
|
|
490
|
-
* X^2 is
|
|
490
|
+
* X^2 is negligible since it is the same for all possible value
|
|
491
491
|
* k of the m-th subcode.
|
|
492
492
|
*
|
|
493
493
|
* 2X * \sum cj is the unary term
|
|
@@ -138,7 +138,7 @@ struct LocalSearchQuantizer : AdditiveQuantizer {
|
|
|
138
138
|
/** Add some perturbation to codebooks
|
|
139
139
|
*
|
|
140
140
|
* @param T temperature of simulated annealing
|
|
141
|
-
* @param stddev standard
|
|
141
|
+
* @param stddev standard deviations of each dimension in training data
|
|
142
142
|
*/
|
|
143
143
|
void perturb_codebooks(
|
|
144
144
|
float T,
|
|
@@ -63,7 +63,7 @@ struct DummyScaler {
|
|
|
63
63
|
};
|
|
64
64
|
|
|
65
65
|
/// consumes 2x4 bits to encode a norm as a scalar additive quantizer
|
|
66
|
-
/// the norm is scaled because its range
|
|
66
|
+
/// the norm is scaled because its range is larger than other components
|
|
67
67
|
struct NormTableScaler {
|
|
68
68
|
static constexpr int nscale = 2;
|
|
69
69
|
int scale_int;
|
|
@@ -177,7 +177,7 @@ void NNDescent::join(DistanceComputer& qdis) {
|
|
|
177
177
|
}
|
|
178
178
|
}
|
|
179
179
|
|
|
180
|
-
/// Sample neighbors for each node to
|
|
180
|
+
/// Sample neighbors for each node to perform local join later
|
|
181
181
|
/// Store them in nn_new and nn_old
|
|
182
182
|
void NNDescent::update() {
|
|
183
183
|
// Step 1.
|
|
@@ -34,7 +34,7 @@ namespace faiss {
|
|
|
34
34
|
*
|
|
35
35
|
* Dong, Wei, Charikar Moses, and Kai Li, WWW 2011
|
|
36
36
|
*
|
|
37
|
-
* This
|
|
37
|
+
* This implementation is heavily influenced by the efanna
|
|
38
38
|
* implementation by Cong Fu and the KGraph library by Wei Dong
|
|
39
39
|
* (https://github.com/ZJULearning/efanna_graph)
|
|
40
40
|
* (https://github.com/aaalgo/kgraph)
|
|
@@ -117,7 +117,7 @@ struct NNDescent {
|
|
|
117
117
|
/// Perform local join on each node
|
|
118
118
|
void join(DistanceComputer& qdis);
|
|
119
119
|
|
|
120
|
-
/// Sample new neighbors for each node to
|
|
120
|
+
/// Sample new neighbors for each node to perform local join later
|
|
121
121
|
void update();
|
|
122
122
|
|
|
123
123
|
/// Sample a small number of points to evaluate the quality of KNNG built
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
3
|
+
*
|
|
4
|
+
* This source code is licensed under the MIT license found in the
|
|
5
|
+
* LICENSE file in the root directory of this source tree.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
#include <faiss/impl/Panorama.h>
|
|
9
|
+
|
|
10
|
+
#include <algorithm>
|
|
11
|
+
#include <cmath>
|
|
12
|
+
#include <cstring>
|
|
13
|
+
#include <vector>
|
|
14
|
+
|
|
15
|
+
namespace faiss {
|
|
16
|
+
|
|
17
|
+
/**************************************************************
|
|
18
|
+
* Panorama structure implementation
|
|
19
|
+
**************************************************************/
|
|
20
|
+
|
|
21
|
+
Panorama::Panorama(size_t code_size, size_t n_levels, size_t batch_size)
|
|
22
|
+
: code_size(code_size), n_levels(n_levels), batch_size(batch_size) {
|
|
23
|
+
set_derived_values();
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
void Panorama::set_derived_values() {
|
|
27
|
+
this->d = code_size / sizeof(float);
|
|
28
|
+
this->level_width_floats = ((d + n_levels - 1) / n_levels);
|
|
29
|
+
this->level_width = this->level_width_floats * sizeof(float);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* @brief Copy codes to level-oriented layout
|
|
34
|
+
* @param codes The base pointer to codes
|
|
35
|
+
* @param offset Where to start writing new data (in number of vectors)
|
|
36
|
+
* @param n_entry The number of new vectors to write
|
|
37
|
+
* @param code The new vector data
|
|
38
|
+
*/
|
|
39
|
+
void Panorama::copy_codes_to_level_layout(
|
|
40
|
+
uint8_t* codes,
|
|
41
|
+
size_t offset,
|
|
42
|
+
size_t n_entry,
|
|
43
|
+
const uint8_t* code) {
|
|
44
|
+
for (size_t entry_idx = 0; entry_idx < n_entry; entry_idx++) {
|
|
45
|
+
size_t current_pos = offset + entry_idx;
|
|
46
|
+
|
|
47
|
+
// Determine which batch we're in and position within that batch.
|
|
48
|
+
size_t batch_no = current_pos / batch_size;
|
|
49
|
+
size_t pos_in_batch = current_pos % batch_size;
|
|
50
|
+
|
|
51
|
+
// Copy entry into level-oriented layout for this batch.
|
|
52
|
+
size_t batch_offset = batch_no * batch_size * code_size;
|
|
53
|
+
for (size_t level = 0; level < n_levels; level++) {
|
|
54
|
+
size_t level_offset = level * level_width * batch_size;
|
|
55
|
+
size_t start_byte = level * level_width;
|
|
56
|
+
size_t actual_level_width =
|
|
57
|
+
std::min(level_width, code_size - level * level_width);
|
|
58
|
+
|
|
59
|
+
const uint8_t* src = code + entry_idx * code_size + start_byte;
|
|
60
|
+
uint8_t* dest = codes + batch_offset + level_offset +
|
|
61
|
+
pos_in_batch * actual_level_width;
|
|
62
|
+
|
|
63
|
+
memcpy(dest, src, actual_level_width);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
void Panorama::compute_cumulative_sums(
|
|
69
|
+
float* cumsum_base,
|
|
70
|
+
size_t offset,
|
|
71
|
+
size_t n_entry,
|
|
72
|
+
const float* vectors) {
|
|
73
|
+
std::vector<float> suffix_sums(d + 1);
|
|
74
|
+
|
|
75
|
+
for (size_t entry_idx = 0; entry_idx < n_entry; entry_idx++) {
|
|
76
|
+
size_t current_pos = offset + entry_idx;
|
|
77
|
+
size_t batch_no = current_pos / batch_size;
|
|
78
|
+
size_t pos_in_batch = current_pos % batch_size;
|
|
79
|
+
|
|
80
|
+
const float* vector = vectors + entry_idx * d;
|
|
81
|
+
|
|
82
|
+
// Compute suffix sums of squared values.
|
|
83
|
+
suffix_sums[d] = 0.0f;
|
|
84
|
+
for (int j = d - 1; j >= 0; j--) {
|
|
85
|
+
float squared_val = vector[j] * vector[j];
|
|
86
|
+
suffix_sums[j] = suffix_sums[j + 1] + squared_val;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// Store cumulative sums in batch-oriented layout.
|
|
90
|
+
size_t cumsum_batch_offset = batch_no * batch_size * (n_levels + 1);
|
|
91
|
+
|
|
92
|
+
for (size_t level = 0; level < n_levels; level++) {
|
|
93
|
+
size_t start_idx = level * level_width_floats;
|
|
94
|
+
size_t cumsum_offset =
|
|
95
|
+
cumsum_batch_offset + level * batch_size + pos_in_batch;
|
|
96
|
+
if (start_idx < d) {
|
|
97
|
+
cumsum_base[cumsum_offset] = std::sqrt(suffix_sums[start_idx]);
|
|
98
|
+
} else {
|
|
99
|
+
cumsum_base[cumsum_offset] = 0.0f;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Last level sum is always 0.
|
|
104
|
+
size_t cumsum_offset =
|
|
105
|
+
cumsum_batch_offset + n_levels * batch_size + pos_in_batch;
|
|
106
|
+
cumsum_base[cumsum_offset] = 0.0f;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
void Panorama::compute_query_cum_sums(const float* query, float* query_cum_sums)
|
|
111
|
+
const {
|
|
112
|
+
std::vector<float> suffix_sums(d + 1);
|
|
113
|
+
suffix_sums[d] = 0.0f;
|
|
114
|
+
|
|
115
|
+
for (int j = d - 1; j >= 0; j--) {
|
|
116
|
+
float squared_val = query[j] * query[j];
|
|
117
|
+
suffix_sums[j] = suffix_sums[j + 1] + squared_val;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
for (size_t level = 0; level < n_levels; level++) {
|
|
121
|
+
size_t start_idx = level * level_width_floats;
|
|
122
|
+
if (start_idx < d) {
|
|
123
|
+
query_cum_sums[level] = std::sqrt(suffix_sums[start_idx]);
|
|
124
|
+
} else {
|
|
125
|
+
query_cum_sums[level] = 0.0f;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
query_cum_sums[n_levels] = 0.0f;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
void Panorama::reconstruct(idx_t key, float* recons, const uint8_t* codes_base)
|
|
133
|
+
const {
|
|
134
|
+
uint8_t* recons_buffer = reinterpret_cast<uint8_t*>(recons);
|
|
135
|
+
|
|
136
|
+
size_t batch_no = key / batch_size;
|
|
137
|
+
size_t pos_in_batch = key % batch_size;
|
|
138
|
+
size_t batch_offset = batch_no * batch_size * code_size;
|
|
139
|
+
|
|
140
|
+
for (size_t level = 0; level < n_levels; level++) {
|
|
141
|
+
size_t level_offset = level * level_width * batch_size;
|
|
142
|
+
const uint8_t* src = codes_base + batch_offset + level_offset +
|
|
143
|
+
pos_in_batch * level_width;
|
|
144
|
+
uint8_t* dest = recons_buffer + level * level_width;
|
|
145
|
+
size_t copy_size =
|
|
146
|
+
std::min(level_width, code_size - level * level_width);
|
|
147
|
+
memcpy(dest, src, copy_size);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
void Panorama::copy_entry(
|
|
152
|
+
uint8_t* dest_codes,
|
|
153
|
+
uint8_t* src_codes,
|
|
154
|
+
float* dest_cum_sums,
|
|
155
|
+
float* src_cum_sums,
|
|
156
|
+
size_t dest_idx,
|
|
157
|
+
size_t src_idx) const {
|
|
158
|
+
// Calculate positions
|
|
159
|
+
size_t src_batch_no = src_idx / batch_size;
|
|
160
|
+
size_t src_pos_in_batch = src_idx % batch_size;
|
|
161
|
+
size_t dest_batch_no = dest_idx / batch_size;
|
|
162
|
+
size_t dest_pos_in_batch = dest_idx % batch_size;
|
|
163
|
+
|
|
164
|
+
// Calculate offsets
|
|
165
|
+
size_t src_batch_offset = src_batch_no * batch_size * code_size;
|
|
166
|
+
size_t dest_batch_offset = dest_batch_no * batch_size * code_size;
|
|
167
|
+
size_t src_cumsum_batch_offset = src_batch_no * batch_size * (n_levels + 1);
|
|
168
|
+
size_t dest_cumsum_batch_offset =
|
|
169
|
+
dest_batch_no * batch_size * (n_levels + 1);
|
|
170
|
+
|
|
171
|
+
for (size_t level = 0; level < n_levels; level++) {
|
|
172
|
+
// Copy code
|
|
173
|
+
size_t level_offset = level * level_width * batch_size;
|
|
174
|
+
size_t actual_level_width =
|
|
175
|
+
std::min(level_width, code_size - level * level_width);
|
|
176
|
+
|
|
177
|
+
const uint8_t* src = src_codes + src_batch_offset + level_offset +
|
|
178
|
+
src_pos_in_batch * actual_level_width;
|
|
179
|
+
uint8_t* dest = dest_codes + dest_batch_offset + level_offset +
|
|
180
|
+
dest_pos_in_batch * actual_level_width;
|
|
181
|
+
memcpy(dest, src, actual_level_width);
|
|
182
|
+
|
|
183
|
+
// Copy cum_sums
|
|
184
|
+
size_t cumsum_level_offset = level * batch_size;
|
|
185
|
+
|
|
186
|
+
const size_t src_offset = src_cumsum_batch_offset +
|
|
187
|
+
cumsum_level_offset + src_pos_in_batch;
|
|
188
|
+
size_t dest_offset = dest_cumsum_batch_offset + cumsum_level_offset +
|
|
189
|
+
dest_pos_in_batch;
|
|
190
|
+
dest_cum_sums[dest_offset] = src_cum_sums[src_offset];
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
} // namespace faiss
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
3
|
+
*
|
|
4
|
+
* This source code is licensed under the MIT license found in the
|
|
5
|
+
* LICENSE file in the root directory of this source tree.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
// -*- c++ -*-
|
|
9
|
+
|
|
10
|
+
#ifndef FAISS_PANORAMA_H
|
|
11
|
+
#define FAISS_PANORAMA_H
|
|
12
|
+
|
|
13
|
+
#include <faiss/impl/IDSelector.h>
|
|
14
|
+
#include <faiss/impl/PanoramaStats.h>
|
|
15
|
+
#include <faiss/utils/distances.h>
|
|
16
|
+
|
|
17
|
+
#include <algorithm>
|
|
18
|
+
#include <cstddef>
|
|
19
|
+
#include <cstdint>
|
|
20
|
+
#include <vector>
|
|
21
|
+
|
|
22
|
+
namespace faiss {
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Implements the core logic of Panorama-based refinement.
|
|
26
|
+
* arXiv: https://arxiv.org/abs/2510.00566
|
|
27
|
+
*
|
|
28
|
+
* Panorama partitions the dimensions of all vectors into L contiguous levels.
|
|
29
|
+
* During the refinement stage of ANNS, it computes distances between the query
|
|
30
|
+
* and its candidates level-by-level. After processing each level, it prunes the
|
|
31
|
+
* candidates whose lower bound exceeds the k-th best distance.
|
|
32
|
+
*
|
|
33
|
+
* In order to enable speedups, the dimensions (or codes) of each vector are
|
|
34
|
+
* stored in a batched, level-major manner. Within each batch of b vectors, the
|
|
35
|
+
* dimensions corresponding to level 1 will be stored first (for all elements in
|
|
36
|
+
* that batch), followed by level 2, and so on. This allows for efficient memory
|
|
37
|
+
* access patterns.
|
|
38
|
+
*
|
|
39
|
+
* Coupled with the appropriate orthogonal PreTransform (e.g. PCA, Cayley,
|
|
40
|
+
* etc.), Panorama can prune the vast majority of dimensions, greatly
|
|
41
|
+
* accelerating the refinement stage.
|
|
42
|
+
*/
|
|
43
|
+
struct Panorama {
|
|
44
|
+
size_t d = 0;
|
|
45
|
+
size_t code_size = 0;
|
|
46
|
+
size_t n_levels = 0;
|
|
47
|
+
size_t level_width = 0;
|
|
48
|
+
size_t level_width_floats = 0;
|
|
49
|
+
size_t batch_size = 0;
|
|
50
|
+
|
|
51
|
+
explicit Panorama(size_t code_size, size_t n_levels, size_t batch_size);
|
|
52
|
+
|
|
53
|
+
void set_derived_values();
|
|
54
|
+
|
|
55
|
+
/// Helper method to copy codes into level-oriented batch layout at a given
|
|
56
|
+
/// offset in the list.
|
|
57
|
+
void copy_codes_to_level_layout(
|
|
58
|
+
uint8_t* codes,
|
|
59
|
+
size_t offset,
|
|
60
|
+
size_t n_entry,
|
|
61
|
+
const uint8_t* code);
|
|
62
|
+
|
|
63
|
+
/// Helper method to compute the cumulative sums of the codes.
|
|
64
|
+
/// The cumsums also follow the level-oriented batch layout to minimize the
|
|
65
|
+
/// number of random memory accesses.
|
|
66
|
+
void compute_cumulative_sums(
|
|
67
|
+
float* cumsum_base,
|
|
68
|
+
size_t offset,
|
|
69
|
+
size_t n_entry,
|
|
70
|
+
const float* vectors);
|
|
71
|
+
|
|
72
|
+
/// Compute the cumulative sums of the query vector.
|
|
73
|
+
void compute_query_cum_sums(const float* query, float* query_cum_sums)
|
|
74
|
+
const;
|
|
75
|
+
|
|
76
|
+
/// Copy single entry (code and cum_sum) from one location to another.
|
|
77
|
+
void copy_entry(
|
|
78
|
+
uint8_t* dest_codes,
|
|
79
|
+
uint8_t* src_codes,
|
|
80
|
+
float* dest_cum_sums,
|
|
81
|
+
float* src_cum_sums,
|
|
82
|
+
size_t dest_idx,
|
|
83
|
+
size_t src_idx) const;
|
|
84
|
+
|
|
85
|
+
/// Panorama's core progressive filtering algorithm:
|
|
86
|
+
/// Process vectors in batches for cache efficiency. For each batch:
|
|
87
|
+
/// 1. Apply ID selection filter and initialize distances
|
|
88
|
+
/// (||y||^2 + ||x||^2).
|
|
89
|
+
/// 2. Maintain an "active set" of candidate indices that haven't been
|
|
90
|
+
/// pruned yet.
|
|
91
|
+
/// 3. For each level, refine distances incrementally and compact the active
|
|
92
|
+
/// set:
|
|
93
|
+
/// - Compute dot product for current level: exact_dist -= 2*<x,y>.
|
|
94
|
+
/// - Use Cauchy-Schwarz bound on remaining levels to get lower bound
|
|
95
|
+
/// - Prune candidates whose lower bound exceeds k-th best distance.
|
|
96
|
+
/// - Compact active_indices to remove pruned candidates (branchless)
|
|
97
|
+
/// 4. After all levels, survivors are exact distances; update heap.
|
|
98
|
+
/// This achieves early termination while maintaining SIMD-friendly
|
|
99
|
+
/// sequential access patterns in the level-oriented storage layout.
|
|
100
|
+
template <typename C>
|
|
101
|
+
size_t progressive_filter_batch(
|
|
102
|
+
const uint8_t* codes_base,
|
|
103
|
+
const float* cum_sums,
|
|
104
|
+
const float* query,
|
|
105
|
+
const float* query_cum_sums,
|
|
106
|
+
size_t batch_no,
|
|
107
|
+
size_t list_size,
|
|
108
|
+
const IDSelector* sel,
|
|
109
|
+
const idx_t* ids,
|
|
110
|
+
bool use_sel,
|
|
111
|
+
std::vector<uint32_t>& active_indices,
|
|
112
|
+
std::vector<float>& exact_distances,
|
|
113
|
+
float threshold,
|
|
114
|
+
PanoramaStats& local_stats) const;
|
|
115
|
+
|
|
116
|
+
void reconstruct(idx_t key, float* recons, const uint8_t* codes_base) const;
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
template <typename C>
|
|
120
|
+
size_t Panorama::progressive_filter_batch(
|
|
121
|
+
const uint8_t* codes_base,
|
|
122
|
+
const float* cum_sums,
|
|
123
|
+
const float* query,
|
|
124
|
+
const float* query_cum_sums,
|
|
125
|
+
size_t batch_no,
|
|
126
|
+
size_t list_size,
|
|
127
|
+
const IDSelector* sel,
|
|
128
|
+
const idx_t* ids,
|
|
129
|
+
bool use_sel,
|
|
130
|
+
std::vector<uint32_t>& active_indices,
|
|
131
|
+
std::vector<float>& exact_distances,
|
|
132
|
+
float threshold,
|
|
133
|
+
PanoramaStats& local_stats) const {
|
|
134
|
+
size_t batch_start = batch_no * batch_size;
|
|
135
|
+
size_t curr_batch_size = std::min(list_size - batch_start, batch_size);
|
|
136
|
+
|
|
137
|
+
size_t cumsum_batch_offset = batch_no * batch_size * (n_levels + 1);
|
|
138
|
+
const float* batch_cum_sums = cum_sums + cumsum_batch_offset;
|
|
139
|
+
const float* level_cum_sums = batch_cum_sums + batch_size;
|
|
140
|
+
float q_norm = query_cum_sums[0] * query_cum_sums[0];
|
|
141
|
+
|
|
142
|
+
size_t batch_offset = batch_no * batch_size * code_size;
|
|
143
|
+
const uint8_t* storage_base = codes_base + batch_offset;
|
|
144
|
+
|
|
145
|
+
// Initialize active set with ID-filtered vectors.
|
|
146
|
+
size_t num_active = 0;
|
|
147
|
+
for (size_t i = 0; i < curr_batch_size; i++) {
|
|
148
|
+
size_t global_idx = batch_start + i;
|
|
149
|
+
idx_t id = (ids == nullptr) ? global_idx : ids[global_idx];
|
|
150
|
+
bool include = !use_sel || sel->is_member(id);
|
|
151
|
+
|
|
152
|
+
active_indices[num_active] = i;
|
|
153
|
+
float cum_sum = batch_cum_sums[i];
|
|
154
|
+
exact_distances[i] = cum_sum * cum_sum + q_norm;
|
|
155
|
+
|
|
156
|
+
num_active += include;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
if (num_active == 0) {
|
|
160
|
+
return 0;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
size_t total_active = num_active;
|
|
164
|
+
for (size_t level = 0; level < n_levels; level++) {
|
|
165
|
+
local_stats.total_dims_scanned += num_active;
|
|
166
|
+
local_stats.total_dims += total_active;
|
|
167
|
+
|
|
168
|
+
float query_cum_norm = query_cum_sums[level + 1];
|
|
169
|
+
|
|
170
|
+
size_t level_offset = level * level_width * batch_size;
|
|
171
|
+
const float* level_storage =
|
|
172
|
+
(const float*)(storage_base + level_offset);
|
|
173
|
+
|
|
174
|
+
size_t next_active = 0;
|
|
175
|
+
for (size_t i = 0; i < num_active; i++) {
|
|
176
|
+
uint32_t idx = active_indices[i];
|
|
177
|
+
size_t actual_level_width = std::min(
|
|
178
|
+
level_width_floats, d - level * level_width_floats);
|
|
179
|
+
|
|
180
|
+
const float* yj = level_storage + idx * actual_level_width;
|
|
181
|
+
const float* query_level = query + level * level_width_floats;
|
|
182
|
+
|
|
183
|
+
float dot_product =
|
|
184
|
+
fvec_inner_product(query_level, yj, actual_level_width);
|
|
185
|
+
|
|
186
|
+
exact_distances[idx] -= 2.0f * dot_product;
|
|
187
|
+
|
|
188
|
+
float cum_sum = level_cum_sums[idx];
|
|
189
|
+
float cauchy_schwarz_bound = 2.0f * cum_sum * query_cum_norm;
|
|
190
|
+
float lower_bound = exact_distances[idx] - cauchy_schwarz_bound;
|
|
191
|
+
|
|
192
|
+
active_indices[next_active] = idx;
|
|
193
|
+
next_active += C::cmp(threshold, lower_bound) ? 1 : 0;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
num_active = next_active;
|
|
197
|
+
level_cum_sums += batch_size;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
return num_active;
|
|
201
|
+
}
|
|
202
|
+
} // namespace faiss
|
|
203
|
+
|
|
204
|
+
#endif
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
3
|
+
*
|
|
4
|
+
* This source code is licensed under the MIT license found in the
|
|
5
|
+
* LICENSE file in the root directory of this source tree.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
// -*- c++ -*-
|
|
9
|
+
|
|
10
|
+
#include <faiss/impl/PanoramaStats.h>
|
|
11
|
+
|
|
12
|
+
namespace faiss {
|
|
13
|
+
|
|
14
|
+
void PanoramaStats::reset() {
|
|
15
|
+
total_dims_scanned = 0;
|
|
16
|
+
total_dims = 0;
|
|
17
|
+
ratio_dims_scanned = 1.0f;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
void PanoramaStats::add(const PanoramaStats& other) {
|
|
21
|
+
total_dims_scanned += other.total_dims_scanned;
|
|
22
|
+
total_dims += other.total_dims;
|
|
23
|
+
if (total_dims > 0) {
|
|
24
|
+
ratio_dims_scanned =
|
|
25
|
+
static_cast<float>(total_dims_scanned) / total_dims;
|
|
26
|
+
} else {
|
|
27
|
+
ratio_dims_scanned = 1.0f;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
PanoramaStats indexPanorama_stats;
|
|
32
|
+
|
|
33
|
+
} // namespace faiss
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
3
|
+
*
|
|
4
|
+
* This source code is licensed under the MIT license found in the
|
|
5
|
+
* LICENSE file in the root directory of this source tree.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
// -*- c++ -*-
|
|
9
|
+
|
|
10
|
+
#ifndef FAISS_PANORAMA_STATS_H
|
|
11
|
+
#define FAISS_PANORAMA_STATS_H
|
|
12
|
+
|
|
13
|
+
#include <faiss/impl/platform_macros.h>
|
|
14
|
+
|
|
15
|
+
namespace faiss {
|
|
16
|
+
|
|
17
|
+
/// Statistics are not robust to internal threading nor to
|
|
18
|
+
/// concurrent Panorama searches. Use these values in a
|
|
19
|
+
/// single-threaded context to accurately gauge Panorama's
|
|
20
|
+
/// pruning effectiveness.
|
|
21
|
+
struct PanoramaStats {
|
|
22
|
+
uint64_t total_dims_scanned = 0; // total dimensions scanned
|
|
23
|
+
uint64_t total_dims = 0; // total dimensions
|
|
24
|
+
float ratio_dims_scanned = 1.0f; // fraction of dimensions actually scanned
|
|
25
|
+
|
|
26
|
+
PanoramaStats() {
|
|
27
|
+
reset();
|
|
28
|
+
}
|
|
29
|
+
void reset();
|
|
30
|
+
void add(const PanoramaStats& other);
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
// Single global var for all Panorama indexes
|
|
34
|
+
FAISS_API extern PanoramaStats indexPanorama_stats;
|
|
35
|
+
|
|
36
|
+
} // namespace faiss
|
|
37
|
+
|
|
38
|
+
#endif
|
|
@@ -178,7 +178,7 @@ struct ReproduceWithHammingObjective : PermutationObjective {
|
|
|
178
178
|
return x * x;
|
|
179
179
|
}
|
|
180
180
|
|
|
181
|
-
//
|
|
181
|
+
// weighting of distances: it is more important to reproduce small
|
|
182
182
|
// distances well
|
|
183
183
|
double dis_weight(double x) const {
|
|
184
184
|
return exp(-dis_weight_factor * x);
|
|
@@ -295,7 +295,7 @@ struct ReproduceWithHammingObjective : PermutationObjective {
|
|
|
295
295
|
|
|
296
296
|
} // anonymous namespace
|
|
297
297
|
|
|
298
|
-
//
|
|
298
|
+
// weighting of distances: it is more important to reproduce small
|
|
299
299
|
// distances well
|
|
300
300
|
double ReproduceDistancesObjective::dis_weight(double x) const {
|
|
301
301
|
return exp(-dis_weight_factor * x);
|
|
@@ -636,7 +636,7 @@ struct Score3Computer : PermutationObjective {
|
|
|
636
636
|
return accu;
|
|
637
637
|
}
|
|
638
638
|
|
|
639
|
-
/// PermutationObjective
|
|
639
|
+
/// PermutationObjective implementation (just negates the scores
|
|
640
640
|
/// for minimization)
|
|
641
641
|
|
|
642
642
|
double compute_cost(const int* perm) const override {
|
|
@@ -689,7 +689,7 @@ struct RankingScore2 : Score3Computer<float, double> {
|
|
|
689
689
|
/// count nb of i, j in a x b st. i < j
|
|
690
690
|
/// a and b should be sorted on input
|
|
691
691
|
/// they are the ranks of j and k respectively.
|
|
692
|
-
/// specific version for diff-of-rank weighting, cannot
|
|
692
|
+
/// specific version for diff-of-rank weighting, cannot optimize
|
|
693
693
|
/// with a cumulative table
|
|
694
694
|
double accum_gt_weight_diff(
|
|
695
695
|
const std::vector<int>& a,
|
|
@@ -985,7 +985,7 @@ size_t PolysemousTraining::memory_usage_per_thread(
|
|
|
985
985
|
return n * n * n * sizeof(float);
|
|
986
986
|
}
|
|
987
987
|
|
|
988
|
-
FAISS_THROW_MSG("Invalid
|
|
988
|
+
FAISS_THROW_MSG("Invalid optimization type");
|
|
989
989
|
return 0;
|
|
990
990
|
}
|
|
991
991
|
|
|
@@ -154,7 +154,7 @@ void ProductAdditiveQuantizer::compute_unpacked_codes(
|
|
|
154
154
|
int32_t* unpacked_codes,
|
|
155
155
|
size_t n,
|
|
156
156
|
const float* centroids) const {
|
|
157
|
-
/// TODO:
|
|
157
|
+
/// TODO: actually we do not need to unpack and pack
|
|
158
158
|
size_t offset_d = 0, offset_m = 0;
|
|
159
159
|
std::vector<float> xsub;
|
|
160
160
|
std::vector<uint8_t> codes;
|
|
@@ -166,7 +166,7 @@ struct ProductQuantizer : Quantizer {
|
|
|
166
166
|
/// Symmetric Distance Table
|
|
167
167
|
std::vector<float> sdc_table;
|
|
168
168
|
|
|
169
|
-
//
|
|
169
|
+
// initialize the SDC table from the centroids
|
|
170
170
|
void compute_sdc_table();
|
|
171
171
|
|
|
172
172
|
void search_sdc(
|