faiss 0.6.3 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/faiss/version.rb +1 -1
- data/vendor/faiss/faiss/Clustering.h +8 -0
- data/vendor/faiss/faiss/Index.h +1 -1
- data/vendor/faiss/faiss/IndexBinaryFromFloat.cpp +23 -2
- data/vendor/faiss/faiss/IndexBinaryHNSW.cpp +12 -140
- data/vendor/faiss/faiss/IndexBinaryHNSW.h +0 -6
- data/vendor/faiss/faiss/IndexBinaryIVF.cpp +6 -14
- data/vendor/faiss/faiss/IndexBinaryIVF.h +8 -1
- data/vendor/faiss/faiss/IndexFlat.cpp +8 -4
- data/vendor/faiss/faiss/IndexFlat.h +2 -2
- data/vendor/faiss/faiss/IndexHNSW.cpp +301 -91
- data/vendor/faiss/faiss/IndexHNSW.h +31 -7
- data/vendor/faiss/faiss/IndexIDMap.cpp +11 -6
- data/vendor/faiss/faiss/IndexIVF.cpp +19 -4
- data/vendor/faiss/faiss/IndexIVFPQ.cpp +9 -1
- data/vendor/faiss/faiss/IndexIVFPQ.h +11 -0
- data/vendor/faiss/faiss/IndexShards.cpp +3 -3
- data/vendor/faiss/faiss/IndexShardsIVF.cpp +4 -3
- data/vendor/faiss/faiss/SuperKMeans.cpp +31 -8
- data/vendor/faiss/faiss/SuperKMeans.h +3 -2
- data/vendor/faiss/faiss/VectorTransform.cpp +38 -0
- data/vendor/faiss/faiss/VectorTransform.h +3 -0
- data/vendor/faiss/faiss/clone_index.cpp +3 -0
- data/vendor/faiss/faiss/gpu/GpuClonerOptions.h +1 -5
- data/vendor/faiss/faiss/gpu/GpuDistance.h +2 -5
- data/vendor/faiss/faiss/gpu/GpuIndex.h +4 -5
- data/vendor/faiss/faiss/gpu/GpuIndexCagra.h +65 -42
- data/vendor/faiss/faiss/impl/AdditiveQuantizer.cpp +1 -1
- data/vendor/faiss/faiss/impl/ClusteringInitialization.cpp +5 -2
- data/vendor/faiss/faiss/impl/HNSW.cpp +419 -115
- data/vendor/faiss/faiss/impl/HNSW.h +46 -7
- data/vendor/faiss/faiss/impl/IDSelector.h +44 -0
- data/vendor/faiss/faiss/impl/NNDescent.cpp +10 -3
- data/vendor/faiss/faiss/impl/Panorama.h +11 -2
- data/vendor/faiss/faiss/impl/ProductQuantizer.cpp +4 -4
- data/vendor/faiss/faiss/impl/RaBitQUtils.cpp +1 -1
- data/vendor/faiss/faiss/impl/RaBitQuantizer.cpp +74 -8
- data/vendor/faiss/faiss/impl/RaBitQuantizer.h +47 -4
- data/vendor/faiss/faiss/impl/RaBitQuantizerMultiBit.cpp +2 -2
- data/vendor/faiss/faiss/impl/ResultHandler.h +9 -3
- data/vendor/faiss/faiss/impl/ScalarQuantizer.cpp +32 -30
- data/vendor/faiss/faiss/impl/ScalarQuantizer.h +6 -10
- data/vendor/faiss/faiss/impl/binary_hamming/IndexBinaryIVF_impl.h +89 -13
- data/vendor/faiss/faiss/impl/expanded_scanners.h +5 -1
- data/vendor/faiss/faiss/impl/fast_scan/decompose_qbs.h +1 -0
- data/vendor/faiss/faiss/impl/index_read.cpp +130 -29
- data/vendor/faiss/faiss/impl/index_write.cpp +31 -26
- data/vendor/faiss/faiss/impl/pq_code_distance/IVFPQScanner_impl.h +73 -39
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-generic.cpp +4 -4
- data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer.cpp +6 -1
- data/vendor/faiss/faiss/impl/scalar_quantizer/distance_computers.h +2 -2
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx2.cpp +6 -2
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-dispatch.h +37 -17
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-neon.cpp +109 -10
- data/vendor/faiss/faiss/impl/simd_dispatch.h +40 -22
- data/vendor/faiss/faiss/index_factory.cpp +8 -1
- data/vendor/faiss/faiss/invlists/InvertedLists.cpp +7 -4
- data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.cpp +10 -4
- data/vendor/faiss/faiss/svs/IndexSVSVamana.cpp +16 -4
- data/vendor/faiss/faiss/svs/IndexSVSVamana.h +8 -1
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.cpp +4 -2
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.h +2 -1
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.cpp +9 -2
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.h +2 -1
- data/vendor/faiss/faiss/utils/distances.cpp +20 -13
- data/vendor/faiss/faiss/utils/distances_dispatch.h +30 -24
- data/vendor/faiss/faiss/utils/distances_fused/distances_fused.cpp +1 -1
- data/vendor/faiss/faiss/utils/hamming.cpp +15 -9
- data/vendor/faiss/faiss/utils/hamming.h +10 -1
- data/vendor/faiss/faiss/utils/hamming_distance/common.h +14 -3
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx512_vpopcnt.cpp +24 -0
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512.h +1 -1
- data/vendor/faiss/faiss/utils/hamming_distance/{hamming_computer-avx512_spr.h → hamming_computer-avx512_vpopcnt.h} +85 -24
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_impl.h +141 -0
- data/vendor/faiss/faiss/utils/simd_impl/distances_arm_sve.cpp +194 -30
- data/vendor/faiss/faiss/utils/simd_impl/distances_rvv.cpp +93 -55
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512.cpp +57 -40
- data/vendor/faiss/faiss/utils/simd_impl/{rabitq_avx512_spr.cpp → rabitq_avx512_vpopcnt.cpp} +21 -24
- data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_dispatch.h +2 -7
- data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels.h +6 -1
- data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels_sve.cpp +34 -0
- data/vendor/faiss/faiss/utils/simd_levels.cpp +172 -67
- data/vendor/faiss/faiss/utils/simd_levels.h +26 -15
- metadata +6 -5
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx512_spr.cpp +0 -15
|
@@ -7,7 +7,8 @@
|
|
|
7
7
|
|
|
8
8
|
#include <faiss/IndexHNSW.h>
|
|
9
9
|
|
|
10
|
-
#include <
|
|
10
|
+
#include <faiss/IndexRaBitQ.h>
|
|
11
|
+
|
|
11
12
|
#include <atomic>
|
|
12
13
|
#include <cinttypes>
|
|
13
14
|
#include <cstdio>
|
|
@@ -17,7 +18,6 @@
|
|
|
17
18
|
#include <limits>
|
|
18
19
|
#include <memory>
|
|
19
20
|
#include <queue>
|
|
20
|
-
#include <random>
|
|
21
21
|
|
|
22
22
|
#include <cstdint>
|
|
23
23
|
#include "faiss/Index.h"
|
|
@@ -30,6 +30,7 @@
|
|
|
30
30
|
#include <faiss/impl/FaissException.h>
|
|
31
31
|
#include <faiss/impl/ResultHandler.h>
|
|
32
32
|
#include <faiss/impl/VisitedTable.h>
|
|
33
|
+
#include <faiss/impl/hnsw/LockVector.h>
|
|
33
34
|
#include <faiss/impl/hnsw/MinimaxHeap.h>
|
|
34
35
|
#include <faiss/utils/random.h>
|
|
35
36
|
#include <faiss/utils/sorting.h>
|
|
@@ -60,19 +61,29 @@ DistanceComputer* storage_distance_computer(const Index* storage) {
|
|
|
60
61
|
return storage->get_distance_computer();
|
|
61
62
|
}
|
|
62
63
|
|
|
63
|
-
|
|
64
|
-
|
|
64
|
+
} // namespace
|
|
65
|
+
|
|
66
|
+
// Deterministic HNSW graph build inspired by ParlayANN: points are bucketed by
|
|
67
|
+
// level and inserted in prefix-doubling batches. Phase A computes forward
|
|
68
|
+
// links against a frozen snapshot and records reverse edges; phase B merges
|
|
69
|
+
// those in a fixed order.
|
|
70
|
+
// https://arxiv.org/abs/2305.04359
|
|
71
|
+
void hnsw_add_vertices_deterministic(
|
|
72
|
+
HNSW& hnsw,
|
|
65
73
|
size_t n0,
|
|
66
74
|
size_t n,
|
|
67
|
-
|
|
75
|
+
int d,
|
|
76
|
+
bool init_level0,
|
|
77
|
+
bool keep_max_size_level0,
|
|
78
|
+
bool preset_levels,
|
|
68
79
|
bool verbose,
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
80
|
+
const std::function<DistanceComputer*()>& make_distance_computer,
|
|
81
|
+
const std::function<void(DistanceComputer&, HNSW::storage_idx_t)>&
|
|
82
|
+
set_query) {
|
|
72
83
|
size_t ntotal = n0 + n;
|
|
73
84
|
double t0 = getmillisecs();
|
|
74
85
|
if (verbose) {
|
|
75
|
-
printf("
|
|
86
|
+
printf("hnsw_add_vertices_deterministic: adding %zd elements on top of %zd "
|
|
76
87
|
"(preset_levels=%d)\n",
|
|
77
88
|
n,
|
|
78
89
|
n0,
|
|
@@ -83,16 +94,16 @@ void hnsw_add_vertices(
|
|
|
83
94
|
return;
|
|
84
95
|
}
|
|
85
96
|
|
|
86
|
-
|
|
97
|
+
// init_level0=false (CAGRA import) skips the level-0-only bucket; higher-
|
|
98
|
+
// level points still build their own level-0 links.
|
|
99
|
+
const int min_bucket_level = init_level0 ? 0 : 1;
|
|
87
100
|
|
|
101
|
+
int max_level = hnsw.prepare_level_tab(n, preset_levels);
|
|
88
102
|
if (verbose) {
|
|
89
103
|
printf(" max_level = %d\n", max_level);
|
|
90
104
|
}
|
|
91
105
|
|
|
92
|
-
|
|
93
|
-
locks.prepare(ntotal);
|
|
94
|
-
|
|
95
|
-
// add vectors from highest to lowest level
|
|
106
|
+
// Bucket the new points by level, highest first.
|
|
96
107
|
std::vector<int> hist;
|
|
97
108
|
std::vector<int> order(n);
|
|
98
109
|
|
|
@@ -122,98 +133,243 @@ void hnsw_add_vertices(
|
|
|
122
133
|
}
|
|
123
134
|
}
|
|
124
135
|
|
|
125
|
-
|
|
126
|
-
|
|
136
|
+
// Upper bound on batch size (ParlayANN's theta), 2% of the index.
|
|
137
|
+
const size_t theta =
|
|
138
|
+
std::max<size_t>(1, static_cast<size_t>(0.02 * ntotal));
|
|
127
139
|
|
|
128
|
-
|
|
129
|
-
|
|
140
|
+
// Polled inside phase A: a batch can be 2% of ntotal, so cancelling only
|
|
141
|
+
// at batch boundaries would take minutes on large indexes.
|
|
142
|
+
const idx_t check_period = InterruptCallback::get_period_hint(
|
|
143
|
+
max_level * d * hnsw.efConstruction);
|
|
130
144
|
|
|
131
|
-
|
|
145
|
+
RandomGenerator rng2(789);
|
|
146
|
+
size_t i1 = n;
|
|
132
147
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
148
|
+
for (int pt_level = static_cast<int>(hist.size()) - 1;
|
|
149
|
+
pt_level >= min_bucket_level;
|
|
150
|
+
pt_level--) {
|
|
151
|
+
size_t i0 = i1 - hist[pt_level];
|
|
152
|
+
if (i0 == i1) {
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
137
155
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
156
|
+
if (verbose) {
|
|
157
|
+
printf("Adding %zu elements at level %d\n", i1 - i0, pt_level);
|
|
158
|
+
}
|
|
141
159
|
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
160
|
+
// random permutation to get rid of dataset order bias
|
|
161
|
+
for (size_t j = i0; j < i1; j++) {
|
|
162
|
+
std::swap(
|
|
163
|
+
order[j],
|
|
164
|
+
order[j + rng2.rand_int(static_cast<int>(i1 - j))]);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Bootstrap only when the graph is empty. Raising entry_point to a
|
|
168
|
+
// point that is not yet linked orphans everything already inserted,
|
|
169
|
+
// which on an incremental add() is the entire prior graph. order[i0]
|
|
170
|
+
// is the sole member of the first batch below, so defer until then.
|
|
171
|
+
bool raise_entry_point = false;
|
|
172
|
+
if (hnsw.entry_point == -1) {
|
|
173
|
+
hnsw.max_level = pt_level;
|
|
174
|
+
hnsw.entry_point = order[i0];
|
|
175
|
+
} else if (pt_level > hnsw.max_level) {
|
|
176
|
+
raise_entry_point = true;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// Prefix-doubling batches within this bucket.
|
|
180
|
+
size_t s = i0;
|
|
181
|
+
while (s < i1) {
|
|
182
|
+
size_t done = s - i0;
|
|
183
|
+
size_t grow = std::min(done == 0 ? size_t(1) : done, theta);
|
|
184
|
+
size_t e = std::min(i1, s + grow);
|
|
185
|
+
|
|
186
|
+
// Phase A: compute forward links against the snapshot
|
|
187
|
+
// A reverse edge: `dest` gains an incoming link from `src`.
|
|
188
|
+
struct Edge {
|
|
189
|
+
int level;
|
|
190
|
+
HNSW::storage_idx_t dest;
|
|
191
|
+
HNSW::storage_idx_t src;
|
|
192
|
+
};
|
|
193
|
+
|
|
194
|
+
// One buffer, sized to an upper bound so it never reallocates.
|
|
195
|
+
// The fill order is nondeterministic but harmless: phase B
|
|
196
|
+
// regroups by destination.
|
|
197
|
+
size_t cap_sum = 0;
|
|
198
|
+
for (int64_t i = s; i < static_cast<int64_t>(e); i++) {
|
|
199
|
+
storage_idx_t pid = order[i];
|
|
200
|
+
cap_sum += hnsw.offsets[pid + 1] - hnsw.offsets[pid];
|
|
147
201
|
}
|
|
202
|
+
// Default-initialised to skip a memset per sub-batch; safe because
|
|
203
|
+
// edge_counter only hands out slots that are written before read.
|
|
204
|
+
std::unique_ptr<Edge[]> reverse_edges(new Edge[cap_sum]);
|
|
205
|
+
std::atomic<size_t> edge_counter{0};
|
|
148
206
|
|
|
207
|
+
// Thrown after the region (cannot throw out of one); the
|
|
208
|
+
// unsynchronized write costs at most one extra poll.
|
|
149
209
|
bool interrupt = false;
|
|
150
210
|
|
|
151
|
-
#pragma omp parallel if (
|
|
211
|
+
#pragma omp parallel if (e - s > 100)
|
|
152
212
|
{
|
|
153
213
|
std::unique_ptr<VisitedTable> vt =
|
|
154
214
|
VisitedTable::create(ntotal, hnsw.use_visited_hashset);
|
|
155
215
|
|
|
156
|
-
std::unique_ptr<DistanceComputer> dis(
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
size_t prev_display = 0;
|
|
216
|
+
std::unique_ptr<DistanceComputer> dis(make_distance_computer());
|
|
217
|
+
std::vector<std::pair<HNSW::storage_idx_t, int>>
|
|
218
|
+
pt_reverse_edges;
|
|
160
219
|
size_t counter = 0;
|
|
161
220
|
|
|
162
|
-
// here we should do schedule(dynamic) but this segfaults for
|
|
163
|
-
// some versions of LLVM. The performance impact should not be
|
|
164
|
-
// too large when (i1 - i0) / num_threads >> 1
|
|
165
221
|
#pragma omp for schedule(static)
|
|
166
|
-
for (int64_t i =
|
|
167
|
-
storage_idx_t pt_id = order[i];
|
|
168
|
-
dis->set_query(x + (pt_id - n0) * d);
|
|
169
|
-
|
|
170
|
-
// cannot break
|
|
222
|
+
for (int64_t i = s; i < static_cast<int64_t>(e); i++) {
|
|
171
223
|
if (interrupt) {
|
|
172
|
-
continue;
|
|
224
|
+
continue; // cannot break out of an OpenMP for loop
|
|
173
225
|
}
|
|
226
|
+
storage_idx_t pt_id = order[i];
|
|
227
|
+
int lvl = hnsw.levels[pt_id] - 1;
|
|
228
|
+
set_query(*dis, pt_id);
|
|
174
229
|
|
|
175
|
-
|
|
230
|
+
pt_reverse_edges.clear();
|
|
231
|
+
hnsw.compute_forward_links_deterministic(
|
|
176
232
|
*dis,
|
|
177
|
-
|
|
233
|
+
lvl,
|
|
178
234
|
pt_id,
|
|
179
|
-
locks,
|
|
180
235
|
*vt,
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
236
|
+
pt_reverse_edges,
|
|
237
|
+
keep_max_size_level0);
|
|
238
|
+
|
|
239
|
+
size_t off = edge_counter.fetch_add(
|
|
240
|
+
pt_reverse_edges.size(), std::memory_order_relaxed);
|
|
241
|
+
for (size_t k = 0; k < pt_reverse_edges.size(); k++) {
|
|
242
|
+
reverse_edges[off + k] = {
|
|
243
|
+
pt_reverse_edges[k].second,
|
|
244
|
+
pt_reverse_edges[k].first,
|
|
245
|
+
pt_id};
|
|
187
246
|
}
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
247
|
+
|
|
248
|
+
if (counter++ % check_period == 0 &&
|
|
249
|
+
InterruptCallback::is_interrupted()) {
|
|
250
|
+
interrupt = true;
|
|
192
251
|
}
|
|
193
|
-
counter++;
|
|
194
252
|
}
|
|
195
253
|
}
|
|
196
254
|
if (interrupt) {
|
|
197
255
|
FAISS_THROW_MSG("computation interrupted");
|
|
198
256
|
}
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
257
|
+
|
|
258
|
+
// Phase B: merge the reverse edges
|
|
259
|
+
// Group by destination so each node is merged by exactly one
|
|
260
|
+
// thread: lock-free and thread-count-independent.
|
|
261
|
+
const size_t total = edge_counter.load();
|
|
262
|
+
|
|
263
|
+
constexpr int kBucketBits = 8;
|
|
264
|
+
constexpr uint32_t kNumBuckets = 1u << kBucketBits;
|
|
265
|
+
constexpr uint32_t kBucketMask = kNumBuckets - 1;
|
|
266
|
+
|
|
267
|
+
// bstart[b]..bstart[b+1] delimits bucket b after partitioning.
|
|
268
|
+
std::vector<size_t> bstart(kNumBuckets + 1, 0);
|
|
269
|
+
for (size_t idx = 0; idx < total; idx++) {
|
|
270
|
+
bstart[(static_cast<uint32_t>(reverse_edges[idx].dest) &
|
|
271
|
+
kBucketMask) +
|
|
272
|
+
1]++;
|
|
273
|
+
}
|
|
274
|
+
for (uint32_t b = 0; b < kNumBuckets; b++) {
|
|
275
|
+
bstart[b + 1] += bstart[b];
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
// In-place partition (cycle sort): each step lands at least one
|
|
279
|
+
// edge in its bucket, so O(total) with no auxiliary buffer.
|
|
280
|
+
{
|
|
281
|
+
std::vector<size_t> head(bstart.begin(), bstart.end() - 1);
|
|
282
|
+
for (uint32_t b = 0; b < kNumBuckets; b++) {
|
|
283
|
+
size_t end_b = bstart[b + 1];
|
|
284
|
+
while (head[b] < end_b) {
|
|
285
|
+
Edge cur_e = reverse_edges[head[b]];
|
|
286
|
+
if ((static_cast<uint32_t>(cur_e.dest) & kBucketMask) ==
|
|
287
|
+
b) {
|
|
288
|
+
head[b]++;
|
|
289
|
+
continue;
|
|
290
|
+
}
|
|
291
|
+
while ((static_cast<uint32_t>(cur_e.dest) &
|
|
292
|
+
kBucketMask) != b) {
|
|
293
|
+
uint32_t tb = static_cast<uint32_t>(cur_e.dest) &
|
|
294
|
+
kBucketMask;
|
|
295
|
+
std::swap(cur_e, reverse_edges[head[tb]]);
|
|
296
|
+
head[tb]++;
|
|
297
|
+
}
|
|
298
|
+
reverse_edges[head[b]] = cur_e;
|
|
299
|
+
head[b]++;
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
Edge* base = reverse_edges.get();
|
|
304
|
+
|
|
305
|
+
#pragma omp parallel if (total > 100)
|
|
306
|
+
{
|
|
307
|
+
std::unique_ptr<DistanceComputer> dis(make_distance_computer());
|
|
308
|
+
// Not schedule(dynamic): the libomp dynamic dispatcher
|
|
309
|
+
// segfaults in some build configs.
|
|
310
|
+
#pragma omp for schedule(static)
|
|
311
|
+
for (int64_t b = 0; b < static_cast<int64_t>(kNumBuckets);
|
|
312
|
+
b++) {
|
|
313
|
+
Edge* p = base + bstart[b];
|
|
314
|
+
size_t m = bstart[b + 1] - bstart[b];
|
|
315
|
+
if (m == 0) {
|
|
316
|
+
continue;
|
|
317
|
+
}
|
|
318
|
+
// src order does not matter; the merge re-sorts each set.
|
|
319
|
+
std::sort(p, p + m, [](const Edge& a, const Edge& c) {
|
|
320
|
+
if (a.level != c.level) {
|
|
321
|
+
return a.level < c.level;
|
|
322
|
+
}
|
|
323
|
+
return a.dest < c.dest;
|
|
324
|
+
});
|
|
325
|
+
std::vector<HNSW::storage_idx_t> incoming;
|
|
326
|
+
size_t g = 0;
|
|
327
|
+
while (g < m) {
|
|
328
|
+
size_t h = g + 1;
|
|
329
|
+
while (h < m && p[h].level == p[g].level &&
|
|
330
|
+
p[h].dest == p[g].dest) {
|
|
331
|
+
h++;
|
|
332
|
+
}
|
|
333
|
+
incoming.clear();
|
|
334
|
+
incoming.reserve(h - g);
|
|
335
|
+
for (size_t kk = g; kk < h; kk++) {
|
|
336
|
+
incoming.push_back(p[kk].src);
|
|
337
|
+
}
|
|
338
|
+
hnsw.merge_reverse_links_deterministic(
|
|
339
|
+
*dis,
|
|
340
|
+
p[g].dest,
|
|
341
|
+
p[g].level,
|
|
342
|
+
incoming,
|
|
343
|
+
keep_max_size_level0 && (p[g].level == 0));
|
|
344
|
+
g = h;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
InterruptCallback::check();
|
|
350
|
+
s = e;
|
|
351
|
+
|
|
352
|
+
if (raise_entry_point) {
|
|
353
|
+
hnsw.max_level = pt_level;
|
|
354
|
+
hnsw.entry_point = order[i0];
|
|
355
|
+
raise_entry_point = false;
|
|
356
|
+
}
|
|
205
357
|
}
|
|
358
|
+
|
|
359
|
+
i1 = i0;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
if (init_level0) {
|
|
363
|
+
FAISS_ASSERT(i1 == 0);
|
|
364
|
+
} else {
|
|
365
|
+
FAISS_ASSERT((i1 - hist[0]) == 0);
|
|
206
366
|
}
|
|
367
|
+
|
|
207
368
|
if (verbose) {
|
|
208
369
|
printf("Done in %.3f ms\n", getmillisecs() - t0);
|
|
209
370
|
}
|
|
210
|
-
if (!index_hnsw.retain_locks) {
|
|
211
|
-
locks.clear();
|
|
212
|
-
}
|
|
213
371
|
}
|
|
214
372
|
|
|
215
|
-
} // namespace
|
|
216
|
-
|
|
217
373
|
/**************************************************************
|
|
218
374
|
* IndexHNSW implementation
|
|
219
375
|
**************************************************************/
|
|
@@ -269,6 +425,7 @@ void hnsw_search(
|
|
|
269
425
|
}
|
|
270
426
|
}
|
|
271
427
|
size_t n1 = 0, n2 = 0, ndis = 0, nhops = 0;
|
|
428
|
+
size_t n_rabitq_1bit = 0, n_rabitq_refine = 0;
|
|
272
429
|
|
|
273
430
|
idx_t check_period = InterruptCallback::get_period_hint(
|
|
274
431
|
hnsw.max_level * index->d * efSearch);
|
|
@@ -294,7 +451,9 @@ void hnsw_search(
|
|
|
294
451
|
omp_capture_exception(ex, [&] { interrupt = true; });
|
|
295
452
|
}
|
|
296
453
|
|
|
297
|
-
#pragma omp for reduction(
|
|
454
|
+
#pragma omp for reduction( \
|
|
455
|
+
+ : n1, n2, ndis, nhops, n_rabitq_1bit, n_rabitq_refine) \
|
|
456
|
+
schedule(guided)
|
|
298
457
|
for (idx_t i = i0; i < i1; i++) {
|
|
299
458
|
if (interrupt.load(std::memory_order_relaxed)) {
|
|
300
459
|
continue;
|
|
@@ -302,6 +461,10 @@ void hnsw_search(
|
|
|
302
461
|
try {
|
|
303
462
|
res->begin(i);
|
|
304
463
|
dis->set_query(x + i * index->d);
|
|
464
|
+
auto* rq = dynamic_cast<RaBitQDistanceComputer*>(dis.get());
|
|
465
|
+
if (rq) {
|
|
466
|
+
rq->stats.reset();
|
|
467
|
+
}
|
|
305
468
|
|
|
306
469
|
HNSWStats stats =
|
|
307
470
|
hnsw.search(*dis, index, *res, *vt, params);
|
|
@@ -309,6 +472,10 @@ void hnsw_search(
|
|
|
309
472
|
n2 += stats.n2;
|
|
310
473
|
ndis += stats.ndis;
|
|
311
474
|
nhops += stats.nhops;
|
|
475
|
+
if (rq) {
|
|
476
|
+
n_rabitq_1bit += rq->stats.n_1bit;
|
|
477
|
+
n_rabitq_refine += rq->stats.n_refine;
|
|
478
|
+
}
|
|
312
479
|
res->end();
|
|
313
480
|
vt->advance();
|
|
314
481
|
} catch (...) {
|
|
@@ -321,6 +488,7 @@ void hnsw_search(
|
|
|
321
488
|
}
|
|
322
489
|
|
|
323
490
|
hnsw_stats.combine({n1, n2, ndis, nhops});
|
|
491
|
+
rabitq_stats.add({n_rabitq_1bit, n_rabitq_refine});
|
|
324
492
|
}
|
|
325
493
|
|
|
326
494
|
} // anonymous namespace
|
|
@@ -384,18 +552,25 @@ void IndexHNSW::add(idx_t n, const float* x) {
|
|
|
384
552
|
storage->add(n, x);
|
|
385
553
|
ntotal = storage->ntotal;
|
|
386
554
|
|
|
387
|
-
|
|
388
|
-
|
|
555
|
+
bool preset_levels = hnsw.levels.size() == static_cast<size_t>(ntotal);
|
|
556
|
+
|
|
557
|
+
hnsw_add_vertices_deterministic(
|
|
558
|
+
hnsw,
|
|
389
559
|
n0,
|
|
390
560
|
n,
|
|
391
|
-
|
|
561
|
+
d,
|
|
562
|
+
init_level0,
|
|
563
|
+
keep_max_size_level0,
|
|
564
|
+
preset_levels,
|
|
392
565
|
verbose,
|
|
393
|
-
|
|
566
|
+
[this] { return storage_distance_computer(storage); },
|
|
567
|
+
[this, x, n0](DistanceComputer& dc, HNSW::storage_idx_t pt_id) {
|
|
568
|
+
dc.set_query(x + (pt_id - n0) * d);
|
|
569
|
+
});
|
|
394
570
|
}
|
|
395
571
|
|
|
396
572
|
void IndexHNSW::reset() {
|
|
397
573
|
hnsw.reset();
|
|
398
|
-
locks.clear();
|
|
399
574
|
storage->reset();
|
|
400
575
|
ntotal = 0;
|
|
401
576
|
}
|
|
@@ -479,6 +654,7 @@ void IndexHNSW::search_level_0(
|
|
|
479
654
|
{
|
|
480
655
|
std::unique_ptr<DistanceComputer> qdis;
|
|
481
656
|
HNSWStats search_stats;
|
|
657
|
+
RaBitQStats rq_search_stats;
|
|
482
658
|
VisitedTable* vt = nullptr;
|
|
483
659
|
std::unique_ptr<typename RH::SingleResultHandler> res;
|
|
484
660
|
try {
|
|
@@ -498,6 +674,11 @@ void IndexHNSW::search_level_0(
|
|
|
498
674
|
try {
|
|
499
675
|
res->begin(i);
|
|
500
676
|
qdis->set_query(x + i * d);
|
|
677
|
+
auto* rq =
|
|
678
|
+
dynamic_cast<RaBitQDistanceComputer*>(qdis.get());
|
|
679
|
+
if (rq) {
|
|
680
|
+
rq->stats.reset();
|
|
681
|
+
}
|
|
501
682
|
|
|
502
683
|
hnsw.search_level_0(
|
|
503
684
|
*qdis.get(),
|
|
@@ -509,6 +690,9 @@ void IndexHNSW::search_level_0(
|
|
|
509
690
|
search_stats,
|
|
510
691
|
*vt,
|
|
511
692
|
params);
|
|
693
|
+
if (rq) {
|
|
694
|
+
rq_search_stats.add(rq->stats);
|
|
695
|
+
}
|
|
512
696
|
res->end();
|
|
513
697
|
vt->advance();
|
|
514
698
|
} catch (...) {
|
|
@@ -518,6 +702,7 @@ void IndexHNSW::search_level_0(
|
|
|
518
702
|
#pragma omp critical
|
|
519
703
|
{
|
|
520
704
|
hnsw_stats.combine(search_stats);
|
|
705
|
+
rabitq_stats.add(rq_search_stats);
|
|
521
706
|
}
|
|
522
707
|
}
|
|
523
708
|
omp_rethrow_if_exception(ex);
|
|
@@ -576,7 +761,7 @@ void IndexHNSW::init_level_0_from_entry_points(
|
|
|
576
761
|
int n,
|
|
577
762
|
const storage_idx_t* points,
|
|
578
763
|
const storage_idx_t* nearests) {
|
|
579
|
-
locks
|
|
764
|
+
LockVector locks(ntotal);
|
|
580
765
|
|
|
581
766
|
#pragma omp parallel
|
|
582
767
|
{
|
|
@@ -606,10 +791,6 @@ void IndexHNSW::init_level_0_from_entry_points(
|
|
|
606
791
|
if (verbose) {
|
|
607
792
|
printf("\n");
|
|
608
793
|
}
|
|
609
|
-
|
|
610
|
-
if (!retain_locks) {
|
|
611
|
-
locks.clear();
|
|
612
|
-
}
|
|
613
794
|
}
|
|
614
795
|
|
|
615
796
|
void IndexHNSW::reorder_links() {
|
|
@@ -741,7 +922,7 @@ IndexHNSWFlatPanorama::IndexHNSWFlatPanorama(
|
|
|
741
922
|
// Enable Panorama search mode.
|
|
742
923
|
// This is not ideal, but is still more simple than making a subclass of
|
|
743
924
|
// HNSW and overriding the search logic.
|
|
744
|
-
hnsw.
|
|
925
|
+
hnsw.search_method = HNSW::SM_PANORAMA;
|
|
745
926
|
}
|
|
746
927
|
|
|
747
928
|
void IndexHNSWFlatPanorama::add(idx_t n, const float* x) {
|
|
@@ -808,6 +989,38 @@ IndexHNSWSQ::IndexHNSWSQ(
|
|
|
808
989
|
|
|
809
990
|
IndexHNSWSQ::IndexHNSWSQ() = default;
|
|
810
991
|
|
|
992
|
+
/**************************************************************
|
|
993
|
+
* IndexHNSWRaBitQ implementation
|
|
994
|
+
**************************************************************/
|
|
995
|
+
|
|
996
|
+
IndexHNSWRaBitQ::IndexHNSWRaBitQ() = default;
|
|
997
|
+
|
|
998
|
+
namespace {
|
|
999
|
+
|
|
1000
|
+
IndexRaBitQ* make_hnsw_rabitq_storage(
|
|
1001
|
+
int d,
|
|
1002
|
+
uint8_t nb_bits,
|
|
1003
|
+
MetricType metric) {
|
|
1004
|
+
FAISS_THROW_IF_NOT_MSG(
|
|
1005
|
+
metric == METRIC_L2, "IndexHNSWRaBitQ supports only the L2 metric");
|
|
1006
|
+
return new IndexRaBitQ(d, metric, nb_bits);
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
} // namespace
|
|
1010
|
+
|
|
1011
|
+
IndexHNSWRaBitQ::IndexHNSWRaBitQ(
|
|
1012
|
+
int d,
|
|
1013
|
+
int M,
|
|
1014
|
+
uint8_t nb_bits,
|
|
1015
|
+
MetricType metric)
|
|
1016
|
+
: IndexHNSW(make_hnsw_rabitq_storage(d, nb_bits, metric), M) {
|
|
1017
|
+
own_fields = true;
|
|
1018
|
+
is_trained = storage->is_trained;
|
|
1019
|
+
// 1-bit codes store plain SignBitFactors with no f_error, so there is no
|
|
1020
|
+
// bound to prune with and the staged path does not apply.
|
|
1021
|
+
hnsw.search_method = nb_bits >= 2 ? HNSW::SM_RABITQ : HNSW::SM_DEFAULT;
|
|
1022
|
+
}
|
|
1023
|
+
|
|
811
1024
|
/**************************************************************
|
|
812
1025
|
* IndexHNSW2Level implementation
|
|
813
1026
|
**************************************************************/
|
|
@@ -1123,13 +1336,11 @@ void IndexHNSWCagra::search(
|
|
|
1123
1336
|
// first real candidate will always be strictly better.
|
|
1124
1337
|
nearest_d[i] = C::neutral();
|
|
1125
1338
|
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
std::uniform_int_distribution<idx_t> distrib(
|
|
1129
|
-
0, this->ntotal - 1);
|
|
1339
|
+
// Seeded per query so entrypoints are reproducible.
|
|
1340
|
+
SplitMix64RandomGenerator gen(i);
|
|
1130
1341
|
|
|
1131
1342
|
for (idx_t j = 0; j < num_base_level_search_entrypoints; j++) {
|
|
1132
|
-
|
|
1343
|
+
idx_t idx = gen.rand_int64() % this->ntotal;
|
|
1133
1344
|
auto distance = (*dis)(idx);
|
|
1134
1345
|
if (C::cmp(nearest_d[i], distance)) {
|
|
1135
1346
|
nearest[i] = static_cast<storage_idx_t>(idx);
|
|
@@ -1188,12 +1399,11 @@ void IndexHNSWCagra::range_search(
|
|
|
1188
1399
|
// real candidate will always be strictly better.
|
|
1189
1400
|
float nearest_d = C::neutral();
|
|
1190
1401
|
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
std::uniform_int_distribution<idx_t> distrib(0, ntotal - 1);
|
|
1402
|
+
// For reproducible entrypoint.
|
|
1403
|
+
SplitMix64RandomGenerator gen(i);
|
|
1194
1404
|
|
|
1195
1405
|
for (idx_t j = 0; j < num_base_level_search_entrypoints; j++) {
|
|
1196
|
-
|
|
1406
|
+
idx_t idx = gen.rand_int64() % ntotal;
|
|
1197
1407
|
auto distance = (*dis)(idx);
|
|
1198
1408
|
// C::cmp(nearest_d, distance) is true iff distance is
|
|
1199
1409
|
// strictly better than the current nearest_d.
|
|
@@ -18,7 +18,6 @@
|
|
|
18
18
|
#include <faiss/IndexScalarQuantizer.h>
|
|
19
19
|
#include <faiss/impl/HNSW.h>
|
|
20
20
|
#include <faiss/impl/Panorama.h>
|
|
21
|
-
#include <faiss/impl/hnsw/LockVector.h>
|
|
22
21
|
#include <faiss/utils/utils.h>
|
|
23
22
|
|
|
24
23
|
namespace faiss {
|
|
@@ -53,11 +52,6 @@ struct IndexHNSW : Index {
|
|
|
53
52
|
// See impl/VisitedTable.h.
|
|
54
53
|
std::optional<bool> use_visited_hashset;
|
|
55
54
|
|
|
56
|
-
// Per-node locks for HNSW graph construction.
|
|
57
|
-
LockVector locks;
|
|
58
|
-
// locks are freed after each call to add() unless this flag is set.
|
|
59
|
-
bool retain_locks = false;
|
|
60
|
-
|
|
61
55
|
explicit IndexHNSW(int d = 0, int M = 32, MetricType metric = METRIC_L2);
|
|
62
56
|
explicit IndexHNSW(Index* storage, int M = 32);
|
|
63
57
|
|
|
@@ -216,6 +210,36 @@ struct IndexHNSWSQ : IndexHNSW {
|
|
|
216
210
|
MetricType metric = METRIC_L2);
|
|
217
211
|
};
|
|
218
212
|
|
|
213
|
+
/** HNSW index whose storage is RaBitQ-compressed.
|
|
214
|
+
*
|
|
215
|
+
* The graph is built directly from the compressed codes. Neighbor-diversity
|
|
216
|
+
* pruning uses a fast 1-bit symmetric L2 estimator, including when the storage
|
|
217
|
+
* contains multi-bit codes.
|
|
218
|
+
*
|
|
219
|
+
* With nb_bits >= 2 the codes carry a per-vector error factor, so search uses
|
|
220
|
+
* the staged search method: a 1-bit estimate for every neighbor and the full
|
|
221
|
+
* multi-bit distance only for candidates the error bound cannot rule out.
|
|
222
|
+
* nb_bits = 1 has no error factor and uses ordinary HNSW search.
|
|
223
|
+
*/
|
|
224
|
+
struct IndexHNSWRaBitQ : IndexHNSW {
|
|
225
|
+
IndexHNSWRaBitQ();
|
|
226
|
+
IndexHNSWRaBitQ(
|
|
227
|
+
int d,
|
|
228
|
+
int M,
|
|
229
|
+
uint8_t nb_bits = 1,
|
|
230
|
+
MetricType metric = METRIC_L2);
|
|
231
|
+
|
|
232
|
+
IndexHNSWRaBitQ& operator=(const IndexHNSWRaBitQ&) = delete;
|
|
233
|
+
|
|
234
|
+
private:
|
|
235
|
+
// clone_index() replaces the shallow-copied storage with a deep copy
|
|
236
|
+
// before returning it. Keep ordinary C++ copies from sharing ownership.
|
|
237
|
+
IndexHNSWRaBitQ(const IndexHNSWRaBitQ&) = default;
|
|
238
|
+
#ifndef SWIG
|
|
239
|
+
friend IndexHNSW* clone_IndexHNSW(const IndexHNSW* index);
|
|
240
|
+
#endif
|
|
241
|
+
};
|
|
242
|
+
|
|
219
243
|
/** 2-level code structure with fast random access
|
|
220
244
|
*/
|
|
221
245
|
struct IndexHNSW2Level : IndexHNSW {
|
|
@@ -253,7 +277,7 @@ struct IndexHNSWCagra : IndexHNSW {
|
|
|
253
277
|
/// searches only the base level knn graph of the HNSW index.
|
|
254
278
|
/// This parameter selects the entry point by randomly selecting
|
|
255
279
|
/// some points and using the best one.
|
|
256
|
-
int num_base_level_search_entrypoints =
|
|
280
|
+
int num_base_level_search_entrypoints = 256;
|
|
257
281
|
|
|
258
282
|
void add(idx_t n, const float* x) override;
|
|
259
283
|
|
|
@@ -229,15 +229,20 @@ void IndexIDMapTemplate<IndexT>::range_search(
|
|
|
229
229
|
typename IndexT::distance_t radius,
|
|
230
230
|
RangeSearchResult* result,
|
|
231
231
|
const SearchParameters* params) const {
|
|
232
|
+
IDSelectorTranslated this_idtrans(this->id_map, nullptr);
|
|
233
|
+
ScopedSelChange sel_change;
|
|
234
|
+
|
|
232
235
|
if (params && params->sel) {
|
|
233
|
-
|
|
234
|
-
IDSelectorTranslated id_selector_translated(id_map, params->sel);
|
|
235
|
-
internal_search_parameters.sel = &id_selector_translated;
|
|
236
|
+
auto idtrans = dynamic_cast<const IDSelectorTranslated*>(params->sel);
|
|
236
237
|
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
238
|
+
if (!idtrans) {
|
|
239
|
+
// Preserve the concrete params type and its index-specific fields.
|
|
240
|
+
auto params_non_const = const_cast<SearchParameters*>(params);
|
|
241
|
+
this_idtrans.sel = params->sel;
|
|
242
|
+
sel_change.set(params_non_const, &this_idtrans);
|
|
243
|
+
}
|
|
240
244
|
}
|
|
245
|
+
index->range_search(n, x, radius, result, params);
|
|
241
246
|
|
|
242
247
|
const idx_t id_map_size = static_cast<idx_t>(id_map.size());
|
|
243
248
|
#pragma omp parallel for
|