faiss 0.6.1 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/faiss/version.rb +1 -1
- data/vendor/faiss/faiss/Index.h +1 -1
- data/vendor/faiss/faiss/IndexBinaryHNSW.cpp +6 -7
- data/vendor/faiss/faiss/IndexBinaryIVF.cpp +3 -3
- data/vendor/faiss/faiss/IndexHNSW.cpp +173 -143
- data/vendor/faiss/faiss/IndexIVF.cpp +2 -2
- data/vendor/faiss/faiss/IndexIVFAdditiveQuantizer.cpp +2 -2
- data/vendor/faiss/faiss/IndexIVFFlat.cpp +3 -1
- data/vendor/faiss/faiss/IndexIVFFlatPanorama.cpp +3 -3
- data/vendor/faiss/faiss/IndexIVFPQ.cpp +2 -3
- data/vendor/faiss/faiss/IndexIVFPQR.cpp +2 -3
- data/vendor/faiss/faiss/IndexIVFRaBitQ.cpp +4 -13
- data/vendor/faiss/faiss/IndexNNDescent.cpp +1 -1
- data/vendor/faiss/faiss/IndexNSG.cpp +1 -2
- data/vendor/faiss/faiss/IndexScalarQuantizer.cpp +68 -6
- data/vendor/faiss/faiss/IndexScalarQuantizer.h +10 -0
- data/vendor/faiss/faiss/cppcontrib/SaDecodeKernels.h +1 -1
- data/vendor/faiss/faiss/cppcontrib/sa_decode/Level2-neon-inl.h +902 -12
- data/vendor/faiss/faiss/cppcontrib/sa_decode/PQ-neon-inl.h +702 -10
- data/vendor/faiss/faiss/factory_tools.cpp +4 -0
- data/vendor/faiss/faiss/gpu/GpuResources.h +3 -2
- data/vendor/faiss/faiss/gpu/StandardGpuResources.cpp +11 -12
- data/vendor/faiss/faiss/gpu/StandardGpuResources.h +3 -3
- data/vendor/faiss/faiss/gpu_metal/MetalDistance.h +87 -0
- data/vendor/faiss/faiss/gpu_metal/MetalIndex.h +7 -0
- data/vendor/faiss/faiss/gpu_metal/MetalIndexIVFFlat.h +181 -0
- data/vendor/faiss/faiss/gpu_metal/MetalKernels.h +48 -3
- data/vendor/faiss/faiss/gpu_metal/MetalPythonBridge.h +45 -0
- data/vendor/faiss/faiss/gpu_metal/impl/MetalIVFFlat.h +193 -0
- data/vendor/faiss/faiss/impl/HNSW.cpp +556 -199
- data/vendor/faiss/faiss/impl/HNSW.h +51 -13
- data/vendor/faiss/faiss/impl/NSG.cpp +15 -11
- data/vendor/faiss/faiss/impl/Panorama.h +11 -0
- data/vendor/faiss/faiss/impl/ProductQuantizer.cpp +25 -2
- data/vendor/faiss/faiss/impl/RaBitQUtils.cpp +1 -1
- data/vendor/faiss/faiss/impl/RaBitQuantizer.cpp +7 -1
- data/vendor/faiss/faiss/impl/ResultHandler.h +1 -0
- data/vendor/faiss/faiss/impl/ScalarQuantizer.cpp +271 -8
- data/vendor/faiss/faiss/impl/ScalarQuantizer.h +50 -0
- data/vendor/faiss/faiss/impl/VisitedTable.cpp +10 -10
- data/vendor/faiss/faiss/impl/VisitedTable.h +69 -34
- data/vendor/faiss/faiss/impl/fast_scan/dispatching.h +3 -1
- data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.cpp +35 -43
- data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.h +64 -15
- data/vendor/faiss/faiss/impl/hnsw/avx2.cpp +86 -40
- data/vendor/faiss/faiss/impl/hnsw/avx512.cpp +81 -50
- data/vendor/faiss/faiss/impl/index_read.cpp +100 -39
- data/vendor/faiss/faiss/impl/index_write.cpp +1 -0
- data/vendor/faiss/faiss/impl/io_macros.h +25 -0
- data/vendor/faiss/faiss/impl/platform_macros.h +12 -8
- data/vendor/faiss/faiss/impl/pq_code_distance/avx2.cpp +2 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/avx512.cpp +2 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/neon.cpp +2 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-generic.cpp +20 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-inl.h +36 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-sve.cpp +5 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_scan_impl.h +105 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/rvv.cpp +2 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/distance_computers.h +6 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/quantizers.h +327 -18
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx2.cpp +264 -27
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-impl.h +553 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-spr.cpp +559 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512.cpp +199 -27
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-dispatch.h +366 -3
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-neon.cpp +144 -19
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-rvv.cpp +26 -0
- data/vendor/faiss/faiss/impl/simd_dispatch.h +65 -8
- data/vendor/faiss/faiss/index_factory.cpp +5 -1
- data/vendor/faiss/faiss/index_io.h +16 -0
- data/vendor/faiss/faiss/invlists/DirectMap.cpp +4 -1
- data/vendor/faiss/faiss/invlists/InvertedLists.cpp +13 -13
- data/vendor/faiss/faiss/invlists/InvertedLists.h +2 -2
- data/vendor/faiss/faiss/svs/IndexSVSVamana.cpp +119 -22
- data/vendor/faiss/faiss/svs/IndexSVSVamana.h +15 -5
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.cpp +3 -2
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.h +2 -1
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.cpp +65 -24
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.h +3 -2
- data/vendor/faiss/faiss/utils/bf16.h +34 -0
- data/vendor/faiss/faiss/utils/distances_simd.cpp +0 -1
- data/vendor/faiss/faiss/utils/hamming.cpp +8 -8
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx2.cpp +2 -1
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx512_spr.cpp +15 -0
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512.h +6 -30
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512_spr.h +171 -0
- data/vendor/faiss/faiss/utils/partitioning.cpp +0 -2
- data/vendor/faiss/faiss/utils/simd_impl/partitioning_simdlib256.h +14 -68
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512_spr.cpp +343 -0
- data/vendor/faiss/faiss/utils/simd_levels.cpp +12 -2
- metadata +12 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3f9b24e9b85c954d799a4c19009626bcd90b5071a0b05136a182030920150b5a
|
|
4
|
+
data.tar.gz: 1fe65e3ee60d8f77471b8dc6dc77354d7d7f9e7a81a4c12be909ca3ac75fcea1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 277f548905897091a697479e256a79d2e2f9c56647be54921ea5fd6caec6381966dd859b58cdc0d09fb9da5670ef9a7248dd5ba487e50eef58f315e96d952986
|
|
7
|
+
data.tar.gz: 3a837ec44d583d6256f3490da8c6e17eec7e2a3854f4721826f6d03c4949e95163f6d67c70d7626d1b520a4b406fee9a3449b3d8fc7fbdd29fe5ff78dffd2e99
|
data/CHANGELOG.md
CHANGED
data/lib/faiss/version.rb
CHANGED
data/vendor/faiss/faiss/Index.h
CHANGED
|
@@ -36,7 +36,6 @@
|
|
|
36
36
|
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
37
37
|
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
38
38
|
#include <faiss/impl/binary_hamming/IndexBinaryHNSW_impl.h>
|
|
39
|
-
#include <faiss/utils/hamming_distance/hamming_computer-generic.h>
|
|
40
39
|
#undef THE_SIMD_LEVEL
|
|
41
40
|
|
|
42
41
|
namespace faiss {
|
|
@@ -129,7 +128,7 @@ void hnsw_add_vertices(
|
|
|
129
128
|
|
|
130
129
|
#pragma omp parallel
|
|
131
130
|
{
|
|
132
|
-
VisitedTable vt(ntotal);
|
|
131
|
+
std::unique_ptr<VisitedTable> vt = VisitedTable::create(ntotal);
|
|
133
132
|
|
|
134
133
|
std::unique_ptr<DistanceComputer> dis(
|
|
135
134
|
index_hnsw.get_distance_computer());
|
|
@@ -147,7 +146,7 @@ void hnsw_add_vertices(
|
|
|
147
146
|
pt_level,
|
|
148
147
|
pt_id,
|
|
149
148
|
locks,
|
|
150
|
-
vt,
|
|
149
|
+
*vt,
|
|
151
150
|
index_hnsw.keep_max_size_level0 && (pt_level == 0));
|
|
152
151
|
|
|
153
152
|
if (do_display && i - i0 > prev_display + 10000) {
|
|
@@ -237,7 +236,7 @@ void IndexBinaryHNSW::search(
|
|
|
237
236
|
|
|
238
237
|
#pragma omp parallel
|
|
239
238
|
{
|
|
240
|
-
VisitedTable vt(ntotal);
|
|
239
|
+
std::unique_ptr<VisitedTable> vt = VisitedTable::create(ntotal);
|
|
241
240
|
std::unique_ptr<DistanceComputer> dis(get_distance_computer());
|
|
242
241
|
RH::SingleResultHandler res(bres);
|
|
243
242
|
|
|
@@ -249,7 +248,7 @@ void IndexBinaryHNSW::search(
|
|
|
249
248
|
// as the index parameter. This state does not get used in the
|
|
250
249
|
// search function, as it is merely there to enable Panorama
|
|
251
250
|
// execution for IndexHNSWFlatPanorama.
|
|
252
|
-
HNSWStats stats = hnsw.search(*dis, nullptr, res, vt, params_in);
|
|
251
|
+
HNSWStats stats = hnsw.search(*dis, nullptr, res, *vt, params_in);
|
|
253
252
|
n1 += stats.n1;
|
|
254
253
|
n2 += stats.n2;
|
|
255
254
|
ndis += stats.ndis;
|
|
@@ -377,7 +376,7 @@ void IndexBinaryHNSWCagra::search(
|
|
|
377
376
|
|
|
378
377
|
#pragma omp parallel
|
|
379
378
|
{
|
|
380
|
-
VisitedTable vt(ntotal);
|
|
379
|
+
std::unique_ptr<VisitedTable> vt = VisitedTable::create(ntotal);
|
|
381
380
|
std::unique_ptr<DistanceComputer> dis(get_distance_computer());
|
|
382
381
|
HNSWStats search_stats;
|
|
383
382
|
RH::SingleResultHandler res(bres);
|
|
@@ -395,7 +394,7 @@ void IndexBinaryHNSWCagra::search(
|
|
|
395
394
|
&nearest_d[i],
|
|
396
395
|
1, // search_type
|
|
397
396
|
search_stats,
|
|
398
|
-
vt,
|
|
397
|
+
*vt,
|
|
399
398
|
params);
|
|
400
399
|
|
|
401
400
|
res.end();
|
|
@@ -22,7 +22,6 @@
|
|
|
22
22
|
#include <faiss/impl/AuxIndexStructures.h>
|
|
23
23
|
#include <faiss/impl/FaissAssert.h>
|
|
24
24
|
#include <faiss/utils/hamming.h>
|
|
25
|
-
#include <faiss/utils/sorting.h>
|
|
26
25
|
#include <faiss/utils/utils.h>
|
|
27
26
|
|
|
28
27
|
#include <faiss/impl/simd_dispatch.h>
|
|
@@ -32,7 +31,6 @@
|
|
|
32
31
|
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
33
32
|
// NOLINTNEXTLINE(facebook-hte-InlineHeader)
|
|
34
33
|
#include <faiss/impl/binary_hamming/IndexBinaryIVF_impl.h>
|
|
35
|
-
#include <faiss/utils/hamming_distance/hamming_computer-generic.h>
|
|
36
34
|
#undef THE_SIMD_LEVEL
|
|
37
35
|
|
|
38
36
|
namespace faiss {
|
|
@@ -257,7 +255,9 @@ void IndexBinaryIVF::reconstruct_from_offset(
|
|
|
257
255
|
idx_t list_no,
|
|
258
256
|
idx_t offset,
|
|
259
257
|
uint8_t* recons) const {
|
|
260
|
-
memcpy(recons,
|
|
258
|
+
memcpy(recons,
|
|
259
|
+
InvertedLists::ScopedCodes(invlists, list_no, offset).get(),
|
|
260
|
+
code_size);
|
|
261
261
|
}
|
|
262
262
|
|
|
263
263
|
void IndexBinaryIVF::reset() {
|
|
@@ -47,12 +47,17 @@ HNSWStats hnsw_stats;
|
|
|
47
47
|
|
|
48
48
|
namespace {
|
|
49
49
|
|
|
50
|
+
// Returns the storage's native distance computer. For similarity metrics
|
|
51
|
+
// (e.g. METRIC_INNER_PRODUCT), distance values are real similarity scores
|
|
52
|
+
// (larger = better); HNSW handles the ordering via `hnsw.is_similarity`.
|
|
53
|
+
//
|
|
54
|
+
// NOTE: callers that drive the legacy max-heap-only code paths (notably
|
|
55
|
+
// `search_from_candidates_2` in the IndexHNSW2Level mixed search) cannot
|
|
56
|
+
// consume similarity scores directly; they assume smaller-is-better.
|
|
57
|
+
// Those paths only fire for the (default) L2 IndexHNSW2Level + Index2Layer
|
|
58
|
+
// configuration today, so passing the native DC is safe in practice.
|
|
50
59
|
DistanceComputer* storage_distance_computer(const Index* storage) {
|
|
51
|
-
|
|
52
|
-
return new NegativeDistanceComputer(storage->get_distance_computer());
|
|
53
|
-
} else {
|
|
54
|
-
return storage->get_distance_computer();
|
|
55
|
-
}
|
|
60
|
+
return storage->get_distance_computer();
|
|
56
61
|
}
|
|
57
62
|
|
|
58
63
|
void hnsw_add_vertices(
|
|
@@ -145,7 +150,8 @@ void hnsw_add_vertices(
|
|
|
145
150
|
|
|
146
151
|
#pragma omp parallel if (i1 > i0 + 100)
|
|
147
152
|
{
|
|
148
|
-
VisitedTable vt
|
|
153
|
+
std::unique_ptr<VisitedTable> vt =
|
|
154
|
+
VisitedTable::create(ntotal, hnsw.use_visited_hashset);
|
|
149
155
|
|
|
150
156
|
std::unique_ptr<DistanceComputer> dis(
|
|
151
157
|
storage_distance_computer(index_hnsw.storage));
|
|
@@ -171,7 +177,7 @@ void hnsw_add_vertices(
|
|
|
171
177
|
pt_level,
|
|
172
178
|
pt_id,
|
|
173
179
|
locks,
|
|
174
|
-
vt,
|
|
180
|
+
*vt,
|
|
175
181
|
index_hnsw.keep_max_size_level0 && (pt_level == 0));
|
|
176
182
|
|
|
177
183
|
if (do_display && i - i0 > prev_display + 10000) {
|
|
@@ -213,13 +219,16 @@ void hnsw_add_vertices(
|
|
|
213
219
|
**************************************************************/
|
|
214
220
|
|
|
215
221
|
IndexHNSW::IndexHNSW(int d_in, int M, MetricType metric)
|
|
216
|
-
: Index(d_in, metric), hnsw(M) {
|
|
222
|
+
: Index(d_in, metric), hnsw(M) {
|
|
223
|
+
hnsw.is_similarity = is_similarity_metric(metric);
|
|
224
|
+
}
|
|
217
225
|
|
|
218
226
|
IndexHNSW::IndexHNSW(Index* storage_in, int M)
|
|
219
227
|
: Index(storage_in->d, storage_in->metric_type),
|
|
220
228
|
hnsw(M),
|
|
221
229
|
storage(storage_in) {
|
|
222
230
|
metric_arg = storage->metric_arg;
|
|
231
|
+
hnsw.is_similarity = is_similarity_metric(metric_type);
|
|
223
232
|
}
|
|
224
233
|
|
|
225
234
|
IndexHNSW::~IndexHNSW() {
|
|
@@ -276,7 +285,7 @@ void hnsw_search(
|
|
|
276
285
|
res;
|
|
277
286
|
std::unique_ptr<DistanceComputer> dis;
|
|
278
287
|
try {
|
|
279
|
-
vt =
|
|
288
|
+
vt = VisitedTable::create(
|
|
280
289
|
index->ntotal, hnsw.use_visited_hashset);
|
|
281
290
|
res = std::make_unique<
|
|
282
291
|
typename BlockResultHandler::SingleResultHandler>(bres);
|
|
@@ -325,16 +334,14 @@ void IndexHNSW::search(
|
|
|
325
334
|
const SearchParameters* params) const {
|
|
326
335
|
FAISS_THROW_IF_NOT(k > 0);
|
|
327
336
|
|
|
328
|
-
using RH = HeapBlockResultHandler<HNSW::C>;
|
|
329
|
-
RH bres(n, distances, labels, k);
|
|
330
|
-
|
|
331
|
-
hnsw_search(this, n, x, bres, params);
|
|
332
|
-
|
|
333
337
|
if (is_similarity_metric(this->metric_type)) {
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
+
using RH = HeapBlockResultHandler<HNSW::C_similarity>;
|
|
339
|
+
RH bres(n, distances, labels, k);
|
|
340
|
+
hnsw_search(this, n, x, bres, params);
|
|
341
|
+
} else {
|
|
342
|
+
using RH = HeapBlockResultHandler<HNSW::C_distance>;
|
|
343
|
+
RH bres(n, distances, labels, k);
|
|
344
|
+
hnsw_search(this, n, x, bres, params);
|
|
338
345
|
}
|
|
339
346
|
}
|
|
340
347
|
|
|
@@ -344,16 +351,14 @@ void IndexHNSW::range_search(
|
|
|
344
351
|
float radius,
|
|
345
352
|
RangeSearchResult* result,
|
|
346
353
|
const SearchParameters* params) const {
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
result->distances[i] = -result->distances[i];
|
|
356
|
-
}
|
|
354
|
+
if (is_similarity_metric(metric_type)) {
|
|
355
|
+
using RH = RangeSearchBlockResultHandler<HNSW::C_similarity>;
|
|
356
|
+
RH bres(result, radius);
|
|
357
|
+
hnsw_search(this, n, x, bres, params);
|
|
358
|
+
} else {
|
|
359
|
+
using RH = RangeSearchBlockResultHandler<HNSW::C_distance>;
|
|
360
|
+
RH bres(result, radius);
|
|
361
|
+
hnsw_search(this, n, x, bres, params);
|
|
357
362
|
}
|
|
358
363
|
}
|
|
359
364
|
|
|
@@ -361,8 +366,13 @@ void IndexHNSW::search1(
|
|
|
361
366
|
const float* x,
|
|
362
367
|
ResultHandler& handler,
|
|
363
368
|
SearchParameters* params) const {
|
|
364
|
-
|
|
365
|
-
|
|
369
|
+
if (is_similarity_metric(metric_type)) {
|
|
370
|
+
SingleQueryBlockResultHandler<HNSW::C_similarity, false> bres(handler);
|
|
371
|
+
hnsw_search(this, 1, x, bres, params);
|
|
372
|
+
} else {
|
|
373
|
+
SingleQueryBlockResultHandler<HNSW::C_distance, false> bres(handler);
|
|
374
|
+
hnsw_search(this, 1, x, bres, params);
|
|
375
|
+
}
|
|
366
376
|
}
|
|
367
377
|
|
|
368
378
|
void IndexHNSW::add(idx_t n, const float* x) {
|
|
@@ -459,63 +469,64 @@ void IndexHNSW::search_level_0(
|
|
|
459
469
|
|
|
460
470
|
size_t hnsw_ntotal = hnsw.levels.size();
|
|
461
471
|
|
|
462
|
-
|
|
463
|
-
|
|
472
|
+
auto run = [&]<class C>() {
|
|
473
|
+
using RH = HeapBlockResultHandler<C>;
|
|
474
|
+
RH bres(n, distances, labels, k);
|
|
464
475
|
|
|
465
|
-
|
|
466
|
-
|
|
476
|
+
std::exception_ptr ex;
|
|
477
|
+
std::atomic<bool> interrupt{false};
|
|
467
478
|
#pragma omp parallel
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
try {
|
|
474
|
-
qdis.reset(storage_distance_computer(storage));
|
|
475
|
-
vt = std::make_unique<VisitedTable>(
|
|
476
|
-
hnsw_ntotal, hnsw.use_visited_hashset);
|
|
477
|
-
res = std::make_unique<RH::SingleResultHandler>(bres);
|
|
478
|
-
} catch (...) {
|
|
479
|
-
omp_capture_exception(ex, [&] { interrupt = true; });
|
|
480
|
-
}
|
|
481
|
-
|
|
482
|
-
#pragma omp for
|
|
483
|
-
for (idx_t i = 0; i < n; i++) {
|
|
484
|
-
if (interrupt.load(std::memory_order_relaxed)) {
|
|
485
|
-
continue;
|
|
486
|
-
}
|
|
479
|
+
{
|
|
480
|
+
std::unique_ptr<DistanceComputer> qdis;
|
|
481
|
+
HNSWStats search_stats;
|
|
482
|
+
std::unique_ptr<VisitedTable> vt;
|
|
483
|
+
std::unique_ptr<typename RH::SingleResultHandler> res;
|
|
487
484
|
try {
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
*qdis.get(),
|
|
493
|
-
*res,
|
|
494
|
-
nprobe,
|
|
495
|
-
nearest + i * nprobe,
|
|
496
|
-
nearest_d + i * nprobe,
|
|
497
|
-
search_type,
|
|
498
|
-
search_stats,
|
|
499
|
-
*vt,
|
|
500
|
-
params);
|
|
501
|
-
res->end();
|
|
502
|
-
vt->advance();
|
|
485
|
+
qdis.reset(storage_distance_computer(storage));
|
|
486
|
+
vt = VisitedTable::create(
|
|
487
|
+
hnsw_ntotal, hnsw.use_visited_hashset);
|
|
488
|
+
res = std::make_unique<typename RH::SingleResultHandler>(bres);
|
|
503
489
|
} catch (...) {
|
|
504
490
|
omp_capture_exception(ex, [&] { interrupt = true; });
|
|
505
491
|
}
|
|
506
|
-
|
|
492
|
+
|
|
493
|
+
#pragma omp for
|
|
494
|
+
for (idx_t i = 0; i < n; i++) {
|
|
495
|
+
if (interrupt.load(std::memory_order_relaxed)) {
|
|
496
|
+
continue;
|
|
497
|
+
}
|
|
498
|
+
try {
|
|
499
|
+
res->begin(i);
|
|
500
|
+
qdis->set_query(x + i * d);
|
|
501
|
+
|
|
502
|
+
hnsw.search_level_0(
|
|
503
|
+
*qdis.get(),
|
|
504
|
+
*res,
|
|
505
|
+
nprobe,
|
|
506
|
+
nearest + i * nprobe,
|
|
507
|
+
nearest_d + i * nprobe,
|
|
508
|
+
search_type,
|
|
509
|
+
search_stats,
|
|
510
|
+
*vt,
|
|
511
|
+
params);
|
|
512
|
+
res->end();
|
|
513
|
+
vt->advance();
|
|
514
|
+
} catch (...) {
|
|
515
|
+
omp_capture_exception(ex, [&] { interrupt = true; });
|
|
516
|
+
}
|
|
517
|
+
}
|
|
507
518
|
#pragma omp critical
|
|
508
|
-
|
|
509
|
-
|
|
519
|
+
{
|
|
520
|
+
hnsw_stats.combine(search_stats);
|
|
521
|
+
}
|
|
510
522
|
}
|
|
511
|
-
|
|
512
|
-
|
|
523
|
+
omp_rethrow_if_exception(ex);
|
|
524
|
+
};
|
|
525
|
+
|
|
513
526
|
if (is_similarity_metric(this->metric_type)) {
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
distances[i] = -distances[i];
|
|
518
|
-
}
|
|
527
|
+
run.template operator()<HNSW::C_similarity>();
|
|
528
|
+
} else {
|
|
529
|
+
run.template operator()<HNSW::C_distance>();
|
|
519
530
|
}
|
|
520
531
|
}
|
|
521
532
|
|
|
@@ -569,7 +580,8 @@ void IndexHNSW::init_level_0_from_entry_points(
|
|
|
569
580
|
|
|
570
581
|
#pragma omp parallel
|
|
571
582
|
{
|
|
572
|
-
VisitedTable vt
|
|
583
|
+
std::unique_ptr<VisitedTable> vt =
|
|
584
|
+
VisitedTable::create(ntotal, hnsw.use_visited_hashset);
|
|
573
585
|
|
|
574
586
|
std::unique_ptr<DistanceComputer> dis(
|
|
575
587
|
storage_distance_computer(storage));
|
|
@@ -583,7 +595,7 @@ void IndexHNSW::init_level_0_from_entry_points(
|
|
|
583
595
|
dis->set_query(vec.data());
|
|
584
596
|
|
|
585
597
|
hnsw.add_links_starting_from(
|
|
586
|
-
*dis, pt_id, nearest, (*dis)(nearest), 0, locks, vt);
|
|
598
|
+
*dis, pt_id, nearest, (*dis)(nearest), 0, locks, *vt);
|
|
587
599
|
|
|
588
600
|
if (verbose && i % 10000 == 0) {
|
|
589
601
|
printf(" %d / %d\r", i, n);
|
|
@@ -824,7 +836,7 @@ int search_from_candidates_2(
|
|
|
824
836
|
idx_t* I,
|
|
825
837
|
float* D,
|
|
826
838
|
MinimaxHeap& candidates,
|
|
827
|
-
|
|
839
|
+
VisitedTableVector& vt,
|
|
828
840
|
HNSWStats& stats,
|
|
829
841
|
int level,
|
|
830
842
|
int nres_in = 0) {
|
|
@@ -934,8 +946,7 @@ void IndexHNSW2Level::search(
|
|
|
934
946
|
constexpr int candidates_size = 1;
|
|
935
947
|
std::unique_ptr<MinimaxHeap> candidates;
|
|
936
948
|
try {
|
|
937
|
-
vt =
|
|
938
|
-
ntotal, /*use_hashset=*/false);
|
|
949
|
+
vt = VisitedTable::create(ntotal, /*use_hashset=*/false);
|
|
939
950
|
dis.reset(storage_distance_computer(storage));
|
|
940
951
|
candidates = std::make_unique<MinimaxHeap>(candidates_size);
|
|
941
952
|
} catch (...) {
|
|
@@ -987,7 +998,7 @@ void IndexHNSW2Level::search(
|
|
|
987
998
|
idxi,
|
|
988
999
|
simi,
|
|
989
1000
|
*candidates,
|
|
990
|
-
*vt,
|
|
1001
|
+
static_cast<VisitedTableVector&>(*vt),
|
|
991
1002
|
search_stats,
|
|
992
1003
|
0,
|
|
993
1004
|
k);
|
|
@@ -1101,28 +1112,40 @@ void IndexHNSWCagra::search(
|
|
|
1101
1112
|
std::vector<storage_idx_t> nearest(n);
|
|
1102
1113
|
std::vector<float> nearest_d(n);
|
|
1103
1114
|
|
|
1115
|
+
auto pick_entrypoints = [&]<class C>() {
|
|
1104
1116
|
#pragma omp parallel for
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1117
|
+
for (idx_t i = 0; i < n; i++) {
|
|
1118
|
+
std::unique_ptr<DistanceComputer> dis(
|
|
1119
|
+
storage_distance_computer(this->storage));
|
|
1120
|
+
dis->set_query(x + i * d);
|
|
1121
|
+
nearest[i] = -1;
|
|
1122
|
+
// C::neutral() is the "worst possible" value: +inf for
|
|
1123
|
+
// CMax (distance) and -inf for CMin (similarity). The
|
|
1124
|
+
// first real candidate will always be strictly better.
|
|
1125
|
+
nearest_d[i] = C::neutral();
|
|
1126
|
+
|
|
1127
|
+
std::random_device rd;
|
|
1128
|
+
std::mt19937 gen(rd());
|
|
1129
|
+
std::uniform_int_distribution<idx_t> distrib(
|
|
1130
|
+
0, this->ntotal - 1);
|
|
1131
|
+
|
|
1132
|
+
for (idx_t j = 0; j < num_base_level_search_entrypoints; j++) {
|
|
1133
|
+
auto idx = distrib(gen);
|
|
1134
|
+
auto distance = (*dis)(idx);
|
|
1135
|
+
if (C::cmp(nearest_d[i], distance)) {
|
|
1136
|
+
nearest[i] = static_cast<storage_idx_t>(idx);
|
|
1137
|
+
nearest_d[i] = distance;
|
|
1138
|
+
}
|
|
1122
1139
|
}
|
|
1140
|
+
FAISS_THROW_IF_NOT_MSG(
|
|
1141
|
+
nearest[i] >= 0, "Could not find a valid entrypoint.");
|
|
1123
1142
|
}
|
|
1124
|
-
|
|
1125
|
-
|
|
1143
|
+
};
|
|
1144
|
+
|
|
1145
|
+
if (is_similarity_metric(metric_type)) {
|
|
1146
|
+
pick_entrypoints.template operator()<HNSW::C_similarity>();
|
|
1147
|
+
} else {
|
|
1148
|
+
pick_entrypoints.template operator()<HNSW::C_distance>();
|
|
1126
1149
|
}
|
|
1127
1150
|
|
|
1128
1151
|
search_level_0(
|
|
@@ -1150,56 +1173,63 @@ void IndexHNSWCagra::range_search(
|
|
|
1150
1173
|
return;
|
|
1151
1174
|
}
|
|
1152
1175
|
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1176
|
+
auto run = [&]<class C>() {
|
|
1177
|
+
const HNSW& hnsw = this->hnsw;
|
|
1178
|
+
size_t n1 = 0, n2 = 0, ndis = 0, nhops = 0;
|
|
1179
|
+
RangeSearchPartialResult pres(result);
|
|
1157
1180
|
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1181
|
+
for (idx_t i = 0; i < n; i++) {
|
|
1182
|
+
std::unique_ptr<DistanceComputer> dis(
|
|
1183
|
+
storage_distance_computer(storage));
|
|
1184
|
+
dis->set_query(x + i * d);
|
|
1162
1185
|
|
|
1163
|
-
|
|
1164
|
-
|
|
1186
|
+
storage_idx_t nearest = -1;
|
|
1187
|
+
// C::neutral() is the "worst possible" value under C: +inf for
|
|
1188
|
+
// CMax (distance) and -inf for CMin (similarity). The first
|
|
1189
|
+
// real candidate will always be strictly better.
|
|
1190
|
+
float nearest_d = C::neutral();
|
|
1165
1191
|
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1192
|
+
std::random_device rd;
|
|
1193
|
+
std::mt19937 gen(rd());
|
|
1194
|
+
std::uniform_int_distribution<idx_t> distrib(0, ntotal - 1);
|
|
1169
1195
|
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
nearest_d
|
|
1196
|
+
for (idx_t j = 0; j < num_base_level_search_entrypoints; j++) {
|
|
1197
|
+
auto idx = distrib(gen);
|
|
1198
|
+
auto distance = (*dis)(idx);
|
|
1199
|
+
// C::cmp(nearest_d, distance) is true iff distance is
|
|
1200
|
+
// strictly better than the current nearest_d.
|
|
1201
|
+
if (C::cmp(nearest_d, distance)) {
|
|
1202
|
+
nearest = idx;
|
|
1203
|
+
nearest_d = distance;
|
|
1204
|
+
}
|
|
1176
1205
|
}
|
|
1206
|
+
FAISS_THROW_IF_NOT_MSG(
|
|
1207
|
+
nearest >= 0, "Could not find a valid entrypoint.");
|
|
1208
|
+
|
|
1209
|
+
RangeQueryResult& qres = pres.new_result(i);
|
|
1210
|
+
RangeResultHandler<C> res(&qres, radius);
|
|
1211
|
+
std::unique_ptr<VisitedTable> vt =
|
|
1212
|
+
VisitedTable::create(ntotal, hnsw.use_visited_hashset);
|
|
1213
|
+
HNSWStats stats;
|
|
1214
|
+
hnsw.search_level_0(
|
|
1215
|
+
*dis, res, 1, &nearest, &nearest_d, 1, stats, *vt, params);
|
|
1216
|
+
n1 += stats.n1;
|
|
1217
|
+
n2 += stats.n2;
|
|
1218
|
+
ndis += stats.ndis;
|
|
1219
|
+
nhops += stats.nhops;
|
|
1177
1220
|
}
|
|
1178
|
-
FAISS_THROW_IF_NOT_MSG(
|
|
1179
|
-
nearest >= 0, "Could not find a valid entrypoint.");
|
|
1180
|
-
|
|
1181
|
-
RangeQueryResult& qres = pres.new_result(i);
|
|
1182
|
-
RangeResultHandler<HNSW::C> res(&qres, threshold);
|
|
1183
|
-
VisitedTable vt(ntotal, hnsw.use_visited_hashset);
|
|
1184
|
-
HNSWStats stats;
|
|
1185
|
-
hnsw.search_level_0(
|
|
1186
|
-
*dis, res, 1, &nearest, &nearest_d, 1, stats, vt, params);
|
|
1187
|
-
n1 += stats.n1;
|
|
1188
|
-
n2 += stats.n2;
|
|
1189
|
-
ndis += stats.ndis;
|
|
1190
|
-
nhops += stats.nhops;
|
|
1191
|
-
}
|
|
1192
1221
|
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1222
|
+
pres.set_lims();
|
|
1223
|
+
result->do_allocation();
|
|
1224
|
+
pres.copy_result();
|
|
1196
1225
|
|
|
1197
|
-
|
|
1226
|
+
hnsw_stats.combine({n1, n2, ndis, nhops});
|
|
1227
|
+
};
|
|
1198
1228
|
|
|
1199
1229
|
if (is_similarity_metric(metric_type)) {
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1230
|
+
run.template operator()<HNSW::C_similarity>();
|
|
1231
|
+
} else {
|
|
1232
|
+
run.template operator()<HNSW::C_distance>();
|
|
1203
1233
|
}
|
|
1204
1234
|
}
|
|
1205
1235
|
|
|
@@ -1223,7 +1223,7 @@ void IndexIVF::search_and_return_codes(
|
|
|
1223
1223
|
} else {
|
|
1224
1224
|
size_t list_no = lo_listno(key);
|
|
1225
1225
|
size_t offset = lo_offset(key);
|
|
1226
|
-
|
|
1226
|
+
InvertedLists::ScopedCodes cc(invlists, list_no, offset);
|
|
1227
1227
|
|
|
1228
1228
|
labels[ij] = invlists->get_single_id(list_no, offset);
|
|
1229
1229
|
|
|
@@ -1231,7 +1231,7 @@ void IndexIVF::search_and_return_codes(
|
|
|
1231
1231
|
encode_listno(list_no, code1);
|
|
1232
1232
|
code1 += code_size_1 - code_size;
|
|
1233
1233
|
}
|
|
1234
|
-
memcpy(code1, cc, code_size);
|
|
1234
|
+
memcpy(code1, cc.get(), code_size);
|
|
1235
1235
|
}
|
|
1236
1236
|
}
|
|
1237
1237
|
}
|
|
@@ -145,8 +145,8 @@ void IndexIVFAdditiveQuantizer::reconstruct_from_offset(
|
|
|
145
145
|
int64_t list_no,
|
|
146
146
|
int64_t offset,
|
|
147
147
|
float* recons) const {
|
|
148
|
-
|
|
149
|
-
aq->decode(
|
|
148
|
+
InvertedLists::ScopedCodes sc(invlists, list_no, offset);
|
|
149
|
+
aq->decode(sc.get(), recons, 1);
|
|
150
150
|
if (by_residual) {
|
|
151
151
|
std::vector<float> centroid(d);
|
|
152
152
|
quantizer->reconstruct(list_no, centroid.data());
|
|
@@ -164,7 +164,9 @@ void IndexIVFFlat::reconstruct_from_offset(
|
|
|
164
164
|
int64_t list_no,
|
|
165
165
|
int64_t offset,
|
|
166
166
|
float* recons) const {
|
|
167
|
-
memcpy(recons,
|
|
167
|
+
memcpy(recons,
|
|
168
|
+
InvertedLists::ScopedCodes(invlists, list_no, offset).get(),
|
|
169
|
+
code_size);
|
|
168
170
|
}
|
|
169
171
|
|
|
170
172
|
/*****************************************
|
|
@@ -194,9 +194,9 @@ void IndexIVFFlatPanorama::reconstruct_from_offset(
|
|
|
194
194
|
int64_t list_no,
|
|
195
195
|
int64_t offset,
|
|
196
196
|
float* recons) const {
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
197
|
+
memcpy(recons,
|
|
198
|
+
InvertedLists::ScopedCodes(invlists, list_no, offset).get(),
|
|
199
|
+
code_size);
|
|
200
200
|
}
|
|
201
201
|
|
|
202
202
|
} // namespace faiss
|
|
@@ -341,9 +341,8 @@ void IndexIVFPQ::reconstruct_from_offset(
|
|
|
341
341
|
int64_t list_no,
|
|
342
342
|
int64_t offset,
|
|
343
343
|
float* recons) const {
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
pq.decode(code, recons);
|
|
344
|
+
InvertedLists::ScopedCodes sc(invlists, list_no, offset);
|
|
345
|
+
pq.decode(sc.get(), recons);
|
|
347
346
|
if (by_residual) {
|
|
348
347
|
std::vector<float> centroid(d);
|
|
349
348
|
quantizer->reconstruct(list_no, centroid.data());
|
|
@@ -187,9 +187,8 @@ void IndexIVFPQR::search_preassigned(
|
|
|
187
187
|
quantizer->compute_residual(xq, residual_1.get(), list_no);
|
|
188
188
|
|
|
189
189
|
// 2nd level residual
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
pq.decode(l2code, residual_2);
|
|
190
|
+
InvertedLists::ScopedCodes l2sc(invlists, list_no, ofs);
|
|
191
|
+
pq.decode(l2sc.get(), residual_2);
|
|
193
192
|
for (int l = 0; l < d; l++) {
|
|
194
193
|
residual_2[l] = residual_1[l] - residual_2[l];
|
|
195
194
|
}
|