faiss 0.6.1 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/lib/faiss/version.rb +1 -1
- data/vendor/faiss/faiss/AutoTune.cpp +3 -1
- data/vendor/faiss/faiss/Clustering.cpp +9 -1
- data/vendor/faiss/faiss/IVFlib.cpp +14 -3
- data/vendor/faiss/faiss/Index.h +2 -2
- data/vendor/faiss/faiss/IndexAdditiveQuantizer.cpp +9 -10
- data/vendor/faiss/faiss/IndexAdditiveQuantizerFastScan.cpp +2 -3
- data/vendor/faiss/faiss/IndexBinaryFromFloat.cpp +1 -2
- data/vendor/faiss/faiss/IndexBinaryHNSW.cpp +10 -12
- data/vendor/faiss/faiss/IndexBinaryHash.cpp +5 -9
- data/vendor/faiss/faiss/IndexBinaryIVF.cpp +5 -7
- data/vendor/faiss/faiss/IndexEDEN.cpp +273 -0
- data/vendor/faiss/faiss/IndexEDEN.h +57 -0
- data/vendor/faiss/faiss/IndexFastScan.cpp +15 -4
- data/vendor/faiss/faiss/IndexFlat.cpp +13 -50
- data/vendor/faiss/faiss/IndexHNSW.cpp +177 -148
- data/vendor/faiss/faiss/IndexIDMap.cpp +16 -3
- data/vendor/faiss/faiss/IndexIDMap.h +2 -0
- data/vendor/faiss/faiss/IndexIVF.cpp +19 -8
- data/vendor/faiss/faiss/IndexIVFAdditiveQuantizer.cpp +3 -3
- data/vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.cpp +3 -4
- data/vendor/faiss/faiss/IndexIVFEDEN.cpp +302 -0
- data/vendor/faiss/faiss/IndexIVFEDEN.h +70 -0
- data/vendor/faiss/faiss/IndexIVFFastScan.cpp +5 -6
- data/vendor/faiss/faiss/IndexIVFFlat.cpp +6 -5
- data/vendor/faiss/faiss/IndexIVFFlatPanorama.cpp +3 -3
- data/vendor/faiss/faiss/IndexIVFIndependentQuantizer.cpp +1 -1
- data/vendor/faiss/faiss/IndexIVFPQ.cpp +42 -25
- data/vendor/faiss/faiss/IndexIVFPQFastScan.cpp +0 -1
- data/vendor/faiss/faiss/IndexIVFPQR.cpp +2 -3
- data/vendor/faiss/faiss/IndexIVFRaBitQ.cpp +23 -62
- data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.cpp +180 -76
- data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.h +5 -4
- data/vendor/faiss/faiss/IndexIVFSpectralHash.cpp +8 -6
- data/vendor/faiss/faiss/IndexLSH.cpp +2 -3
- data/vendor/faiss/faiss/IndexLattice.cpp +5 -0
- data/vendor/faiss/faiss/IndexNNDescent.cpp +10 -3
- data/vendor/faiss/faiss/IndexNSG.cpp +8 -4
- data/vendor/faiss/faiss/IndexPQ.cpp +6 -8
- data/vendor/faiss/faiss/IndexPreTransform.cpp +15 -0
- data/vendor/faiss/faiss/IndexRaBitQ.cpp +2 -2
- data/vendor/faiss/faiss/IndexRaBitQFastScan.cpp +1 -2
- data/vendor/faiss/faiss/IndexRaBitQFastScan.h +5 -1
- data/vendor/faiss/faiss/IndexRefine.cpp +30 -1
- data/vendor/faiss/faiss/IndexReplicas.cpp +1 -2
- data/vendor/faiss/faiss/IndexScalarQuantizer.cpp +68 -6
- data/vendor/faiss/faiss/IndexScalarQuantizer.h +10 -0
- data/vendor/faiss/faiss/IndexShards.cpp +2 -2
- data/vendor/faiss/faiss/IndexShardsIVF.cpp +2 -2
- data/vendor/faiss/faiss/MetaIndexes.cpp +2 -4
- data/vendor/faiss/faiss/SuperKMeans.cpp +256 -240
- data/vendor/faiss/faiss/SuperKMeans.h +30 -0
- data/vendor/faiss/faiss/VectorTransform.cpp +33 -2
- data/vendor/faiss/faiss/clone_index.cpp +5 -0
- data/vendor/faiss/faiss/cppcontrib/SaDecodeKernels.h +1 -1
- data/vendor/faiss/faiss/cppcontrib/sa_decode/Level2-neon-inl.h +902 -12
- data/vendor/faiss/faiss/cppcontrib/sa_decode/PQ-neon-inl.h +702 -10
- data/vendor/faiss/faiss/factory_tools.cpp +51 -4
- data/vendor/faiss/faiss/gpu/GpuCloner.cpp +11 -11
- data/vendor/faiss/faiss/gpu/GpuIndex.h +34 -11
- data/vendor/faiss/faiss/gpu/GpuIndexCagra.h +47 -0
- data/vendor/faiss/faiss/gpu/GpuIndexIVF.h +17 -0
- data/vendor/faiss/faiss/gpu/GpuIndexIVFScalarQuantizer.h +16 -0
- data/vendor/faiss/faiss/gpu/GpuResources.h +3 -2
- data/vendor/faiss/faiss/gpu/StandardGpuResources.cpp +11 -12
- data/vendor/faiss/faiss/gpu/StandardGpuResources.h +3 -3
- data/vendor/faiss/faiss/gpu/perf/PerfClustering.cpp +1 -1
- data/vendor/faiss/faiss/gpu/perf/PerfIVFPQAdd.cpp +2 -2
- data/vendor/faiss/faiss/gpu/test/TestGpuIndexIVFScalarQuantizer.cpp +180 -0
- data/vendor/faiss/faiss/gpu_metal/MetalDistance.h +87 -0
- data/vendor/faiss/faiss/gpu_metal/MetalIndex.h +7 -0
- data/vendor/faiss/faiss/gpu_metal/MetalIndexIVFFlat.h +177 -0
- data/vendor/faiss/faiss/gpu_metal/MetalIndexIVFPQ.h +88 -0
- data/vendor/faiss/faiss/gpu_metal/MetalKernels.h +48 -3
- data/vendor/faiss/faiss/gpu_metal/MetalPythonBridge.h +45 -0
- data/vendor/faiss/faiss/gpu_metal/impl/MetalIVFFlat.h +193 -0
- data/vendor/faiss/faiss/gpu_metal/impl/MetalIVFPQ.h +134 -0
- data/vendor/faiss/faiss/impl/ClusteringInitialization.cpp +2 -2
- data/vendor/faiss/faiss/impl/DistanceComputer.h +34 -0
- data/vendor/faiss/faiss/impl/EDENQuantizer.h +119 -0
- data/vendor/faiss/faiss/impl/HNSW.cpp +658 -344
- data/vendor/faiss/faiss/impl/HNSW.h +51 -13
- data/vendor/faiss/faiss/impl/LocalSearchQuantizer.cpp +2 -2
- data/vendor/faiss/faiss/impl/NSG.cpp +18 -12
- data/vendor/faiss/faiss/impl/Panorama.h +20 -7
- data/vendor/faiss/faiss/impl/PolysemousTraining.cpp +152 -84
- data/vendor/faiss/faiss/impl/ProductQuantizer.cpp +59 -24
- data/vendor/faiss/faiss/impl/RaBitQUtils.cpp +45 -37
- data/vendor/faiss/faiss/impl/RaBitQUtils.h +35 -0
- data/vendor/faiss/faiss/impl/RaBitQuantizer.cpp +175 -68
- data/vendor/faiss/faiss/impl/RaBitQuantizer.h +19 -0
- data/vendor/faiss/faiss/impl/RaBitQuantizerMultiBit.cpp +2 -11
- data/vendor/faiss/faiss/impl/ResultHandler.h +26 -31
- data/vendor/faiss/faiss/impl/ScalarQuantizer.cpp +522 -58
- data/vendor/faiss/faiss/impl/ScalarQuantizer.h +70 -0
- data/vendor/faiss/faiss/impl/ThreadedIndex-inl.h +2 -2
- data/vendor/faiss/faiss/impl/VisitedTable.cpp +33 -13
- data/vendor/faiss/faiss/impl/VisitedTable.h +88 -33
- data/vendor/faiss/faiss/impl/binary_hamming/IndexBinaryIVF_impl.h +1 -1
- data/vendor/faiss/faiss/impl/binary_hamming/avx2.cpp +4 -4
- data/vendor/faiss/faiss/impl/fast_scan/dispatching.h +38 -3
- data/vendor/faiss/faiss/impl/hnsw/LockVector.cpp +1 -1
- data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.cpp +35 -43
- data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.h +64 -15
- data/vendor/faiss/faiss/impl/hnsw/avx2.cpp +86 -40
- data/vendor/faiss/faiss/impl/hnsw/avx512.cpp +81 -50
- data/vendor/faiss/faiss/impl/index_read.cpp +476 -75
- data/vendor/faiss/faiss/impl/index_write.cpp +56 -4
- data/vendor/faiss/faiss/impl/io_macros.h +25 -0
- data/vendor/faiss/faiss/impl/lattice_Zn.cpp +8 -9
- data/vendor/faiss/faiss/impl/platform_macros.h +15 -9
- data/vendor/faiss/faiss/impl/polysemous_training/avx512.cpp +284 -0
- data/vendor/faiss/faiss/impl/polysemous_training/dispatch.h +115 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/IVFPQ_QueryTables.cpp +0 -1
- data/vendor/faiss/faiss/impl/pq_code_distance/PQDistanceComputer_impl.h +26 -15
- data/vendor/faiss/faiss/impl/pq_code_distance/avx2.cpp +6 -4
- data/vendor/faiss/faiss/impl/pq_code_distance/avx512.cpp +2 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/neon.cpp +2 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-generic.cpp +20 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-inl.h +36 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-sve.cpp +5 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/pq_scan_impl.h +105 -0
- data/vendor/faiss/faiss/impl/pq_code_distance/rvv.cpp +2 -0
- data/vendor/faiss/faiss/impl/result_handler/ResultHandler.cpp +195 -0
- data/vendor/faiss/faiss/impl/result_handler/avx2.cpp +133 -0
- data/vendor/faiss/faiss/impl/result_handler/avx512.cpp +281 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx2.cpp +72 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx512.cpp +228 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer.cpp +882 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/distance_computers.h +6 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/quantizers.h +336 -26
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx2.cpp +331 -32
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-impl.h +553 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-spr.cpp +558 -0
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512.cpp +284 -45
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-dispatch.h +502 -3
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-neon.cpp +157 -32
- data/vendor/faiss/faiss/impl/scalar_quantizer/sq-rvv.cpp +26 -0
- data/vendor/faiss/faiss/impl/simd_dispatch.h +86 -8
- data/vendor/faiss/faiss/index_factory.cpp +37 -7
- data/vendor/faiss/faiss/index_io.h +16 -0
- data/vendor/faiss/faiss/invlists/DirectMap.cpp +5 -2
- data/vendor/faiss/faiss/invlists/InvertedLists.cpp +15 -15
- data/vendor/faiss/faiss/invlists/InvertedLists.h +2 -2
- data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.cpp +19 -4
- data/vendor/faiss/faiss/python/python_callbacks.cpp +3 -1
- data/vendor/faiss/faiss/svs/IndexSVSFaissUtils.h +60 -0
- data/vendor/faiss/faiss/svs/IndexSVSFlat.cpp +26 -1
- data/vendor/faiss/faiss/svs/IndexSVSFlat.h +13 -0
- data/vendor/faiss/faiss/svs/IndexSVSIVF.cpp +1 -1
- data/vendor/faiss/faiss/svs/IndexSVSIVFLeanVec.cpp +1 -1
- data/vendor/faiss/faiss/svs/IndexSVSVamana.cpp +150 -23
- data/vendor/faiss/faiss/svs/IndexSVSVamana.h +30 -7
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.cpp +3 -2
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.h +2 -1
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.cpp +65 -25
- data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.h +3 -2
- data/vendor/faiss/faiss/utils/approx_topk_hamming/approx_topk_hamming.h +1 -1
- data/vendor/faiss/faiss/utils/bf16.h +34 -0
- data/vendor/faiss/faiss/utils/distances.cpp +14 -2
- data/vendor/faiss/faiss/utils/distances_simd.cpp +4 -4
- data/vendor/faiss/faiss/utils/extra_distances.cpp +4 -14
- data/vendor/faiss/faiss/utils/extra_distances.h +1 -2
- data/vendor/faiss/faiss/utils/hamming.cpp +9 -9
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx2.cpp +2 -1
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx512_spr.cpp +15 -0
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512.h +6 -30
- data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512_spr.h +171 -0
- data/vendor/faiss/faiss/utils/partitioning.cpp +0 -2
- data/vendor/faiss/faiss/utils/quantize_lut.cpp +29 -8
- data/vendor/faiss/faiss/utils/rabitq_simd.h +202 -0
- data/vendor/faiss/faiss/utils/simd_impl/distances_avx2.cpp +0 -1
- data/vendor/faiss/faiss/utils/simd_impl/distances_avx512.cpp +263 -15
- data/vendor/faiss/faiss/utils/simd_impl/distances_rvv.cpp +160 -18
- data/vendor/faiss/faiss/utils/simd_impl/partitioning_simdlib256.h +14 -68
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx2.cpp +245 -0
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512.cpp +273 -0
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512_spr.cpp +435 -0
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_neon.cpp +11 -0
- data/vendor/faiss/faiss/utils/simd_impl/rabitq_rvv.cpp +143 -6
- data/vendor/faiss/faiss/utils/simd_levels.cpp +56 -2
- data/vendor/faiss/faiss/utils/simd_levels.h +14 -0
- data/vendor/faiss/faiss/utils/utils.cpp +9 -27
- metadata +27 -2
|
@@ -405,7 +405,8 @@ void OnDiskInvertedLists::update_entries(
|
|
|
405
405
|
size_t n_entry,
|
|
406
406
|
const idx_t* ids_in,
|
|
407
407
|
const uint8_t* codes_in) {
|
|
408
|
-
|
|
408
|
+
FAISS_THROW_IF_MSG(
|
|
409
|
+
read_only, "cannot modify a read-only OnDiskInvertedLists");
|
|
409
410
|
if (n_entry == 0) {
|
|
410
411
|
return;
|
|
411
412
|
}
|
|
@@ -422,7 +423,8 @@ size_t OnDiskInvertedLists::add_entries(
|
|
|
422
423
|
size_t n_entry,
|
|
423
424
|
const idx_t* ids,
|
|
424
425
|
const uint8_t* code) {
|
|
425
|
-
|
|
426
|
+
FAISS_THROW_IF_MSG(
|
|
427
|
+
read_only, "cannot modify a read-only OnDiskInvertedLists");
|
|
426
428
|
locks->lock_1(list_no);
|
|
427
429
|
size_t o = list_size(list_no);
|
|
428
430
|
resize_locked(list_no, n_entry + o);
|
|
@@ -432,7 +434,8 @@ size_t OnDiskInvertedLists::add_entries(
|
|
|
432
434
|
}
|
|
433
435
|
|
|
434
436
|
void OnDiskInvertedLists::resize(size_t list_no, size_t new_size) {
|
|
435
|
-
|
|
437
|
+
FAISS_THROW_IF_MSG(
|
|
438
|
+
read_only, "cannot modify a read-only OnDiskInvertedLists");
|
|
436
439
|
locks->lock_1(list_no);
|
|
437
440
|
resize_locked(list_no, new_size);
|
|
438
441
|
locks->unlock_1(list_no);
|
|
@@ -799,7 +802,19 @@ InvertedLists* OnDiskInvertedListsIOHook::read_ArrayInvertedLists(
|
|
|
799
802
|
OnDiskInvertedLists::List& l = ails->lists[i];
|
|
800
803
|
l.size = l.capacity = sizes[i];
|
|
801
804
|
l.offset = o;
|
|
802
|
-
|
|
805
|
+
size_t elem_size = add_no_overflow(
|
|
806
|
+
sizeof(idx_t), ails->code_size, "OnDisk inverted list element");
|
|
807
|
+
size_t list_bytes =
|
|
808
|
+
mul_no_overflow(l.size, elem_size, "OnDisk inverted list");
|
|
809
|
+
o = add_no_overflow(o, list_bytes, "OnDisk inverted list offset");
|
|
810
|
+
FAISS_THROW_IF_NOT_FMT(
|
|
811
|
+
o <= ails->totsize,
|
|
812
|
+
"inverted list %zu at offset %zu with %zu bytes exceeds "
|
|
813
|
+
"mapped file size %zu",
|
|
814
|
+
i,
|
|
815
|
+
l.offset,
|
|
816
|
+
list_bytes,
|
|
817
|
+
ails->totsize);
|
|
803
818
|
}
|
|
804
819
|
// resume normal reading of file
|
|
805
820
|
fseek(fdesc, o, SEEK_SET);
|
|
@@ -151,7 +151,9 @@ int64_t PyCallbackShardingFunction::operator()(int64_t i, int64_t shard_count) {
|
|
|
151
151
|
if (shard_id == nullptr) {
|
|
152
152
|
FAISS_THROW_MSG("propagate py error");
|
|
153
153
|
}
|
|
154
|
-
|
|
154
|
+
int64_t rv = PyLong_AsLongLong(shard_id);
|
|
155
|
+
Py_DECREF(shard_id);
|
|
156
|
+
return rv;
|
|
155
157
|
}
|
|
156
158
|
|
|
157
159
|
PyCallbackShardingFunction::~PyCallbackShardingFunction() {
|
|
@@ -30,6 +30,7 @@
|
|
|
30
30
|
#include <faiss/impl/AuxIndexStructures.h>
|
|
31
31
|
#include <faiss/impl/FaissAssert.h>
|
|
32
32
|
#include <faiss/impl/IDSelector.h>
|
|
33
|
+
#include <faiss/impl/mapped_io.h>
|
|
33
34
|
|
|
34
35
|
#include <algorithm>
|
|
35
36
|
#include <concepts>
|
|
@@ -248,4 +249,63 @@ struct FaissResultsAllocator : public svs_runtime::ResultsAllocator {
|
|
|
248
249
|
using LabelsConverter = OutputBufferConverter<size_t, faiss::idx_t>;
|
|
249
250
|
mutable LabelsConverter labels_converter;
|
|
250
251
|
};
|
|
252
|
+
|
|
253
|
+
// Helper for memory-mapped SVS index loading.
|
|
254
|
+
// Acquires a pointer to the remaining mapped region and returns the data
|
|
255
|
+
// pointer, size in bytes, and starting position. Validates that mmap returns
|
|
256
|
+
// expected byte count. MappedFileIOReader::mmap returns number of items (not
|
|
257
|
+
// bytes); we always request size=1 so items == bytes, but we name variables
|
|
258
|
+
// explicitly to avoid confusion.
|
|
259
|
+
struct MmapSpan {
|
|
260
|
+
void* data = nullptr;
|
|
261
|
+
size_t size_bytes = 0;
|
|
262
|
+
size_t start_pos = 0;
|
|
263
|
+
};
|
|
264
|
+
|
|
265
|
+
inline MmapSpan acquire_mmap_span(MappedFileIOReader* mf) {
|
|
266
|
+
FAISS_THROW_IF_NOT(mf);
|
|
267
|
+
FAISS_THROW_IF_NOT(mf->mmap_owner);
|
|
268
|
+
size_t pos = mf->pos;
|
|
269
|
+
size_t size_to_end = mf->mmap_owner->size() - pos;
|
|
270
|
+
// Reject an empty span: MappedFileIOReader::mmap() returns 0 without
|
|
271
|
+
// writing *ptr when no bytes remain, which would otherwise pass the
|
|
272
|
+
// size check below and hand a nullptr / zero size to the SVS runtime
|
|
273
|
+
// (e.g. for a truncated or corrupt file).
|
|
274
|
+
FAISS_THROW_IF_NOT_FMT(
|
|
275
|
+
size_to_end > 0,
|
|
276
|
+
"acquire_mmap_span: no mapped bytes remain at reader position %zu",
|
|
277
|
+
pos);
|
|
278
|
+
void* data = nullptr;
|
|
279
|
+
// mmap returns actual_nitems; with size=1 this equals bytes
|
|
280
|
+
size_t actual_nitems = mf->mmap(&data, 1, size_to_end);
|
|
281
|
+
FAISS_THROW_IF_NOT_FMT(
|
|
282
|
+
actual_nitems == size_to_end,
|
|
283
|
+
"mmap() returned unexpected size: %zu items (expected %zu bytes)",
|
|
284
|
+
actual_nitems,
|
|
285
|
+
size_to_end);
|
|
286
|
+
return {data, size_to_end, pos};
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
// Adjusts MappedFileIOReader position after SVS consumes read_bytes,
|
|
290
|
+
// storing mmap_owner reference to keep mapping alive.
|
|
291
|
+
inline void finalize_mmap_span(
|
|
292
|
+
MappedFileIOReader* mf,
|
|
293
|
+
const MmapSpan& span,
|
|
294
|
+
size_t read_bytes,
|
|
295
|
+
std::shared_ptr<MmappedFileMappingOwner>& mmap_owner_out) {
|
|
296
|
+
// Take ownership of the mapping BEFORE validating read_bytes. By this point
|
|
297
|
+
// the caller's impl already holds pointers into the mapped region, so the
|
|
298
|
+
// index must keep the mapping alive even on the throw path. Otherwise the
|
|
299
|
+
// index would be left with a live impl but no mmap_owner, and once the
|
|
300
|
+
// reader's mapping reference is released the index destructor (which calls
|
|
301
|
+
// destroy(impl)) would touch unmapped memory.
|
|
302
|
+
mmap_owner_out = mf->mmap_owner;
|
|
303
|
+
FAISS_THROW_IF_NOT_FMT(
|
|
304
|
+
read_bytes > 0 && read_bytes <= span.size_bytes,
|
|
305
|
+
"map_to_memory returned invalid read_bytes: %zu (span size %zu)",
|
|
306
|
+
read_bytes,
|
|
307
|
+
span.size_bytes);
|
|
308
|
+
mf->pos = span.start_pos + read_bytes;
|
|
309
|
+
}
|
|
310
|
+
|
|
251
311
|
} // namespace faiss
|
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
*/
|
|
23
23
|
|
|
24
24
|
#include <faiss/Index.h>
|
|
25
|
+
#include <faiss/impl/mapped_io.h>
|
|
25
26
|
#include <faiss/svs/IndexSVSFaissUtils.h>
|
|
26
27
|
#include <faiss/svs/IndexSVSFlat.h>
|
|
27
28
|
|
|
@@ -55,11 +56,16 @@ void IndexSVSFlat::add(idx_t n, const float* x) {
|
|
|
55
56
|
|
|
56
57
|
void IndexSVSFlat::reset() {
|
|
57
58
|
if (impl) {
|
|
58
|
-
|
|
59
|
+
// Destroy impl to ensure no dangling pointers to mmap region remain.
|
|
60
|
+
// FlatIndex::reset may retain views; destroy guarantees clean state
|
|
61
|
+
// matching Vamana static behavior.
|
|
62
|
+
auto status = svs_runtime::FlatIndex::destroy(impl);
|
|
59
63
|
if (!status.ok()) {
|
|
60
64
|
FAISS_THROW_MSG(status.message());
|
|
61
65
|
}
|
|
66
|
+
impl = nullptr;
|
|
62
67
|
}
|
|
68
|
+
mmap_owner.reset(); // Release the memory mapping
|
|
63
69
|
ntotal = 0;
|
|
64
70
|
}
|
|
65
71
|
|
|
@@ -114,4 +120,23 @@ void IndexSVSFlat::deserialize_impl(std::istream& in) {
|
|
|
114
120
|
FAISS_THROW_IF_NOT_MSG(impl, "Failed to load SVS Flat index.");
|
|
115
121
|
}
|
|
116
122
|
|
|
123
|
+
void IndexSVSFlat::map_to(MappedFileIOReader* mf) {
|
|
124
|
+
FAISS_THROW_IF_MSG(impl, "Cannot map_to: SVS index already loaded.");
|
|
125
|
+
FAISS_THROW_IF_NOT(mf);
|
|
126
|
+
|
|
127
|
+
MmapSpan span = acquire_mmap_span(mf);
|
|
128
|
+
|
|
129
|
+
auto svs_metric = to_svs_metric(metric_type);
|
|
130
|
+
|
|
131
|
+
size_t read_bytes = 0;
|
|
132
|
+
auto status = svs_runtime::FlatIndex::map_to_memory(
|
|
133
|
+
&impl, span.data, span.size_bytes, svs_metric, &read_bytes);
|
|
134
|
+
|
|
135
|
+
if (!status.ok()) {
|
|
136
|
+
FAISS_THROW_MSG(status.message());
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
finalize_mmap_span(mf, span, read_bytes, mmap_owner);
|
|
140
|
+
}
|
|
141
|
+
|
|
117
142
|
} // namespace faiss
|
|
@@ -29,9 +29,14 @@
|
|
|
29
29
|
#include <svs/runtime/flat_index.h>
|
|
30
30
|
|
|
31
31
|
#include <iostream>
|
|
32
|
+
#include <memory>
|
|
32
33
|
|
|
33
34
|
namespace faiss {
|
|
34
35
|
|
|
36
|
+
// Forward declarations
|
|
37
|
+
struct MappedFileIOReader;
|
|
38
|
+
struct MmappedFileMappingOwner;
|
|
39
|
+
|
|
35
40
|
struct IndexSVSFlat : Index {
|
|
36
41
|
// sequential labels
|
|
37
42
|
size_t nlabels{0};
|
|
@@ -56,10 +61,18 @@ struct IndexSVSFlat : Index {
|
|
|
56
61
|
/* The actual SVS implementation */
|
|
57
62
|
svs_runtime::FlatIndex* impl{nullptr};
|
|
58
63
|
|
|
64
|
+
// Holds a reference to the memory-mapped file owner to keep the memory
|
|
65
|
+
// mapping alive for the lifetime of this index. Only used when index is
|
|
66
|
+
// loaded via map_to() with memory-mapped I/O.
|
|
67
|
+
std::shared_ptr<MmappedFileMappingOwner> mmap_owner{nullptr};
|
|
68
|
+
|
|
59
69
|
/* Serialization */
|
|
60
70
|
void serialize_impl(std::ostream& out) const;
|
|
61
71
|
void deserialize_impl(std::istream& in);
|
|
62
72
|
|
|
73
|
+
/* Memory-mapped deserialization */
|
|
74
|
+
void map_to(MappedFileIOReader* mf);
|
|
75
|
+
|
|
63
76
|
protected:
|
|
64
77
|
/* Initializes the implementation*/
|
|
65
78
|
virtual void create_impl();
|
|
@@ -222,7 +222,7 @@ size_t IndexSVSIVF::remove_ids(const IDSelector& sel) {
|
|
|
222
222
|
}
|
|
223
223
|
|
|
224
224
|
void IndexSVSIVF::create_impl(idx_t n, const float* x) {
|
|
225
|
-
|
|
225
|
+
FAISS_THROW_IF_MSG(impl, "impl already created");
|
|
226
226
|
ntotal = 0;
|
|
227
227
|
auto svs_metric = to_svs_metric(metric_type);
|
|
228
228
|
auto svs_storage_kind = to_svs_storage_kind(storage_kind);
|
|
@@ -119,7 +119,7 @@ void IndexSVSIVFLeanVec::deserialize_training_data(std::istream& in) {
|
|
|
119
119
|
}
|
|
120
120
|
|
|
121
121
|
void IndexSVSIVFLeanVec::create_impl(idx_t n, const float* x) {
|
|
122
|
-
|
|
122
|
+
FAISS_THROW_IF_MSG(impl, "impl already created");
|
|
123
123
|
ntotal = 0;
|
|
124
124
|
auto svs_metric = to_svs_metric(metric_type);
|
|
125
125
|
auto svs_storage_kind = to_svs_storage_kind(storage_kind);
|
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
#include <faiss/svs/IndexSVSVamana.h>
|
|
26
26
|
|
|
27
27
|
#include <faiss/Index.h>
|
|
28
|
+
#include <faiss/impl/mapped_io.h>
|
|
28
29
|
|
|
29
30
|
#include <svs/runtime/api_defs.h>
|
|
30
31
|
#include <svs/runtime/dynamic_vamana_index.h>
|
|
@@ -65,8 +66,12 @@ IndexSVSVamana::IndexSVSVamana(
|
|
|
65
66
|
idx_t d,
|
|
66
67
|
size_t degree,
|
|
67
68
|
MetricType metric,
|
|
68
|
-
SVSStorageKind storage
|
|
69
|
-
|
|
69
|
+
SVSStorageKind storage,
|
|
70
|
+
bool is_static)
|
|
71
|
+
: Index(d, metric),
|
|
72
|
+
graph_max_degree{degree},
|
|
73
|
+
is_static{is_static},
|
|
74
|
+
storage_kind{storage} {
|
|
70
75
|
prune_to = graph_max_degree < 4 ? graph_max_degree : graph_max_degree - 4;
|
|
71
76
|
alpha = metric == METRIC_L2 ? 1.2f : 0.95f;
|
|
72
77
|
|
|
@@ -74,8 +79,9 @@ IndexSVSVamana::IndexSVSVamana(
|
|
|
74
79
|
// NB: LVQ/LeanVec are only available on Intel(R) hardware AND when using
|
|
75
80
|
// a build based on LVQ/LeanVec-enabled SVS.
|
|
76
81
|
auto svs_storage = to_svs_storage_kind(storage_kind);
|
|
77
|
-
auto status =
|
|
78
|
-
svs_runtime::
|
|
82
|
+
auto status = is_static
|
|
83
|
+
? svs_runtime::VamanaIndex::check_storage_kind(svs_storage)
|
|
84
|
+
: svs_runtime::DynamicVamanaIndex::check_storage_kind(svs_storage);
|
|
79
85
|
if (!status.ok()) {
|
|
80
86
|
FAISS_THROW_MSG(status.message());
|
|
81
87
|
}
|
|
@@ -83,11 +89,19 @@ IndexSVSVamana::IndexSVSVamana(
|
|
|
83
89
|
|
|
84
90
|
bool IndexSVSVamana::is_lvq_leanvec_enabled() {
|
|
85
91
|
auto lvq = to_svs_storage_kind(SVS_LVQ4x0);
|
|
86
|
-
auto status = svs_runtime::
|
|
92
|
+
auto status = svs_runtime::VamanaIndex::check_storage_kind(lvq);
|
|
93
|
+
if (!status.ok()) {
|
|
94
|
+
return false;
|
|
95
|
+
}
|
|
96
|
+
status = svs_runtime::DynamicVamanaIndex::check_storage_kind(lvq);
|
|
87
97
|
if (!status.ok()) {
|
|
88
98
|
return false;
|
|
89
99
|
}
|
|
90
100
|
auto leanvec = to_svs_storage_kind(SVS_LeanVec4x4);
|
|
101
|
+
status = svs_runtime::VamanaIndex::check_storage_kind(leanvec);
|
|
102
|
+
if (!status.ok()) {
|
|
103
|
+
return false;
|
|
104
|
+
}
|
|
91
105
|
status = svs_runtime::DynamicVamanaIndex::check_storage_kind(leanvec);
|
|
92
106
|
if (!status.ok()) {
|
|
93
107
|
return false;
|
|
@@ -97,21 +111,43 @@ bool IndexSVSVamana::is_lvq_leanvec_enabled() {
|
|
|
97
111
|
|
|
98
112
|
IndexSVSVamana::~IndexSVSVamana() {
|
|
99
113
|
if (impl) {
|
|
100
|
-
|
|
114
|
+
svs_runtime::Status status;
|
|
115
|
+
if (is_static) {
|
|
116
|
+
status = svs_runtime::VamanaIndex::destroy(impl);
|
|
117
|
+
} else {
|
|
118
|
+
status = svs_runtime::DynamicVamanaIndex::destroy(
|
|
119
|
+
static_cast<svs_runtime::DynamicVamanaIndex*>(impl));
|
|
120
|
+
}
|
|
101
121
|
FAISS_ASSERT(status.ok());
|
|
102
122
|
impl = nullptr;
|
|
103
123
|
}
|
|
104
124
|
}
|
|
105
125
|
|
|
106
126
|
void IndexSVSVamana::add(idx_t n, const float* x) {
|
|
127
|
+
if (is_static) {
|
|
128
|
+
FAISS_THROW_IF_MSG(
|
|
129
|
+
impl,
|
|
130
|
+
"Static Vamana index does not support add() after initial "
|
|
131
|
+
"build. All data must be provided in a single add() call.");
|
|
132
|
+
create_impl(n, x);
|
|
133
|
+
if (stored_vectors_valid) {
|
|
134
|
+
stored_vectors.resize(static_cast<size_t>(n) * d);
|
|
135
|
+
std::memcpy(stored_vectors.data(), x, sizeof(float) * n * d);
|
|
136
|
+
}
|
|
137
|
+
ntotal = n;
|
|
138
|
+
is_trained = true;
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
|
|
107
142
|
if (!impl) {
|
|
108
|
-
create_impl();
|
|
143
|
+
create_impl(0, nullptr);
|
|
109
144
|
}
|
|
110
145
|
|
|
111
146
|
std::vector<size_t> labels(n);
|
|
112
147
|
std::iota(labels.begin(), labels.end(), ntotal);
|
|
113
148
|
|
|
114
|
-
auto
|
|
149
|
+
auto* dyn = dynamic_impl();
|
|
150
|
+
auto status = dyn->add(n, labels.data(), x);
|
|
115
151
|
if (!status.ok()) {
|
|
116
152
|
FAISS_THROW_MSG(status.message());
|
|
117
153
|
}
|
|
@@ -137,10 +173,22 @@ void IndexSVSVamana::reconstruct(idx_t key, float* recons) const {
|
|
|
137
173
|
|
|
138
174
|
void IndexSVSVamana::reset() {
|
|
139
175
|
if (impl) {
|
|
140
|
-
|
|
176
|
+
if (is_static) {
|
|
177
|
+
// Static index: destroy the impl so the next add() rebuilds from
|
|
178
|
+
// scratch. The static SVS backend has no in-place reset that
|
|
179
|
+
// permits a follow-up add().
|
|
180
|
+
auto status = svs_runtime::VamanaIndex::destroy(impl);
|
|
181
|
+
FAISS_ASSERT(status.ok());
|
|
182
|
+
impl = nullptr;
|
|
183
|
+
} else {
|
|
184
|
+
// Dynamic index: in-place reset preserves the impl and avoids
|
|
185
|
+
// tearing down its allocated state.
|
|
186
|
+
impl->reset();
|
|
187
|
+
}
|
|
141
188
|
}
|
|
142
189
|
stored_vectors.clear();
|
|
143
190
|
stored_vectors_valid = true;
|
|
191
|
+
mmap_owner.reset(); // Release the memory mapping
|
|
144
192
|
is_trained = false;
|
|
145
193
|
ntotal = 0;
|
|
146
194
|
}
|
|
@@ -204,18 +252,23 @@ void IndexSVSVamana::range_search(
|
|
|
204
252
|
}
|
|
205
253
|
|
|
206
254
|
size_t IndexSVSVamana::remove_ids(const IDSelector& sel) {
|
|
255
|
+
FAISS_THROW_IF_MSG(
|
|
256
|
+
is_static,
|
|
257
|
+
"Static Vamana index does not support remove_ids(). "
|
|
258
|
+
"The index is immutable after creation.");
|
|
207
259
|
FAISS_THROW_IF_NOT(impl);
|
|
260
|
+
auto* dyn = dynamic_impl();
|
|
208
261
|
auto id_filter = FaissIDFilter{sel};
|
|
209
262
|
size_t removed = 0;
|
|
210
|
-
auto Status =
|
|
263
|
+
auto Status = dyn->remove_selected(&removed, id_filter);
|
|
211
264
|
ntotal -= removed;
|
|
212
265
|
stored_vectors.clear();
|
|
213
266
|
stored_vectors_valid = false;
|
|
214
267
|
return removed;
|
|
215
268
|
}
|
|
216
269
|
|
|
217
|
-
void IndexSVSVamana::create_impl() {
|
|
218
|
-
|
|
270
|
+
void IndexSVSVamana::create_impl(idx_t n, const float* x) {
|
|
271
|
+
FAISS_THROW_IF_MSG(impl, "impl already created");
|
|
219
272
|
ntotal = 0;
|
|
220
273
|
auto svs_metric = to_svs_metric(metric_type);
|
|
221
274
|
auto svs_storage_kind = to_svs_storage_kind(storage_kind);
|
|
@@ -231,15 +284,45 @@ void IndexSVSVamana::create_impl() {
|
|
|
231
284
|
.search_window_size = search_window_size,
|
|
232
285
|
.search_buffer_capacity = search_buffer_capacity,
|
|
233
286
|
};
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
287
|
+
|
|
288
|
+
svs_runtime::Status Status;
|
|
289
|
+
if (is_static) {
|
|
290
|
+
FAISS_THROW_IF_NOT_MSG(
|
|
291
|
+
n > 0 && x != nullptr,
|
|
292
|
+
"Static Vamana index requires data at build time.");
|
|
293
|
+
Status = svs_runtime::VamanaIndex::build(
|
|
294
|
+
&impl,
|
|
295
|
+
d,
|
|
296
|
+
svs_metric,
|
|
297
|
+
svs_storage_kind,
|
|
298
|
+
build_params,
|
|
299
|
+
search_params);
|
|
300
|
+
if (!Status.ok()) {
|
|
301
|
+
FAISS_THROW_MSG(Status.message());
|
|
302
|
+
}
|
|
303
|
+
FAISS_THROW_IF_NOT(impl);
|
|
304
|
+
// Populate the static index with the full dataset (one-shot add).
|
|
305
|
+
Status = impl->add(static_cast<size_t>(n), x);
|
|
306
|
+
if (!Status.ok()) {
|
|
307
|
+
// Best-effort cleanup before propagating the error.
|
|
308
|
+
auto destroy_status = svs_runtime::VamanaIndex::destroy(impl);
|
|
309
|
+
FAISS_ASSERT(destroy_status.ok());
|
|
310
|
+
impl = nullptr;
|
|
311
|
+
FAISS_THROW_MSG(Status.message());
|
|
312
|
+
}
|
|
313
|
+
} else {
|
|
314
|
+
svs_runtime::DynamicVamanaIndex* dyn_impl = nullptr;
|
|
315
|
+
Status = svs_runtime::DynamicVamanaIndex::build(
|
|
316
|
+
&dyn_impl,
|
|
317
|
+
d,
|
|
318
|
+
svs_metric,
|
|
319
|
+
svs_storage_kind,
|
|
320
|
+
build_params,
|
|
321
|
+
search_params);
|
|
322
|
+
if (!Status.ok()) {
|
|
323
|
+
FAISS_THROW_MSG(Status.message());
|
|
324
|
+
}
|
|
325
|
+
impl = dyn_impl;
|
|
243
326
|
}
|
|
244
327
|
FAISS_THROW_IF_NOT(impl);
|
|
245
328
|
}
|
|
@@ -258,12 +341,56 @@ void IndexSVSVamana::deserialize_impl(std::istream& in) {
|
|
|
258
341
|
FAISS_THROW_IF_MSG(impl, "Cannot deserialize: SVS index already loaded.");
|
|
259
342
|
auto svs_metric = to_svs_metric(metric_type);
|
|
260
343
|
auto svs_storage_kind = to_svs_storage_kind(storage_kind);
|
|
261
|
-
|
|
262
|
-
|
|
344
|
+
|
|
345
|
+
svs_runtime::Status status;
|
|
346
|
+
if (is_static) {
|
|
347
|
+
status = svs_runtime::VamanaIndex::load(
|
|
348
|
+
&impl, in, svs_metric, svs_storage_kind);
|
|
349
|
+
} else {
|
|
350
|
+
svs_runtime::DynamicVamanaIndex* dyn_impl = nullptr;
|
|
351
|
+
status = svs_runtime::DynamicVamanaIndex::load(
|
|
352
|
+
&dyn_impl, in, svs_metric, svs_storage_kind);
|
|
353
|
+
impl = dyn_impl;
|
|
354
|
+
}
|
|
263
355
|
if (!status.ok()) {
|
|
264
356
|
FAISS_THROW_MSG(status.message());
|
|
265
357
|
}
|
|
266
358
|
FAISS_THROW_IF_NOT_MSG(impl, "Failed to load SVS Vamana index.");
|
|
267
359
|
}
|
|
268
360
|
|
|
361
|
+
svs_runtime::DynamicVamanaIndex* IndexSVSVamana::dynamic_impl() const {
|
|
362
|
+
FAISS_THROW_IF_MSG(
|
|
363
|
+
is_static, "Operation not supported on a static Vamana index.");
|
|
364
|
+
FAISS_THROW_IF_NOT(impl);
|
|
365
|
+
return static_cast<svs_runtime::DynamicVamanaIndex*>(impl);
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
void IndexSVSVamana::map_to(MappedFileIOReader* mf) {
|
|
369
|
+
FAISS_THROW_IF_MSG(
|
|
370
|
+
!is_static,
|
|
371
|
+
"map_to() is only supported for static Vamana indices.");
|
|
372
|
+
FAISS_THROW_IF_MSG(impl, "Cannot map_to: SVS index already loaded.");
|
|
373
|
+
FAISS_THROW_IF_NOT(mf);
|
|
374
|
+
|
|
375
|
+
MmapSpan span = acquire_mmap_span(mf);
|
|
376
|
+
|
|
377
|
+
auto svs_metric = to_svs_metric(metric_type);
|
|
378
|
+
auto svs_storage_kind = to_svs_storage_kind(storage_kind);
|
|
379
|
+
|
|
380
|
+
size_t read_bytes = 0;
|
|
381
|
+
auto status = svs_runtime::VamanaIndex::map_to_memory(
|
|
382
|
+
&impl,
|
|
383
|
+
span.data,
|
|
384
|
+
span.size_bytes,
|
|
385
|
+
svs_metric,
|
|
386
|
+
svs_storage_kind,
|
|
387
|
+
&read_bytes);
|
|
388
|
+
|
|
389
|
+
if (!status.ok()) {
|
|
390
|
+
FAISS_THROW_MSG(status.message());
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
finalize_mmap_span(mf, span, read_bytes, mmap_owner);
|
|
394
|
+
}
|
|
395
|
+
|
|
269
396
|
} // namespace faiss
|
|
@@ -30,11 +30,16 @@
|
|
|
30
30
|
#include <svs/runtime/dynamic_vamana_index.h>
|
|
31
31
|
|
|
32
32
|
#include <iostream>
|
|
33
|
+
#include <memory>
|
|
33
34
|
#include <type_traits>
|
|
34
35
|
#include <vector>
|
|
35
36
|
|
|
36
37
|
namespace faiss {
|
|
37
38
|
|
|
39
|
+
// Forward declarations
|
|
40
|
+
struct MappedFileIOReader;
|
|
41
|
+
struct MmappedFileMappingOwner;
|
|
42
|
+
|
|
38
43
|
struct SearchParametersSVSVamana : public SearchParameters {
|
|
39
44
|
size_t search_window_size = 0;
|
|
40
45
|
size_t search_buffer_capacity = 0;
|
|
@@ -44,7 +49,7 @@ struct SearchParametersSVSVamana : public SearchParameters {
|
|
|
44
49
|
enum SVSStorageKind {
|
|
45
50
|
SVS_FP32,
|
|
46
51
|
SVS_FP16,
|
|
47
|
-
|
|
52
|
+
SVS_SQ8,
|
|
48
53
|
SVS_LVQ4x0,
|
|
49
54
|
SVS_LVQ4x4,
|
|
50
55
|
SVS_LVQ4x8,
|
|
@@ -61,7 +66,7 @@ inline svs_runtime::StorageKind to_svs_storage_kind(SVSStorageKind kind) {
|
|
|
61
66
|
return svs_runtime::StorageKind::FP32;
|
|
62
67
|
case SVS_FP16:
|
|
63
68
|
return svs_runtime::StorageKind::FP16;
|
|
64
|
-
case
|
|
69
|
+
case SVS_SQ8:
|
|
65
70
|
return svs_runtime::StorageKind::SQI8;
|
|
66
71
|
case SVS_LVQ4x0:
|
|
67
72
|
return svs_runtime::StorageKind::LVQ4x0;
|
|
@@ -94,6 +99,9 @@ struct IndexSVSVamana : Index {
|
|
|
94
99
|
size_t max_candidate_pool_size = 200;
|
|
95
100
|
bool use_full_search_history = true;
|
|
96
101
|
|
|
102
|
+
/// Whether this is a static (immutable) Vamana index
|
|
103
|
+
bool is_static = false;
|
|
104
|
+
|
|
97
105
|
SVSStorageKind storage_kind = SVS_FP32;
|
|
98
106
|
|
|
99
107
|
IndexSVSVamana();
|
|
@@ -102,7 +110,8 @@ struct IndexSVSVamana : Index {
|
|
|
102
110
|
idx_t d,
|
|
103
111
|
size_t degree,
|
|
104
112
|
MetricType metric = METRIC_L2,
|
|
105
|
-
SVSStorageKind storage = SVSStorageKind::SVS_FP32
|
|
113
|
+
SVSStorageKind storage = SVSStorageKind::SVS_FP32,
|
|
114
|
+
bool is_static = false);
|
|
106
115
|
|
|
107
116
|
~IndexSVSVamana() override;
|
|
108
117
|
|
|
@@ -137,8 +146,17 @@ struct IndexSVSVamana : Index {
|
|
|
137
146
|
void serialize_impl(std::ostream& out) const;
|
|
138
147
|
virtual void deserialize_impl(std::istream& in);
|
|
139
148
|
|
|
140
|
-
/*
|
|
141
|
-
|
|
149
|
+
/* Memory-mapped deserialization for static indices */
|
|
150
|
+
virtual void map_to(MappedFileIOReader* mf);
|
|
151
|
+
|
|
152
|
+
/* The actual SVS implementation (VamanaIndex is the base for both
|
|
153
|
+
static and dynamic variants) */
|
|
154
|
+
svs_runtime::VamanaIndex* impl{nullptr};
|
|
155
|
+
|
|
156
|
+
// Holds a reference to the memory-mapped file owner to keep the memory
|
|
157
|
+
// mapping alive for the lifetime of this index. Only used when index is
|
|
158
|
+
// loaded via map_to() with memory-mapped I/O.
|
|
159
|
+
std::shared_ptr<MmappedFileMappingOwner> mmap_owner{nullptr};
|
|
142
160
|
|
|
143
161
|
// The SVS runtime API does not expose vector retrieval, so we keep a copy
|
|
144
162
|
// of added vectors to support reconstruct(). When used as a coarse
|
|
@@ -147,8 +165,13 @@ struct IndexSVSVamana : Index {
|
|
|
147
165
|
bool stored_vectors_valid{true};
|
|
148
166
|
|
|
149
167
|
protected:
|
|
150
|
-
/* Initializes the implementation
|
|
151
|
-
|
|
168
|
+
/* Initializes the implementation. For static indexes the data is consumed
|
|
169
|
+
at build time; for dynamic indexes n/x are ignored and add() populates
|
|
170
|
+
the index afterwards. */
|
|
171
|
+
virtual void create_impl(idx_t n, const float* x);
|
|
172
|
+
|
|
173
|
+
/* Returns the dynamic impl pointer, throwing if static */
|
|
174
|
+
svs_runtime::DynamicVamanaIndex* dynamic_impl() const;
|
|
152
175
|
};
|
|
153
176
|
|
|
154
177
|
} // namespace faiss
|
|
@@ -33,7 +33,8 @@ IndexSVSVamanaLVQ::IndexSVSVamanaLVQ(
|
|
|
33
33
|
idx_t d,
|
|
34
34
|
size_t degree,
|
|
35
35
|
MetricType metric,
|
|
36
|
-
SVSStorageKind storage
|
|
37
|
-
|
|
36
|
+
SVSStorageKind storage,
|
|
37
|
+
bool is_static)
|
|
38
|
+
: IndexSVSVamana(d, degree, metric, storage, is_static) {}
|
|
38
39
|
|
|
39
40
|
} // namespace faiss
|
|
@@ -34,7 +34,8 @@ struct IndexSVSVamanaLVQ : IndexSVSVamana {
|
|
|
34
34
|
idx_t d,
|
|
35
35
|
size_t degree,
|
|
36
36
|
MetricType metric = METRIC_L2,
|
|
37
|
-
SVSStorageKind storage = SVSStorageKind::SVS_LVQ4x0
|
|
37
|
+
SVSStorageKind storage = SVSStorageKind::SVS_LVQ4x0,
|
|
38
|
+
bool is_static = false);
|
|
38
39
|
|
|
39
40
|
~IndexSVSVamanaLVQ() override = default;
|
|
40
41
|
};
|