faiss 0.6.1 → 0.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -0
  3. data/lib/faiss/version.rb +1 -1
  4. data/vendor/faiss/faiss/AutoTune.cpp +3 -1
  5. data/vendor/faiss/faiss/Clustering.cpp +9 -1
  6. data/vendor/faiss/faiss/IVFlib.cpp +14 -3
  7. data/vendor/faiss/faiss/Index.h +2 -2
  8. data/vendor/faiss/faiss/IndexAdditiveQuantizer.cpp +9 -10
  9. data/vendor/faiss/faiss/IndexAdditiveQuantizerFastScan.cpp +2 -3
  10. data/vendor/faiss/faiss/IndexBinaryFromFloat.cpp +1 -2
  11. data/vendor/faiss/faiss/IndexBinaryHNSW.cpp +10 -12
  12. data/vendor/faiss/faiss/IndexBinaryHash.cpp +5 -9
  13. data/vendor/faiss/faiss/IndexBinaryIVF.cpp +5 -7
  14. data/vendor/faiss/faiss/IndexEDEN.cpp +273 -0
  15. data/vendor/faiss/faiss/IndexEDEN.h +57 -0
  16. data/vendor/faiss/faiss/IndexFastScan.cpp +15 -4
  17. data/vendor/faiss/faiss/IndexFlat.cpp +13 -50
  18. data/vendor/faiss/faiss/IndexHNSW.cpp +177 -148
  19. data/vendor/faiss/faiss/IndexIDMap.cpp +16 -3
  20. data/vendor/faiss/faiss/IndexIDMap.h +2 -0
  21. data/vendor/faiss/faiss/IndexIVF.cpp +19 -8
  22. data/vendor/faiss/faiss/IndexIVFAdditiveQuantizer.cpp +3 -3
  23. data/vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.cpp +3 -4
  24. data/vendor/faiss/faiss/IndexIVFEDEN.cpp +302 -0
  25. data/vendor/faiss/faiss/IndexIVFEDEN.h +70 -0
  26. data/vendor/faiss/faiss/IndexIVFFastScan.cpp +5 -6
  27. data/vendor/faiss/faiss/IndexIVFFlat.cpp +6 -5
  28. data/vendor/faiss/faiss/IndexIVFFlatPanorama.cpp +3 -3
  29. data/vendor/faiss/faiss/IndexIVFIndependentQuantizer.cpp +1 -1
  30. data/vendor/faiss/faiss/IndexIVFPQ.cpp +42 -25
  31. data/vendor/faiss/faiss/IndexIVFPQFastScan.cpp +0 -1
  32. data/vendor/faiss/faiss/IndexIVFPQR.cpp +2 -3
  33. data/vendor/faiss/faiss/IndexIVFRaBitQ.cpp +23 -62
  34. data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.cpp +180 -76
  35. data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.h +5 -4
  36. data/vendor/faiss/faiss/IndexIVFSpectralHash.cpp +8 -6
  37. data/vendor/faiss/faiss/IndexLSH.cpp +2 -3
  38. data/vendor/faiss/faiss/IndexLattice.cpp +5 -0
  39. data/vendor/faiss/faiss/IndexNNDescent.cpp +10 -3
  40. data/vendor/faiss/faiss/IndexNSG.cpp +8 -4
  41. data/vendor/faiss/faiss/IndexPQ.cpp +6 -8
  42. data/vendor/faiss/faiss/IndexPreTransform.cpp +15 -0
  43. data/vendor/faiss/faiss/IndexRaBitQ.cpp +2 -2
  44. data/vendor/faiss/faiss/IndexRaBitQFastScan.cpp +1 -2
  45. data/vendor/faiss/faiss/IndexRaBitQFastScan.h +5 -1
  46. data/vendor/faiss/faiss/IndexRefine.cpp +30 -1
  47. data/vendor/faiss/faiss/IndexReplicas.cpp +1 -2
  48. data/vendor/faiss/faiss/IndexScalarQuantizer.cpp +68 -6
  49. data/vendor/faiss/faiss/IndexScalarQuantizer.h +10 -0
  50. data/vendor/faiss/faiss/IndexShards.cpp +2 -2
  51. data/vendor/faiss/faiss/IndexShardsIVF.cpp +2 -2
  52. data/vendor/faiss/faiss/MetaIndexes.cpp +2 -4
  53. data/vendor/faiss/faiss/SuperKMeans.cpp +256 -240
  54. data/vendor/faiss/faiss/SuperKMeans.h +30 -0
  55. data/vendor/faiss/faiss/VectorTransform.cpp +33 -2
  56. data/vendor/faiss/faiss/clone_index.cpp +5 -0
  57. data/vendor/faiss/faiss/cppcontrib/SaDecodeKernels.h +1 -1
  58. data/vendor/faiss/faiss/cppcontrib/sa_decode/Level2-neon-inl.h +902 -12
  59. data/vendor/faiss/faiss/cppcontrib/sa_decode/PQ-neon-inl.h +702 -10
  60. data/vendor/faiss/faiss/factory_tools.cpp +51 -4
  61. data/vendor/faiss/faiss/gpu/GpuCloner.cpp +11 -11
  62. data/vendor/faiss/faiss/gpu/GpuIndex.h +34 -11
  63. data/vendor/faiss/faiss/gpu/GpuIndexCagra.h +47 -0
  64. data/vendor/faiss/faiss/gpu/GpuIndexIVF.h +17 -0
  65. data/vendor/faiss/faiss/gpu/GpuIndexIVFScalarQuantizer.h +16 -0
  66. data/vendor/faiss/faiss/gpu/GpuResources.h +3 -2
  67. data/vendor/faiss/faiss/gpu/StandardGpuResources.cpp +11 -12
  68. data/vendor/faiss/faiss/gpu/StandardGpuResources.h +3 -3
  69. data/vendor/faiss/faiss/gpu/perf/PerfClustering.cpp +1 -1
  70. data/vendor/faiss/faiss/gpu/perf/PerfIVFPQAdd.cpp +2 -2
  71. data/vendor/faiss/faiss/gpu/test/TestGpuIndexIVFScalarQuantizer.cpp +180 -0
  72. data/vendor/faiss/faiss/gpu_metal/MetalDistance.h +87 -0
  73. data/vendor/faiss/faiss/gpu_metal/MetalIndex.h +7 -0
  74. data/vendor/faiss/faiss/gpu_metal/MetalIndexIVFFlat.h +177 -0
  75. data/vendor/faiss/faiss/gpu_metal/MetalIndexIVFPQ.h +88 -0
  76. data/vendor/faiss/faiss/gpu_metal/MetalKernels.h +48 -3
  77. data/vendor/faiss/faiss/gpu_metal/MetalPythonBridge.h +45 -0
  78. data/vendor/faiss/faiss/gpu_metal/impl/MetalIVFFlat.h +193 -0
  79. data/vendor/faiss/faiss/gpu_metal/impl/MetalIVFPQ.h +134 -0
  80. data/vendor/faiss/faiss/impl/ClusteringInitialization.cpp +2 -2
  81. data/vendor/faiss/faiss/impl/DistanceComputer.h +34 -0
  82. data/vendor/faiss/faiss/impl/EDENQuantizer.h +119 -0
  83. data/vendor/faiss/faiss/impl/HNSW.cpp +658 -344
  84. data/vendor/faiss/faiss/impl/HNSW.h +51 -13
  85. data/vendor/faiss/faiss/impl/LocalSearchQuantizer.cpp +2 -2
  86. data/vendor/faiss/faiss/impl/NSG.cpp +18 -12
  87. data/vendor/faiss/faiss/impl/Panorama.h +20 -7
  88. data/vendor/faiss/faiss/impl/PolysemousTraining.cpp +152 -84
  89. data/vendor/faiss/faiss/impl/ProductQuantizer.cpp +59 -24
  90. data/vendor/faiss/faiss/impl/RaBitQUtils.cpp +45 -37
  91. data/vendor/faiss/faiss/impl/RaBitQUtils.h +35 -0
  92. data/vendor/faiss/faiss/impl/RaBitQuantizer.cpp +175 -68
  93. data/vendor/faiss/faiss/impl/RaBitQuantizer.h +19 -0
  94. data/vendor/faiss/faiss/impl/RaBitQuantizerMultiBit.cpp +2 -11
  95. data/vendor/faiss/faiss/impl/ResultHandler.h +26 -31
  96. data/vendor/faiss/faiss/impl/ScalarQuantizer.cpp +522 -58
  97. data/vendor/faiss/faiss/impl/ScalarQuantizer.h +70 -0
  98. data/vendor/faiss/faiss/impl/ThreadedIndex-inl.h +2 -2
  99. data/vendor/faiss/faiss/impl/VisitedTable.cpp +33 -13
  100. data/vendor/faiss/faiss/impl/VisitedTable.h +88 -33
  101. data/vendor/faiss/faiss/impl/binary_hamming/IndexBinaryIVF_impl.h +1 -1
  102. data/vendor/faiss/faiss/impl/binary_hamming/avx2.cpp +4 -4
  103. data/vendor/faiss/faiss/impl/fast_scan/dispatching.h +38 -3
  104. data/vendor/faiss/faiss/impl/hnsw/LockVector.cpp +1 -1
  105. data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.cpp +35 -43
  106. data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.h +64 -15
  107. data/vendor/faiss/faiss/impl/hnsw/avx2.cpp +86 -40
  108. data/vendor/faiss/faiss/impl/hnsw/avx512.cpp +81 -50
  109. data/vendor/faiss/faiss/impl/index_read.cpp +476 -75
  110. data/vendor/faiss/faiss/impl/index_write.cpp +56 -4
  111. data/vendor/faiss/faiss/impl/io_macros.h +25 -0
  112. data/vendor/faiss/faiss/impl/lattice_Zn.cpp +8 -9
  113. data/vendor/faiss/faiss/impl/platform_macros.h +15 -9
  114. data/vendor/faiss/faiss/impl/polysemous_training/avx512.cpp +284 -0
  115. data/vendor/faiss/faiss/impl/polysemous_training/dispatch.h +115 -0
  116. data/vendor/faiss/faiss/impl/pq_code_distance/IVFPQ_QueryTables.cpp +0 -1
  117. data/vendor/faiss/faiss/impl/pq_code_distance/PQDistanceComputer_impl.h +26 -15
  118. data/vendor/faiss/faiss/impl/pq_code_distance/avx2.cpp +6 -4
  119. data/vendor/faiss/faiss/impl/pq_code_distance/avx512.cpp +2 -0
  120. data/vendor/faiss/faiss/impl/pq_code_distance/neon.cpp +2 -0
  121. data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-generic.cpp +20 -0
  122. data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-inl.h +36 -0
  123. data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-sve.cpp +5 -0
  124. data/vendor/faiss/faiss/impl/pq_code_distance/pq_scan_impl.h +105 -0
  125. data/vendor/faiss/faiss/impl/pq_code_distance/rvv.cpp +2 -0
  126. data/vendor/faiss/faiss/impl/result_handler/ResultHandler.cpp +195 -0
  127. data/vendor/faiss/faiss/impl/result_handler/avx2.cpp +133 -0
  128. data/vendor/faiss/faiss/impl/result_handler/avx512.cpp +281 -0
  129. data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx2.cpp +72 -0
  130. data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer-avx512.cpp +228 -0
  131. data/vendor/faiss/faiss/impl/scalar_quantizer/EDENQuantizer.cpp +882 -0
  132. data/vendor/faiss/faiss/impl/scalar_quantizer/distance_computers.h +6 -0
  133. data/vendor/faiss/faiss/impl/scalar_quantizer/quantizers.h +336 -26
  134. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx2.cpp +331 -32
  135. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-impl.h +553 -0
  136. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-spr.cpp +558 -0
  137. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512.cpp +284 -45
  138. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-dispatch.h +502 -3
  139. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-neon.cpp +157 -32
  140. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-rvv.cpp +26 -0
  141. data/vendor/faiss/faiss/impl/simd_dispatch.h +86 -8
  142. data/vendor/faiss/faiss/index_factory.cpp +37 -7
  143. data/vendor/faiss/faiss/index_io.h +16 -0
  144. data/vendor/faiss/faiss/invlists/DirectMap.cpp +5 -2
  145. data/vendor/faiss/faiss/invlists/InvertedLists.cpp +15 -15
  146. data/vendor/faiss/faiss/invlists/InvertedLists.h +2 -2
  147. data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.cpp +19 -4
  148. data/vendor/faiss/faiss/python/python_callbacks.cpp +3 -1
  149. data/vendor/faiss/faiss/svs/IndexSVSFaissUtils.h +60 -0
  150. data/vendor/faiss/faiss/svs/IndexSVSFlat.cpp +26 -1
  151. data/vendor/faiss/faiss/svs/IndexSVSFlat.h +13 -0
  152. data/vendor/faiss/faiss/svs/IndexSVSIVF.cpp +1 -1
  153. data/vendor/faiss/faiss/svs/IndexSVSIVFLeanVec.cpp +1 -1
  154. data/vendor/faiss/faiss/svs/IndexSVSVamana.cpp +150 -23
  155. data/vendor/faiss/faiss/svs/IndexSVSVamana.h +30 -7
  156. data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.cpp +3 -2
  157. data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.h +2 -1
  158. data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.cpp +65 -25
  159. data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.h +3 -2
  160. data/vendor/faiss/faiss/utils/approx_topk_hamming/approx_topk_hamming.h +1 -1
  161. data/vendor/faiss/faiss/utils/bf16.h +34 -0
  162. data/vendor/faiss/faiss/utils/distances.cpp +14 -2
  163. data/vendor/faiss/faiss/utils/distances_simd.cpp +4 -4
  164. data/vendor/faiss/faiss/utils/extra_distances.cpp +4 -14
  165. data/vendor/faiss/faiss/utils/extra_distances.h +1 -2
  166. data/vendor/faiss/faiss/utils/hamming.cpp +9 -9
  167. data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx2.cpp +2 -1
  168. data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx512_spr.cpp +15 -0
  169. data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512.h +6 -30
  170. data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512_spr.h +171 -0
  171. data/vendor/faiss/faiss/utils/partitioning.cpp +0 -2
  172. data/vendor/faiss/faiss/utils/quantize_lut.cpp +29 -8
  173. data/vendor/faiss/faiss/utils/rabitq_simd.h +202 -0
  174. data/vendor/faiss/faiss/utils/simd_impl/distances_avx2.cpp +0 -1
  175. data/vendor/faiss/faiss/utils/simd_impl/distances_avx512.cpp +263 -15
  176. data/vendor/faiss/faiss/utils/simd_impl/distances_rvv.cpp +160 -18
  177. data/vendor/faiss/faiss/utils/simd_impl/partitioning_simdlib256.h +14 -68
  178. data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx2.cpp +245 -0
  179. data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512.cpp +273 -0
  180. data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512_spr.cpp +435 -0
  181. data/vendor/faiss/faiss/utils/simd_impl/rabitq_neon.cpp +11 -0
  182. data/vendor/faiss/faiss/utils/simd_impl/rabitq_rvv.cpp +143 -6
  183. data/vendor/faiss/faiss/utils/simd_levels.cpp +56 -2
  184. data/vendor/faiss/faiss/utils/simd_levels.h +14 -0
  185. data/vendor/faiss/faiss/utils/utils.cpp +9 -27
  186. metadata +27 -2
@@ -30,11 +30,13 @@
30
30
  #include <faiss/Index2Layer.h>
31
31
  #include <faiss/IndexAdditiveQuantizer.h>
32
32
  #include <faiss/IndexAdditiveQuantizerFastScan.h>
33
+ #include <faiss/IndexEDEN.h>
33
34
  #include <faiss/IndexFlat.h>
34
35
  #include <faiss/IndexHNSW.h>
35
36
  #include <faiss/IndexIVF.h>
36
37
  #include <faiss/IndexIVFAdditiveQuantizer.h>
37
38
  #include <faiss/IndexIVFAdditiveQuantizerFastScan.h>
39
+ #include <faiss/IndexIVFEDEN.h>
38
40
  #include <faiss/IndexIVFFlat.h>
39
41
  #include <faiss/IndexIVFFlatPanorama.h>
40
42
  #include <faiss/IndexIVFIndependentQuantizer.h>
@@ -68,6 +70,7 @@
68
70
  #include <faiss/IndexScalarQuantizer.h>
69
71
  #include <faiss/MetaIndexes.h>
70
72
  #include <faiss/VectorTransform.h>
73
+ #include <faiss/impl/EDENQuantizer.h>
71
74
 
72
75
  #include <faiss/IndexBinaryFlat.h>
73
76
  #include <faiss/IndexBinaryFromFloat.h>
@@ -86,6 +89,7 @@ namespace faiss {
86
89
  namespace {
87
90
  size_t deserialization_loop_limit_ = 0;
88
91
  size_t deserialization_vector_byte_limit_ = uint64_t{1} << 40; // 1 TB
92
+ size_t deserialization_lattice_r2_limit_ = 0;
89
93
 
90
94
  #ifdef FAISS_ENABLE_SVS
91
95
  // Read and validate an SVSStorageKind from the stream. Centralizes the
@@ -122,6 +126,14 @@ void set_deserialization_vector_byte_limit(size_t value) {
122
126
  deserialization_vector_byte_limit_ = value;
123
127
  }
124
128
 
129
+ size_t get_deserialization_lattice_r2_limit() {
130
+ return deserialization_lattice_r2_limit_;
131
+ }
132
+
133
+ void set_deserialization_lattice_r2_limit(size_t value) {
134
+ deserialization_lattice_r2_limit_ = value;
135
+ }
136
+
125
137
  #define FAISS_CHECK_DESERIALIZATION_LOOP_LIMIT(val, field_name) \
126
138
  do { \
127
139
  auto limit_ = get_deserialization_loop_limit(); \
@@ -279,7 +291,7 @@ static void read_index_header(Index& idx, IOReader* f) {
279
291
  idx_t dummy;
280
292
  READ1(dummy);
281
293
  READ1(dummy);
282
- READ1(idx.is_trained);
294
+ READ1_BOOL(idx.is_trained);
283
295
  int metric_type_int;
284
296
  READ1(metric_type_int);
285
297
  idx.metric_type = metric_type_from_int(metric_type_int);
@@ -307,7 +319,7 @@ std::unique_ptr<VectorTransform> read_VectorTransform_up(IOReader* f) {
307
319
  if (h == fourcc("Pcam")) {
308
320
  READ1(pca->epsilon);
309
321
  }
310
- READ1(pca->random_rotation);
322
+ READ1_BOOL(pca->random_rotation);
311
323
  if (h != fourcc("PCAm")) {
312
324
  READ1(pca->balanced_bins);
313
325
  }
@@ -323,12 +335,14 @@ std::unique_ptr<VectorTransform> read_VectorTransform_up(IOReader* f) {
323
335
  } else if (h == fourcc("LTra")) {
324
336
  lt = std::make_unique<LinearTransform>();
325
337
  }
326
- READ1(lt->have_bias);
338
+ READ1_BOOL(lt->have_bias);
327
339
  READVECTOR(lt->A);
328
340
  READVECTOR(lt->b);
329
341
  FAISS_THROW_IF_NOT(
330
342
  lt->A.size() >= size_t(lt->d_in) * size_t(lt->d_out));
331
- FAISS_THROW_IF_NOT(!lt->have_bias || lt->b.size() >= size_t(lt->d_out));
343
+ FAISS_THROW_IF_MSG(
344
+ lt->have_bias && lt->b.size() < size_t(lt->d_out),
345
+ "bias vector smaller than d_out");
332
346
  lt->set_is_orthonormal();
333
347
  vt = std::move(lt);
334
348
  } else if (h == fourcc("RmDT")) {
@@ -347,7 +361,7 @@ std::unique_ptr<VectorTransform> read_VectorTransform_up(IOReader* f) {
347
361
  auto itqt = std::make_unique<ITQTransform>();
348
362
 
349
363
  READVECTOR(itqt->mean);
350
- READ1(itqt->do_pca);
364
+ READ1_BOOL(itqt->do_pca);
351
365
  {
352
366
  // Read, dereference, discard.
353
367
  auto sub_vt = read_VectorTransform_up(f);
@@ -376,7 +390,7 @@ std::unique_ptr<VectorTransform> read_VectorTransform_up(IOReader* f) {
376
390
  }
377
391
  READ1(vt->d_in);
378
392
  READ1(vt->d_out);
379
- READ1(vt->is_trained);
393
+ READ1_BOOL(vt->is_trained);
380
394
  FAISS_THROW_IF_NOT_FMT(
381
395
  vt->d_in >= 0,
382
396
  "invalid VectorTransform d_in=%d (must be >= 0)",
@@ -723,7 +737,9 @@ void read_ProductQuantizer(ProductQuantizer* pq, IOReader* f) {
723
737
  FAISS_THROW_IF_NOT_FMT(
724
738
  pq->M > 0, "invalid ProductQuantizer M=%zd (must be > 0)", pq->M);
725
739
  FAISS_THROW_IF_NOT_FMT(
726
- pq->nbits <= 24, "invalid ProductQuantizer nbits=%zd", pq->nbits);
740
+ pq->nbits >= 1 && pq->nbits <= 24,
741
+ "invalid ProductQuantizer nbits=%zd (must be in [1, 24])",
742
+ pq->nbits);
727
743
  {
728
744
  size_t ksub = size_t{1} << pq->nbits;
729
745
  size_t n = mul_no_overflow(pq->d, ksub, "PQ centroids");
@@ -762,7 +778,7 @@ static void read_ResidualQuantizer_old(ResidualQuantizer& rq, IOReader* f) {
762
778
  "ResidualQuantizer nbits size %zd != M %zd",
763
779
  rq.nbits.size(),
764
780
  rq.M);
765
- READ1(rq.is_trained);
781
+ READ1_BOOL(rq.is_trained);
766
782
  READ1(rq.train_type);
767
783
  READ1(rq.max_beam_size);
768
784
  READVECTOR(rq.codebooks);
@@ -780,7 +796,7 @@ static void read_AdditiveQuantizer(AdditiveQuantizer& aq, IOReader* f) {
780
796
  FAISS_THROW_IF_NOT_FMT(
781
797
  aq.M > 0, "invalid AdditiveQuantizer M %zd, must be > 0", aq.M);
782
798
  READVECTOR(aq.nbits);
783
- READ1(aq.is_trained);
799
+ READ1_BOOL(aq.is_trained);
784
800
  READVECTOR(aq.codebooks);
785
801
  FAISS_THROW_IF_NOT_FMT(
786
802
  aq.nbits.size() == aq.M,
@@ -853,6 +869,11 @@ static void validate_fastscan_fields(
853
869
  index_type,
854
870
  M,
855
871
  ksub);
872
+ FAISS_THROW_IF_NOT_FMT(
873
+ ksub == 16,
874
+ "%s: invalid ksub=%zd (fast-scan requires nbits=4 / ksub=16)",
875
+ index_type,
876
+ ksub);
856
877
  FAISS_THROW_IF_NOT_FMT(
857
878
  bbs > 0 && bbs % 32 == 0,
858
879
  "%s: invalid bbs=%d (must be > 0 and a multiple of 32)",
@@ -1035,6 +1056,8 @@ void read_ScalarQuantizer(
1035
1056
  ivsc->d,
1036
1057
  idx.d);
1037
1058
  READVECTOR(ivsc->trained);
1059
+ // Populate bits/code_size before the validation block uses ivsc->bits.
1060
+ ivsc->set_derived_sizes();
1038
1061
  // Validate trained vector size matches the quantizer type and dimension.
1039
1062
  // UNIFORM/NON_UNIFORM qtypes require training data; other qtypes
1040
1063
  // (fp16, bf16, 8bit_direct*) need none.
@@ -1075,6 +1098,40 @@ void read_ScalarQuantizer(
1075
1098
  case ScalarQuantizer::QT_8bit_tqmse:
1076
1099
  expected = 256 + 255;
1077
1100
  break;
1101
+ case ScalarQuantizer::QT_1bit_eden:
1102
+ expected = 2 + 1; // 2^bits centroids + (2^bits - 1) boundaries
1103
+ break;
1104
+ case ScalarQuantizer::QT_2bit_eden:
1105
+ expected = 4 + 3;
1106
+ break;
1107
+ case ScalarQuantizer::QT_3bit_eden:
1108
+ expected = 8 + 7;
1109
+ break;
1110
+ case ScalarQuantizer::QT_4bit_eden:
1111
+ expected = 16 + 15;
1112
+ break;
1113
+ case ScalarQuantizer::QT_5bit_eden:
1114
+ expected = 32 + 31;
1115
+ break;
1116
+ case ScalarQuantizer::QT_6bit_eden:
1117
+ expected = 64 + 63;
1118
+ break;
1119
+ case ScalarQuantizer::QT_7bit_eden:
1120
+ expected = 128 + 127;
1121
+ break;
1122
+ case ScalarQuantizer::QT_8bit_eden:
1123
+ expected = 256 + 255;
1124
+ break;
1125
+ case ScalarQuantizer::QT_2bit_tq:
1126
+ case ScalarQuantizer::QT_3bit_tq:
1127
+ case ScalarQuantizer::QT_4bit_tq:
1128
+ case ScalarQuantizer::QT_5bit_tq: {
1129
+ // k centroids + (k-1) boundaries + 3 extra (seed + qjl_type)
1130
+ size_t mse_bits = ivsc->bits - 1;
1131
+ size_t k = size_t(1) << mse_bits;
1132
+ expected = k + (k - 1) + 3;
1133
+ break;
1134
+ }
1078
1135
  }
1079
1136
  if (ivsc->trained.empty() && expected > 0) {
1080
1137
  // Empty trained is only valid for untrained indices.
@@ -1102,7 +1159,19 @@ void read_ScalarQuantizer(
1102
1159
  }
1103
1160
  }
1104
1161
  }
1105
- ivsc->set_derived_sizes();
1162
+
1163
+ // TurboQ full types: extract seed and qjl_type from trained,
1164
+ // regenerate projection matrix.
1165
+ if (ScalarQuantizer::TurboQuantRefine::is_turboq_full(ivsc->qtype) &&
1166
+ ivsc->trained.size() >= 3) {
1167
+ size_t n = ivsc->trained.size();
1168
+ ivsc->turboq_refine.qjl_type =
1169
+ static_cast<uint8_t>(ivsc->trained[n - 1]);
1170
+ ivsc->turboq_refine.seed =
1171
+ ScalarQuantizer::TurboQuantRefine::unpack_seed(
1172
+ ivsc->trained[n - 3], ivsc->trained[n - 2]);
1173
+ ivsc->turboq_refine.init_projection(ivsc->d);
1174
+ }
1106
1175
  }
1107
1176
 
1108
1177
  static void validate_HNSW(const HNSW& hnsw) {
@@ -1246,7 +1315,7 @@ static void read_NSG(NSG& nsg, IOReader* f) {
1246
1315
  READ1(nsg.C);
1247
1316
  READ1(nsg.search_L);
1248
1317
  READ1(nsg.enterpoint);
1249
- READ1(nsg.is_built);
1318
+ READ1_BOOL(nsg.is_built);
1250
1319
 
1251
1320
  FAISS_THROW_IF_NOT_FMT(
1252
1321
  nsg.ntotal >= 0, "invalid NSG ntotal %d", nsg.ntotal);
@@ -1298,7 +1367,7 @@ static void read_NNDescent(NNDescent& nnd, IOReader* f) {
1298
1367
  READ1(nnd.iter);
1299
1368
  READ1(nnd.search_L);
1300
1369
  READ1(nnd.random_seed);
1301
- READ1(nnd.has_built);
1370
+ READ1_BOOL(nnd.has_built);
1302
1371
 
1303
1372
  FAISS_THROW_IF_NOT_FMT(
1304
1373
  nnd.ntotal >= 0, "invalid NNDescent ntotal %d", nnd.ntotal);
@@ -1306,13 +1375,15 @@ static void read_NNDescent(NNDescent& nnd, IOReader* f) {
1306
1375
  READVECTOR(nnd.final_graph);
1307
1376
  // Validate neighbor IDs in the graph
1308
1377
  if (nnd.has_built && nnd.K > 0 && nnd.ntotal > 0) {
1378
+ const size_t expected_final_graph_size = mul_no_overflow(
1379
+ (size_t)nnd.ntotal, (size_t)nnd.K, "NNDescent final_graph");
1309
1380
  FAISS_THROW_IF_NOT_FMT(
1310
- nnd.final_graph.size() == (size_t)nnd.ntotal * (size_t)nnd.K,
1381
+ nnd.final_graph.size() == expected_final_graph_size,
1311
1382
  "NNDescent final_graph size %zu != ntotal * K (%d * %d = %zu)",
1312
1383
  nnd.final_graph.size(),
1313
1384
  nnd.ntotal,
1314
1385
  nnd.K,
1315
- (size_t)nnd.ntotal * (size_t)nnd.K);
1386
+ expected_final_graph_size);
1316
1387
  for (size_t i = 0; i < nnd.final_graph.size(); i++) {
1317
1388
  int id = nnd.final_graph[i];
1318
1389
  FAISS_THROW_IF_NOT_FMT(
@@ -1369,6 +1440,64 @@ static void read_RaBitQuantizer(
1369
1440
  expected_d);
1370
1441
  }
1371
1442
 
1443
+ static void read_EDENScalarQuantizer(
1444
+ ScalarQuantizer& sq,
1445
+ EDENScaleType& scale_type,
1446
+ IOReader* f,
1447
+ int expected_d,
1448
+ MetricType expected_metric_type,
1449
+ bool read_scale_type) {
1450
+ size_t d;
1451
+ size_t stored_code_size;
1452
+ size_t nb_bits;
1453
+
1454
+ READ1(d);
1455
+ READ1(stored_code_size);
1456
+ int metric_type_int;
1457
+ READ1(metric_type_int);
1458
+ const MetricType metric_type = metric_type_from_int(metric_type_int);
1459
+ READ1(nb_bits);
1460
+ if (read_scale_type) {
1461
+ int scale_type_int;
1462
+ READ1(scale_type_int);
1463
+ if (scale_type_int == 0) {
1464
+ scale_type = EDENScaleType_UNBIASED;
1465
+ } else {
1466
+ scale_type = static_cast<EDENScaleType>(scale_type_int);
1467
+ }
1468
+ } else {
1469
+ scale_type = EDENScaleType_UNBIASED;
1470
+ }
1471
+
1472
+ FAISS_THROW_IF_NOT_FMT(
1473
+ d == static_cast<size_t>(expected_d),
1474
+ "EDEN ScalarQuantizer dimension mismatch: sq.d=%zu vs index d=%d",
1475
+ d,
1476
+ expected_d);
1477
+ FAISS_THROW_IF_NOT_FMT(
1478
+ metric_type == expected_metric_type,
1479
+ "EDEN ScalarQuantizer metric mismatch: stored=%d vs index=%d",
1480
+ metric_type_int,
1481
+ static_cast<int>(expected_metric_type));
1482
+ FAISS_THROW_IF_NOT_FMT(
1483
+ nb_bits >= 1 && nb_bits <= 8,
1484
+ "invalid EDEN nb_bits=%zu (must be in [1, 8])",
1485
+ nb_bits);
1486
+ FAISS_THROW_IF_NOT_FMT(
1487
+ scale_type == EDENScaleType_UNBIASED ||
1488
+ scale_type == EDENScaleType_BIASED,
1489
+ "invalid EDEN scale_type=%d",
1490
+ static_cast<int>(scale_type));
1491
+ sq = ScalarQuantizer(d, eden_utils::quantizer_type_for_bits(nb_bits));
1492
+ sq.train(0, nullptr);
1493
+ const size_t expected_code_size = eden_utils::code_size(d, nb_bits);
1494
+ FAISS_THROW_IF_NOT_FMT(
1495
+ stored_code_size == expected_code_size,
1496
+ "EDEN ScalarQuantizer code_size mismatch: stored=%zu vs expected=%zu",
1497
+ stored_code_size,
1498
+ expected_code_size);
1499
+ }
1500
+
1372
1501
  void read_direct_map(DirectMap* dm, IOReader* f) {
1373
1502
  char maintain_direct_map;
1374
1503
  READ1(maintain_direct_map);
@@ -1420,6 +1549,45 @@ ArrayInvertedLists* set_array_invlist(
1420
1549
  return result;
1421
1550
  }
1422
1551
 
1552
+ static void validate_ivfpq_precomputed_table_size(
1553
+ const Index* quantizer,
1554
+ const ProductQuantizer& pq) {
1555
+ // The precomputed table is not stored; precompute_table() rebuilds it on
1556
+ // load at a size derived from attacker-controlled header fields. Bound
1557
+ // every table initialize_IVFPQ_precomputed_table() may allocate.
1558
+ const size_t m_ksub =
1559
+ mul_no_overflow(pq.M, pq.ksub, "IVFPQ precomputed_table");
1560
+ // type 1: nlist (== quantizer->ntotal) * pq.M * pq.ksub.
1561
+ size_t precompute_elems = mul_no_overflow(
1562
+ static_cast<size_t>(quantizer->ntotal),
1563
+ m_ksub,
1564
+ "IVFPQ precomputed_table");
1565
+ // type 2 (MultiIndexQuantizer coarse quantizer): cpq.ksub * pq.M * pq.ksub,
1566
+ // plus a temporary quantizer->d * cpq.ksub centroid table. Both derive from
1567
+ // the coarse PQ's ksub, which is independent of quantizer->ntotal, so the
1568
+ // type-1 bound above does not cover them.
1569
+ if (const auto* miq = dynamic_cast<const MultiIndexQuantizer*>(quantizer)) {
1570
+ const size_t cpq_ksub = miq->pq.ksub;
1571
+ const size_t type2_table =
1572
+ mul_no_overflow(cpq_ksub, m_ksub, "IVFPQ precomputed_table");
1573
+ const size_t type2_centroids = mul_no_overflow(
1574
+ static_cast<size_t>(quantizer->d),
1575
+ cpq_ksub,
1576
+ "IVFPQ precomputed_table");
1577
+ if (type2_table > precompute_elems) {
1578
+ precompute_elems = type2_table;
1579
+ }
1580
+ if (type2_centroids > precompute_elems) {
1581
+ precompute_elems = type2_centroids;
1582
+ }
1583
+ }
1584
+ FAISS_THROW_IF_NOT_MSG(
1585
+ precompute_elems <
1586
+ get_deserialization_vector_byte_limit() / sizeof(float),
1587
+ "IVFPQ precomputed_table allocation would exceed deserialization "
1588
+ "byte limit");
1589
+ }
1590
+
1423
1591
  static std::unique_ptr<IndexIVFPQ> read_ivfpq(
1424
1592
  IOReader* f,
1425
1593
  uint32_t h,
@@ -1437,7 +1605,9 @@ static std::unique_ptr<IndexIVFPQ> read_ivfpq(
1437
1605
 
1438
1606
  std::vector<std::vector<idx_t>> ids;
1439
1607
  read_ivf_header(ivpq.get(), f, legacy ? &ids : nullptr);
1440
- READ1(ivpq->by_residual);
1608
+ FAISS_THROW_IF_NOT_MSG(
1609
+ ivpq->quantizer != nullptr, "IVFPQ coarse quantizer is null");
1610
+ READ1_BOOL(ivpq->by_residual);
1441
1611
  READ1(ivpq->code_size);
1442
1612
  read_ProductQuantizer(&ivpq->pq, f);
1443
1613
 
@@ -1455,6 +1625,8 @@ static std::unique_ptr<IndexIVFPQ> read_ivfpq(
1455
1625
  ivpq->use_precomputed_table = 0;
1456
1626
  if (ivpq->by_residual) {
1457
1627
  if ((io_flags & IO_FLAG_SKIP_PRECOMPUTE_TABLE) == 0) {
1628
+ validate_ivfpq_precomputed_table_size(
1629
+ ivpq->quantizer, ivpq->pq);
1458
1630
  ivpq->precompute_table();
1459
1631
  }
1460
1632
  }
@@ -1483,7 +1655,29 @@ static std::unique_ptr<IndexIVFPQ> read_ivfpq(
1483
1655
 
1484
1656
  int read_old_fmt_hack = 0;
1485
1657
 
1658
+ namespace {
1659
+
1660
+ constexpr int kMaxIndexNestingDepth = 50;
1661
+ thread_local int index_read_nesting_depth = 0;
1662
+
1663
+ struct IndexNestingGuard {
1664
+ IndexNestingGuard() {
1665
+ FAISS_THROW_IF_NOT_FMT(
1666
+ index_read_nesting_depth < kMaxIndexNestingDepth,
1667
+ "faiss index nesting depth exceeds limit of %d; "
1668
+ "input may be corrupt or malicious",
1669
+ kMaxIndexNestingDepth);
1670
+ ++index_read_nesting_depth;
1671
+ }
1672
+ ~IndexNestingGuard() {
1673
+ --index_read_nesting_depth;
1674
+ }
1675
+ };
1676
+
1677
+ } // namespace
1678
+
1486
1679
  std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1680
+ IndexNestingGuard nesting_guard;
1487
1681
  std::unique_ptr<Index> idx;
1488
1682
  uint32_t h;
1489
1683
  READ1(h);
@@ -1497,6 +1691,8 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1497
1691
  READ1(n_levels);
1498
1692
  FAISS_THROW_IF_NOT_FMT(n_levels > 0, "invalid n_levels %zd", n_levels);
1499
1693
  READ1(batch_size);
1694
+ FAISS_THROW_IF_NOT_FMT(
1695
+ batch_size > 0, "invalid IxFP batch_size %zd", batch_size);
1500
1696
  std::unique_ptr<IndexFlatPanorama> idxp;
1501
1697
  if (h == fourcc("IxFP")) {
1502
1698
  idxp = std::make_unique<IndexFlatL2Panorama>(
@@ -1506,9 +1702,24 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1506
1702
  d, n_levels, batch_size);
1507
1703
  }
1508
1704
  READ1(idxp->ntotal);
1509
- READ1(idxp->is_trained);
1705
+ READ1_BOOL(idxp->is_trained);
1510
1706
  READVECTOR(idxp->codes);
1511
1707
  READVECTOR(idxp->cum_sums);
1708
+ size_t num_slots = mul_no_overflow(
1709
+ ((size_t)idxp->ntotal + idxp->batch_size - 1) /
1710
+ idxp->batch_size,
1711
+ idxp->batch_size,
1712
+ "IndexFlatPanorama num_batches*batch_size");
1713
+ FAISS_THROW_IF_NOT(
1714
+ idxp->codes.size() ==
1715
+ mul_no_overflow(
1716
+ num_slots, idxp->code_size, "IndexFlatPanorama codes"));
1717
+ FAISS_THROW_IF_NOT(
1718
+ idxp->cum_sums.size() ==
1719
+ mul_no_overflow(
1720
+ num_slots,
1721
+ idxp->pano.n_levels + 1,
1722
+ "IndexFlatPanorama cum_sums"));
1512
1723
  idxp->verbose = false;
1513
1724
  idx = std::move(idxp);
1514
1725
  } else if (
@@ -1525,14 +1736,18 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1525
1736
  idxf->code_size = idxf->d * sizeof(float);
1526
1737
  read_xb_vector(idxf->codes, f);
1527
1738
  FAISS_THROW_IF_NOT(
1528
- idxf->codes.size() == idxf->ntotal * idxf->code_size);
1739
+ idxf->codes.size() ==
1740
+ mul_no_overflow(
1741
+ (size_t)idxf->ntotal,
1742
+ idxf->code_size,
1743
+ "IndexFlat codes"));
1529
1744
  idx = std::move(idxf);
1530
1745
  } else if (h == fourcc("IxHE") || h == fourcc("IxHe")) {
1531
1746
  auto idxl = std::make_unique<IndexLSH>();
1532
1747
  read_index_header(*idxl, f);
1533
1748
  READ1(idxl->nbits);
1534
- READ1(idxl->rotate_data);
1535
- READ1(idxl->train_thresholds);
1749
+ READ1_BOOL(idxl->rotate_data);
1750
+ READ1_BOOL(idxl->train_thresholds);
1536
1751
  READVECTOR(idxl->thresholds);
1537
1752
  int code_size_i;
1538
1753
  READ1(code_size_i);
@@ -1564,7 +1779,11 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1564
1779
  FAISS_THROW_IF_NOT(
1565
1780
  idxl->rrot.d_in == idxl->d && idxl->rrot.d_out == idxl->nbits);
1566
1781
  FAISS_THROW_IF_NOT(
1567
- idxl->codes.size() == idxl->ntotal * idxl->code_size);
1782
+ idxl->codes.size() ==
1783
+ mul_no_overflow(
1784
+ (size_t)idxl->ntotal,
1785
+ idxl->code_size,
1786
+ "IndexLSH codes"));
1568
1787
  idx = std::move(idxl);
1569
1788
  } else if (
1570
1789
  h == fourcc("IxPQ") || h == fourcc("IxPo") || h == fourcc("IxPq")) {
@@ -1574,11 +1793,19 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1574
1793
  read_ProductQuantizer(&idxp->pq, f);
1575
1794
  idxp->code_size = idxp->pq.code_size;
1576
1795
  read_vector(idxp->codes, f);
1796
+ FAISS_THROW_IF_NOT_MSG(
1797
+ idxp->code_size > 0 || idxp->ntotal == 0,
1798
+ "IndexPQ with ntotal > 0 must have code_size > 0 "
1799
+ "(corrupt ProductQuantizer nbits?)");
1577
1800
  FAISS_THROW_IF_NOT(
1578
- idxp->codes.size() == idxp->ntotal * idxp->code_size);
1801
+ idxp->codes.size() ==
1802
+ mul_no_overflow(
1803
+ (size_t)idxp->ntotal,
1804
+ idxp->code_size,
1805
+ "IndexPQ codes"));
1579
1806
  if (h == fourcc("IxPo") || h == fourcc("IxPq")) {
1580
1807
  READ1(idxp->search_type);
1581
- READ1(idxp->encode_signs);
1808
+ READ1_BOOL(idxp->encode_signs);
1582
1809
  READ1(idxp->polysemous_ht);
1583
1810
  }
1584
1811
  // Old versions of PQ all had metric_type set to INNER_PRODUCT
@@ -1603,7 +1830,11 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1603
1830
  idxr->code_size, idxr->rq.code_size, "IndexResidualQuantizer");
1604
1831
  read_vector(idxr->codes, f);
1605
1832
  FAISS_THROW_IF_NOT(
1606
- idxr->codes.size() == idxr->ntotal * idxr->code_size);
1833
+ idxr->codes.size() ==
1834
+ mul_no_overflow(
1835
+ (size_t)idxr->ntotal,
1836
+ idxr->code_size,
1837
+ "IndexResidualQuantizer codes"));
1607
1838
  idx = std::move(idxr);
1608
1839
  } else if (h == fourcc("IxLS")) {
1609
1840
  auto idxr = std::make_unique<IndexLocalSearchQuantizer>();
@@ -1618,7 +1849,11 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1618
1849
  "IndexLocalSearchQuantizer");
1619
1850
  read_vector(idxr->codes, f);
1620
1851
  FAISS_THROW_IF_NOT(
1621
- idxr->codes.size() == idxr->ntotal * idxr->code_size);
1852
+ idxr->codes.size() ==
1853
+ mul_no_overflow(
1854
+ (size_t)idxr->ntotal,
1855
+ idxr->code_size,
1856
+ "IndexLocalSearchQuantizer codes"));
1622
1857
  idx = std::move(idxr);
1623
1858
  } else if (h == fourcc("IxPR")) {
1624
1859
  auto idxpr = std::make_unique<IndexProductResidualQuantizer>();
@@ -1633,7 +1868,11 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1633
1868
  "IndexProductResidualQuantizer");
1634
1869
  read_vector(idxpr->codes, f);
1635
1870
  FAISS_THROW_IF_NOT(
1636
- idxpr->codes.size() == idxpr->ntotal * idxpr->code_size);
1871
+ idxpr->codes.size() ==
1872
+ mul_no_overflow(
1873
+ (size_t)idxpr->ntotal,
1874
+ idxpr->code_size,
1875
+ "IndexProductResidualQuantizer codes"));
1637
1876
  idx = std::move(idxpr);
1638
1877
  } else if (h == fourcc("IxPL")) {
1639
1878
  auto idxpl = std::make_unique<IndexProductLocalSearchQuantizer>();
@@ -1648,7 +1887,11 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1648
1887
  "IndexProductLocalSearchQuantizer");
1649
1888
  read_vector(idxpl->codes, f);
1650
1889
  FAISS_THROW_IF_NOT(
1651
- idxpl->codes.size() == idxpl->ntotal * idxpl->code_size);
1890
+ idxpl->codes.size() ==
1891
+ mul_no_overflow(
1892
+ (size_t)idxpl->ntotal,
1893
+ idxpl->code_size,
1894
+ "IndexProductLocalSearchQuantizer codes"));
1652
1895
  idx = std::move(idxpl);
1653
1896
  } else if (h == fourcc("ImRQ")) {
1654
1897
  auto idxr = std::make_unique<ResidualCoarseQuantizer>();
@@ -1692,6 +1935,16 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1692
1935
  idxr->ntotal,
1693
1936
  idxr->rq.M);
1694
1937
  }
1938
+ FAISS_THROW_IF_NOT_MSG(
1939
+ idxr->rq.tot_bits <= 63,
1940
+ "ResidualCoarseQuantizer tot_bits too large (max 63)");
1941
+ FAISS_THROW_IF_NOT_FMT(
1942
+ static_cast<size_t>(idxr->ntotal) ==
1943
+ (((size_t)1) << idxr->rq.tot_bits),
1944
+ "ResidualCoarseQuantizer ntotal %" PRId64
1945
+ " inconsistent with 2^tot_bits (tot_bits=%zu)",
1946
+ idxr->ntotal,
1947
+ idxr->rq.tot_bits);
1695
1948
  idxr->set_beam_factor(idxr->beam_factor);
1696
1949
  idx = std::move(idxr);
1697
1950
  } else if (
@@ -1741,7 +1994,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1741
1994
  READ1(idxaqfs->ntotal2);
1742
1995
  READ1(idxaqfs->M2);
1743
1996
 
1744
- READ1(idxaqfs->rescale_norm);
1997
+ READ1_BOOL(idxaqfs->rescale_norm);
1745
1998
  READ1(idxaqfs->norm_scale);
1746
1999
  READ1(idxaqfs->max_train_points);
1747
2000
 
@@ -1791,7 +2044,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1791
2044
  validate_aq_dimension_match(
1792
2045
  *ivaqfs->aq, ivaqfs->d, "IndexIVFAdditiveQuantizerFastScan");
1793
2046
 
1794
- READ1(ivaqfs->by_residual);
2047
+ READ1_BOOL(ivaqfs->by_residual);
1795
2048
  READ1(ivaqfs->implem);
1796
2049
  READ1(ivaqfs->bbs);
1797
2050
  READ1(ivaqfs->qbs);
@@ -1804,7 +2057,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1804
2057
  READ1(ivaqfs->qbs2);
1805
2058
  READ1(ivaqfs->M2);
1806
2059
 
1807
- READ1(ivaqfs->rescale_norm);
2060
+ READ1_BOOL(ivaqfs->rescale_norm);
1808
2061
  READ1(ivaqfs->norm_scale);
1809
2062
  READ1(ivaqfs->max_train_points);
1810
2063
 
@@ -1826,9 +2079,28 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1826
2079
  ivfl->code_size = ivfl->d * sizeof(float);
1827
2080
  ArrayInvertedLists* ail = set_array_invlist(ivfl.get(), ids);
1828
2081
 
2082
+ // Legacy IVF serialized ids and codes as separate vectors.
2083
+ // Check not required in default IVF, due to single sizes vector.
2084
+ auto validate_legacy_codes_size = [&](size_t i) {
2085
+ const size_t expected_codes_bytes = mul_no_overflow(
2086
+ ail->ids[i].size(),
2087
+ ivfl->code_size,
2088
+ "legacy IVFFlat inverted list codes");
2089
+ FAISS_THROW_IF_NOT_FMT(
2090
+ ail->codes[i].size() == expected_codes_bytes,
2091
+ "Legacy IVFFlat inverted list %zu: codes size %zu bytes "
2092
+ "does not match ids size %zu * code_size %zu = %zu bytes",
2093
+ i,
2094
+ ail->codes[i].size(),
2095
+ ail->ids[i].size(),
2096
+ (size_t)ivfl->code_size,
2097
+ expected_codes_bytes);
2098
+ };
2099
+
1829
2100
  if (h == fourcc("IvFL")) {
1830
2101
  for (size_t i = 0; i < ivfl->nlist; i++) {
1831
2102
  READVECTOR(ail->codes[i]);
2103
+ validate_legacy_codes_size(i);
1832
2104
  }
1833
2105
  } else { // old format
1834
2106
  for (size_t i = 0; i < ivfl->nlist; i++) {
@@ -1836,6 +2108,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1836
2108
  READVECTOR(vec);
1837
2109
  ail->codes[i].resize(vec.size() * sizeof(float));
1838
2110
  memcpy(ail->codes[i].data(), vec.data(), ail->codes[i].size());
2111
+ validate_legacy_codes_size(i);
1839
2112
  }
1840
2113
  }
1841
2114
  idx = std::move(ivfl);
@@ -1872,6 +2145,10 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1872
2145
  ivfp->code_size = ivfp->d * sizeof(float);
1873
2146
  READ1(ivfp->n_levels);
1874
2147
  READ1(ivfp->batch_size);
2148
+ FAISS_THROW_IF_NOT_FMT(
2149
+ ivfp->batch_size > 0,
2150
+ "invalid IwP2 batch_size %zd",
2151
+ ivfp->batch_size);
1875
2152
  read_InvertedLists(*ivfp, f, io_flags);
1876
2153
  idx = std::move(ivfp);
1877
2154
  } else if (h == fourcc("IwFl")) {
@@ -1902,6 +2179,24 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1902
2179
  nsq);
1903
2180
  FAISS_THROW_IF_NOT_FMT(
1904
2181
  r2 > 0, "invalid IndexLattice r2 %d (must be > 0)", r2);
2182
+ {
2183
+ // ZnSphereCodecRec constructor populates a decode cache
2184
+ // whose build cost grows polynomially in r2. The
2185
+ // in-codec memory cap (lattice_Zn.cpp) bounds the cache
2186
+ // size but not the CPU cost of building it, so for small
2187
+ // dsq the cap permits enough decode() iterations to far
2188
+ // exceed reasonable load-time budgets. Callers that
2189
+ // operate on untrusted index payloads can opt in to a
2190
+ // tighter bound via set_deserialization_lattice_r2_limit;
2191
+ // the default of 0 preserves existing behavior.
2192
+ auto limit_ = get_deserialization_lattice_r2_limit();
2193
+ FAISS_THROW_IF_NOT_FMT(
2194
+ limit_ == 0 || static_cast<size_t>(r2) <= limit_,
2195
+ "IndexLattice r2=%d exceeds "
2196
+ "deserialization_lattice_r2_limit of %zd",
2197
+ r2,
2198
+ limit_);
2199
+ }
1905
2200
  int dsq = d / nsq;
1906
2201
  FAISS_THROW_IF_NOT_FMT(
1907
2202
  dsq >= 2 && (dsq & (dsq - 1)) == 0,
@@ -1932,6 +2227,16 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1932
2227
  r2);
1933
2228
  }
1934
2229
  read_index_header(*idxl, f);
2230
+ FAISS_THROW_IF_NOT_FMT(
2231
+ idxl->ntotal == 0,
2232
+ "IndexLattice deserialization carries no code storage; "
2233
+ "ntotal=%zd != 0 is corrupt",
2234
+ (size_t)idxl->ntotal);
2235
+ FAISS_THROW_IF_NOT_FMT(
2236
+ idxl->d == d,
2237
+ "IndexLattice header d=%d inconsistent with encoded d=%d",
2238
+ idxl->d,
2239
+ d);
1935
2240
  READVECTOR(idxl->trained);
1936
2241
  idx = std::move(idxl);
1937
2242
  } else if (h == fourcc("IvSQ")) { // legacy
@@ -1956,7 +2261,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1956
2261
  if (h == fourcc("IwSQ")) {
1957
2262
  ivsc->by_residual = true;
1958
2263
  } else {
1959
- READ1(ivsc->by_residual);
2264
+ READ1_BOOL(ivsc->by_residual);
1960
2265
  }
1961
2266
  read_InvertedLists(*ivsc, f, io_flags);
1962
2267
  idx = std::move(ivsc);
@@ -1995,7 +2300,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1995
2300
  iva->code_size,
1996
2301
  iva->aq->code_size,
1997
2302
  "IndexIVFAdditiveQuantizer");
1998
- READ1(iva->by_residual);
2303
+ READ1_BOOL(iva->by_residual);
1999
2304
  READ1(iva->use_precomputed_table);
2000
2305
  read_InvertedLists(*iva, f, io_flags);
2001
2306
  idx = std::move(iva);
@@ -2022,7 +2327,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2022
2327
  read_index_header(*indep, f);
2023
2328
  indep->quantizer = read_index(f, io_flags);
2024
2329
  bool has_vt;
2025
- READ1(has_vt);
2330
+ READ1_BOOL(has_vt);
2026
2331
  if (has_vt) {
2027
2332
  indep->vt = read_VectorTransform(f);
2028
2333
  }
@@ -2107,6 +2412,8 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2107
2412
  read_index_header(*idxrf, f);
2108
2413
  auto base = read_index_up(f, io_flags);
2109
2414
  auto refine = read_index_up(f, io_flags);
2415
+ FAISS_THROW_IF_NOT_MSG(base, "IndexRefine base index is null");
2416
+ FAISS_THROW_IF_NOT_MSG(refine, "IndexRefine refine index is null");
2110
2417
  READ1(idxrf->k_factor);
2111
2418
  // Same rationale as IndexIVFPQR k_factor above.
2112
2419
  FAISS_THROW_IF_NOT_FMT(
@@ -2137,6 +2444,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2137
2444
  : std::make_unique<IndexIDMap>();
2138
2445
  read_index_header(*idxmap, f);
2139
2446
  idxmap->index = read_index(f, io_flags);
2447
+ FAISS_THROW_IF_NOT_MSG(idxmap->index, "IndexIDMap inner index is null");
2140
2448
  idxmap->own_fields = true;
2141
2449
  READVECTOR(idxmap->id_map);
2142
2450
  FAISS_THROW_IF_NOT_FMT(
@@ -2166,6 +2474,10 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2166
2474
  READ1(idxp->code_size_1);
2167
2475
  READ1(idxp->code_size_2);
2168
2476
  READ1(idxp->code_size);
2477
+ validate_code_size_match(
2478
+ idxp->code_size_1,
2479
+ idxp->q1.coarse_code_size(),
2480
+ "Index2Layer code_size_1");
2169
2481
  validate_code_size_match(
2170
2482
  idxp->code_size_2,
2171
2483
  idxp->pq.code_size,
@@ -2175,31 +2487,33 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2175
2487
  idxp->code_size_1 + idxp->code_size_2,
2176
2488
  "Index2Layer");
2177
2489
  read_vector(idxp->codes, f);
2490
+ FAISS_THROW_IF_NOT(
2491
+ idxp->codes.size() ==
2492
+ mul_no_overflow(
2493
+ (size_t)idxp->ntotal,
2494
+ idxp->code_size,
2495
+ "Index2Layer codes"));
2178
2496
  idx = std::move(idxp);
2179
2497
  } else if (
2180
2498
  h == fourcc("IHNf") || h == fourcc("IHNp") || h == fourcc("IHNs") ||
2181
2499
  h == fourcc("IHN2") || h == fourcc("IHNc") || h == fourcc("IHc2") ||
2182
- h == fourcc("IHfP")) {
2500
+ h == fourcc("IHfP") || h == fourcc("IH00")) {
2183
2501
  std::unique_ptr<IndexHNSW> idxhnsw;
2184
- if (h == fourcc("IHNf")) {
2502
+ if (h == fourcc("IH00")) {
2503
+ idxhnsw = std::make_unique<IndexHNSW>();
2504
+ } else if (h == fourcc("IHNf")) {
2185
2505
  idxhnsw = std::make_unique<IndexHNSWFlat>();
2186
- }
2187
- if (h == fourcc("IHfP")) {
2506
+ } else if (h == fourcc("IHfP")) {
2188
2507
  idxhnsw = std::make_unique<IndexHNSWFlatPanorama>();
2189
- }
2190
- if (h == fourcc("IHNp")) {
2508
+ } else if (h == fourcc("IHNp")) {
2191
2509
  idxhnsw = std::make_unique<IndexHNSWPQ>();
2192
- }
2193
- if (h == fourcc("IHNs")) {
2510
+ } else if (h == fourcc("IHNs")) {
2194
2511
  idxhnsw = std::make_unique<IndexHNSWSQ>();
2195
- }
2196
- if (h == fourcc("IHN2")) {
2512
+ } else if (h == fourcc("IHN2")) {
2197
2513
  idxhnsw = std::make_unique<IndexHNSW2Level>();
2198
- }
2199
- if (h == fourcc("IHNc")) {
2514
+ } else if (h == fourcc("IHNc")) {
2200
2515
  idxhnsw = std::make_unique<IndexHNSWCagra>();
2201
- }
2202
- if (h == fourcc("IHc2")) {
2516
+ } else if (h == fourcc("IHc2")) {
2203
2517
  idxhnsw = std::make_unique<IndexHNSWCagra>();
2204
2518
  }
2205
2519
  read_index_header(*idxhnsw, f);
@@ -2215,13 +2529,12 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2215
2529
  const_cast<Panorama&>(idx_panorama->pano) =
2216
2530
  Panorama(idx_panorama->d * sizeof(float), nlevels, 1);
2217
2531
  READVECTOR(idx_panorama->cum_sums);
2218
- }
2219
- if (h == fourcc("IHNc") || h == fourcc("IHc2")) {
2220
- READ1(idxhnsw->keep_max_size_level0);
2532
+ } else if (h == fourcc("IHNc") || h == fourcc("IHc2")) {
2533
+ READ1_BOOL(idxhnsw->keep_max_size_level0);
2221
2534
  auto idx_hnsw_cagra = dynamic_cast<IndexHNSWCagra*>(idxhnsw.get());
2222
2535
  FAISS_THROW_IF_NOT_MSG(
2223
2536
  idx_hnsw_cagra, "dynamic_cast to IndexHNSWCagra failed");
2224
- READ1(idx_hnsw_cagra->base_level_only);
2537
+ READ1_BOOL(idx_hnsw_cagra->base_level_only);
2225
2538
  READ1(idx_hnsw_cagra->num_base_level_search_entrypoints);
2226
2539
  if (h == fourcc("IHc2")) {
2227
2540
  READ1(idx_hnsw_cagra->numeric_type_);
@@ -2237,6 +2550,12 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2237
2550
  idxhnsw->hnsw.levels.size(),
2238
2551
  idxhnsw->ntotal);
2239
2552
  idxhnsw->hnsw.is_panorama = (h == fourcc("IHfP"));
2553
+ // `HNSW::is_similarity` is intentionally not serialized, so we
2554
+ // re-derive it here from the persisted metric type. Without this,
2555
+ // a saved IP/similarity index would come back configured as a
2556
+ // distance index and silently produce wrong rankings on search.
2557
+ idxhnsw->hnsw.is_similarity =
2558
+ is_similarity_metric(idxhnsw->metric_type);
2240
2559
  idxhnsw->storage = read_index(f, io_flags);
2241
2560
  idxhnsw->own_fields = idxhnsw->storage != nullptr;
2242
2561
  // Cross-check storage ntotal and d against index
@@ -2372,7 +2691,10 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2372
2691
  } else if (h == fourcc("IwPf")) {
2373
2692
  auto ivpq = std::make_unique<IndexIVFPQFastScan>();
2374
2693
  read_ivf_header(ivpq.get(), f);
2375
- READ1(ivpq->by_residual);
2694
+ FAISS_THROW_IF_NOT_MSG(
2695
+ ivpq->quantizer != nullptr,
2696
+ "IVFPQFastScan coarse quantizer is null");
2697
+ READ1_BOOL(ivpq->by_residual);
2376
2698
  READ1(ivpq->code_size);
2377
2699
  READ1(ivpq->bbs);
2378
2700
  READ1(ivpq->M2);
@@ -2380,6 +2702,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2380
2702
  READ1(ivpq->qbs2);
2381
2703
  read_ProductQuantizer(&ivpq->pq, f);
2382
2704
  read_InvertedLists(*ivpq, f, io_flags);
2705
+ validate_ivfpq_precomputed_table_size(ivpq->quantizer, ivpq->pq);
2383
2706
  ivpq->precompute_table();
2384
2707
 
2385
2708
  const auto& pq = ivpq->pq;
@@ -2409,6 +2732,45 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2409
2732
  imm->own_fields = true;
2410
2733
 
2411
2734
  idx = std::move(imm);
2735
+ } else if (h == fourcc("IxEd") || h == fourcc("IxEe")) {
2736
+ auto idxe = std::make_unique<IndexEDEN>();
2737
+ read_index_header(*idxe, f);
2738
+ read_EDENScalarQuantizer(
2739
+ idxe->sq,
2740
+ idxe->scale_type,
2741
+ f,
2742
+ idxe->d,
2743
+ idxe->metric_type,
2744
+ h == fourcc("IxEe"));
2745
+ READVECTOR(idxe->codes);
2746
+ READVECTOR(idxe->center);
2747
+
2748
+ idxe->code_size = eden_utils::code_size(idxe->d, idxe->sq.bits);
2749
+ FAISS_THROW_IF_NOT(
2750
+ idxe->codes.size() == idxe->ntotal * idxe->code_size);
2751
+ idx = std::move(idxe);
2752
+ } else if (h == fourcc("IwEd") || h == fourcc("IwEe")) {
2753
+ auto iveden = std::make_unique<IndexIVFEDEN>();
2754
+ read_ivf_header(iveden.get(), f);
2755
+ read_EDENScalarQuantizer(
2756
+ iveden->sq,
2757
+ iveden->scale_type,
2758
+ f,
2759
+ iveden->d,
2760
+ iveden->metric_type,
2761
+ h == fourcc("IwEe"));
2762
+ size_t stored_ivf_code_size;
2763
+ READ1(stored_ivf_code_size);
2764
+ READ1(iveden->by_residual);
2765
+
2766
+ iveden->code_size = eden_utils::code_size(iveden->d, iveden->sq.bits);
2767
+ FAISS_THROW_IF_NOT_FMT(
2768
+ stored_ivf_code_size == iveden->code_size,
2769
+ "IndexIVFEDEN code_size mismatch: stored=%zu vs expected=%zu",
2770
+ stored_ivf_code_size,
2771
+ iveden->code_size);
2772
+ read_InvertedLists(*iveden, f, io_flags);
2773
+ idx = std::move(iveden);
2412
2774
  } else if (h == fourcc("Irfn") || h == fourcc("Irfs")) {
2413
2775
  // Irfn = new format (aux data embedded in SIMD blocks)
2414
2776
  // Irfs = legacy format (flat_storage separate, needs migration)
@@ -2512,7 +2874,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2512
2874
  read_ivf_header(ivrq.get(), f);
2513
2875
  read_RaBitQuantizer(ivrq->rabitq, f, ivrq->d, false);
2514
2876
  READ1(ivrq->code_size);
2515
- READ1(ivrq->by_residual);
2877
+ READ1_BOOL(ivrq->by_residual);
2516
2878
  READ1(ivrq->qb);
2517
2879
  // qb=0: Not quantized - direct distance computation on given float32s.
2518
2880
  // qb>0 && qb<=8: Scalar-quantized with qb bits of precision.
@@ -2535,7 +2897,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2535
2897
  read_RaBitQuantizer(
2536
2898
  ivrq->rabitq, f, ivrq->d, true); // Reads nb_bits from file
2537
2899
  READ1(ivrq->code_size);
2538
- READ1(ivrq->by_residual);
2900
+ READ1_BOOL(ivrq->by_residual);
2539
2901
  READ1(ivrq->qb);
2540
2902
  // qb=0: Not quantized - direct distance computation on given float32s.
2541
2903
  // qb>0 && qb<=8: Scalar-quantized with qb bits of precision.
@@ -2572,9 +2934,10 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2572
2934
  READ1(svs->construction_window_size);
2573
2935
  READ1(svs->max_candidate_pool_size);
2574
2936
  READ1(svs->prune_to);
2575
- READ1(svs->use_full_search_history);
2937
+ READ1_BOOL(svs->use_full_search_history);
2576
2938
 
2577
2939
  svs->storage_kind = read_svs_storage_kind(f);
2940
+ READ1_BOOL(svs->is_static);
2578
2941
 
2579
2942
  if (h == fourcc("ISVL")) {
2580
2943
  auto* leanvec = dynamic_cast<IndexSVSVamanaLeanVec*>(svs.get());
@@ -2584,16 +2947,28 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2584
2947
  }
2585
2948
 
2586
2949
  bool initialized;
2587
- READ1(initialized);
2950
+ READ1_BOOL(initialized);
2588
2951
  if (initialized) {
2589
- faiss::svs_io::ReaderStreambuf rbuf(
2590
- f, get_deserialization_vector_byte_limit());
2591
- std::istream is(&rbuf);
2592
- svs->deserialize_impl(is);
2952
+ if ((io_flags & IO_FLAG_MMAP_IFC) == IO_FLAG_MMAP_IFC &&
2953
+ svs->is_static) {
2954
+ // Use memory-mapped I/O for static indices
2955
+ auto* mf = dynamic_cast<MappedFileIOReader*>(f);
2956
+ FAISS_THROW_IF_NOT_MSG(
2957
+ mf,
2958
+ "IO_FLAG_MMAP_IFC flag set but IOReader is not "
2959
+ "MappedFileIOReader");
2960
+ svs->map_to(mf);
2961
+ } else {
2962
+ // Use standard deserialization
2963
+ faiss::svs_io::ReaderStreambuf rbuf(
2964
+ f, get_deserialization_vector_byte_limit());
2965
+ std::istream is(&rbuf);
2966
+ svs->deserialize_impl(is);
2967
+ }
2593
2968
  }
2594
2969
  if (h == fourcc("ISVL")) {
2595
2970
  bool trained;
2596
- READ1(trained);
2971
+ READ1_BOOL(trained);
2597
2972
  if (trained) {
2598
2973
  faiss::svs_io::ReaderStreambuf rbuf(
2599
2974
  f, get_deserialization_vector_byte_limit());
@@ -2607,6 +2982,13 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2607
2982
  }
2608
2983
  if (h == fourcc("ISV2")) {
2609
2984
  READVECTOR(svs->stored_vectors);
2985
+ FAISS_THROW_IF_NOT_MSG(
2986
+ svs->stored_vectors.size() ==
2987
+ mul_no_overflow(
2988
+ (size_t)svs->ntotal,
2989
+ (size_t)svs->d,
2990
+ "IndexSVSVamana stored_vectors"),
2991
+ "ISV2: stored_vectors size inconsistent with ntotal * d");
2610
2992
  } else {
2611
2993
  svs->stored_vectors_valid = false;
2612
2994
  }
@@ -2616,12 +2998,23 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2616
2998
  read_index_header(*svs, f);
2617
2999
 
2618
3000
  bool initialized;
2619
- READ1(initialized);
3001
+ READ1_BOOL(initialized);
2620
3002
  if (initialized) {
2621
- faiss::svs_io::ReaderStreambuf rbuf(
2622
- f, get_deserialization_vector_byte_limit());
2623
- std::istream is(&rbuf);
2624
- svs->deserialize_impl(is);
3003
+ if ((io_flags & IO_FLAG_MMAP_IFC) == IO_FLAG_MMAP_IFC) {
3004
+ // Use memory-mapped I/O
3005
+ auto* mf = dynamic_cast<MappedFileIOReader*>(f);
3006
+ FAISS_THROW_IF_NOT_MSG(
3007
+ mf,
3008
+ "IO_FLAG_MMAP_IFC flag set but IOReader is not "
3009
+ "MappedFileIOReader");
3010
+ svs->map_to(mf);
3011
+ } else {
3012
+ // Use standard deserialization
3013
+ faiss::svs_io::ReaderStreambuf rbuf(
3014
+ f, get_deserialization_vector_byte_limit());
3015
+ std::istream is(&rbuf);
3016
+ svs->deserialize_impl(is);
3017
+ }
2625
3018
  }
2626
3019
  idx = std::move(svs);
2627
3020
  } else if (
@@ -2639,7 +3032,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2639
3032
  READ1(svs_ivf->num_centroids);
2640
3033
  READ1(svs_ivf->minibatch_size);
2641
3034
  READ1(svs_ivf->num_iterations);
2642
- READ1(svs_ivf->is_hierarchical);
3035
+ READ1_BOOL(svs_ivf->is_hierarchical);
2643
3036
  READ1(svs_ivf->training_fraction);
2644
3037
  READ1(svs_ivf->hierarchical_level1_clusters);
2645
3038
  READ1(svs_ivf->seed);
@@ -2648,7 +3041,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2648
3041
  READ1(svs_ivf->num_threads);
2649
3042
  READ1(svs_ivf->intra_query_threads);
2650
3043
  svs_ivf->storage_kind = read_svs_storage_kind(f);
2651
- READ1(svs_ivf->is_static);
3044
+ READ1_BOOL(svs_ivf->is_static);
2652
3045
  if (h == fourcc("ISIL")) {
2653
3046
  auto* leanvec = dynamic_cast<IndexSVSIVFLeanVec*>(svs_ivf.get());
2654
3047
  FAISS_THROW_IF_NOT_MSG(
@@ -2657,7 +3050,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2657
3050
  }
2658
3051
 
2659
3052
  bool initialized;
2660
- READ1(initialized);
3053
+ READ1_BOOL(initialized);
2661
3054
  if (initialized) {
2662
3055
  faiss::svs_io::ReaderStreambuf rbuf(f);
2663
3056
  std::istream is(&rbuf);
@@ -2665,7 +3058,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2665
3058
  }
2666
3059
  if (h == fourcc("ISIL")) {
2667
3060
  bool trained;
2668
- READ1(trained);
3061
+ READ1_BOOL(trained);
2669
3062
  if (trained) {
2670
3063
  faiss::svs_io::ReaderStreambuf rbuf(f);
2671
3064
  std::istream is(&rbuf);
@@ -2687,7 +3080,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2687
3080
  auto ivrqfs = std::make_unique<IndexIVFRaBitQFastScan>();
2688
3081
  read_ivf_header(ivrqfs.get(), f);
2689
3082
  read_RaBitQuantizer(ivrqfs->rabitq, f, ivrqfs->d);
2690
- READ1(ivrqfs->by_residual);
3083
+ READ1_BOOL(ivrqfs->by_residual);
2691
3084
  READ1(ivrqfs->code_size);
2692
3085
  READ1(ivrqfs->bbs);
2693
3086
  READ1(ivrqfs->qbs2);
@@ -2698,7 +3091,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
2698
3091
  ivrqfs->qb > 0 && ivrqfs->qb <= 8,
2699
3092
  "invalid RaBitQ qb=%d (must be in [1, 8])",
2700
3093
  ivrqfs->qb);
2701
- READ1(ivrqfs->centered);
3094
+ READ1_BOOL(ivrqfs->centered);
2702
3095
 
2703
3096
  std::vector<uint8_t> legacy_flat_storage;
2704
3097
  if (is_legacy) {
@@ -2825,7 +3218,7 @@ static void read_index_binary_header(IndexBinary& idx, IOReader* f) {
2825
3218
  READ1(idx.d);
2826
3219
  READ1(idx.code_size);
2827
3220
  READ1(idx.ntotal);
2828
- READ1(idx.is_trained);
3221
+ READ1_BOOL(idx.is_trained);
2829
3222
  int metric_type_int;
2830
3223
  READ1(metric_type_int);
2831
3224
  idx.metric_type = metric_type_from_int(metric_type_int);
@@ -2965,6 +3358,7 @@ static void read_binary_multi_hash_map(
2965
3358
  }
2966
3359
 
2967
3360
  std::unique_ptr<IndexBinary> read_index_binary_up(IOReader* f, int io_flags) {
3361
+ IndexNestingGuard nesting_guard;
2968
3362
  std::unique_ptr<IndexBinary> idx;
2969
3363
  uint32_t h;
2970
3364
  READ1(h);
@@ -2972,7 +3366,12 @@ std::unique_ptr<IndexBinary> read_index_binary_up(IOReader* f, int io_flags) {
2972
3366
  auto idxf = std::make_unique<IndexBinaryFlat>();
2973
3367
  read_index_binary_header(*idxf, f);
2974
3368
  read_vector(idxf->xb, f);
2975
- FAISS_THROW_IF_NOT(idxf->xb.size() == idxf->ntotal * idxf->code_size);
3369
+ FAISS_THROW_IF_NOT(
3370
+ idxf->xb.size() ==
3371
+ mul_no_overflow(
3372
+ (size_t)idxf->ntotal,
3373
+ idxf->code_size,
3374
+ "IndexBinaryFlat xb"));
2976
3375
  idx = std::move(idxf);
2977
3376
  } else if (h == fourcc("IBwF")) {
2978
3377
  auto ivf = std::make_unique<IndexBinaryIVF>();
@@ -2984,6 +3383,8 @@ std::unique_ptr<IndexBinary> read_index_binary_up(IOReader* f, int io_flags) {
2984
3383
  read_index_binary_header(*idxff, f);
2985
3384
  idxff->own_fields = true;
2986
3385
  idxff->index = read_index(f, io_flags);
3386
+ FAISS_THROW_IF_NOT_MSG(
3387
+ idxff->index, "IndexBinaryFromFloat inner index is null");
2987
3388
  idx = std::move(idxff);
2988
3389
  } else if (h == fourcc("IBHf")) {
2989
3390
  auto idxhnsw = std::make_unique<IndexBinaryHNSW>();
@@ -3009,8 +3410,8 @@ std::unique_ptr<IndexBinary> read_index_binary_up(IOReader* f, int io_flags) {
3009
3410
  } else if (h == fourcc("IBHc")) {
3010
3411
  auto idxhnsw = std::make_unique<IndexBinaryHNSWCagra>();
3011
3412
  read_index_binary_header(*idxhnsw, f);
3012
- READ1(idxhnsw->keep_max_size_level0);
3013
- READ1(idxhnsw->base_level_only);
3413
+ READ1_BOOL(idxhnsw->keep_max_size_level0);
3414
+ READ1_BOOL(idxhnsw->base_level_only);
3014
3415
  READ1(idxhnsw->num_base_level_search_entrypoints);
3015
3416
  read_HNSW(idxhnsw->hnsw, f);
3016
3417
  idxhnsw->hnsw.is_panorama = false;