faiss 0.6.0 → 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (378) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -0
  3. data/ext/faiss/extconf.rb +2 -1
  4. data/ext/faiss/{index_rb.cpp → index.cpp} +1 -1
  5. data/ext/faiss/index_binary.cpp +1 -1
  6. data/ext/faiss/kmeans.cpp +1 -1
  7. data/ext/faiss/pca_matrix.cpp +1 -1
  8. data/ext/faiss/product_quantizer.cpp +1 -1
  9. data/ext/faiss/{utils_rb.cpp → utils.cpp} +1 -1
  10. data/lib/faiss/version.rb +1 -1
  11. data/vendor/faiss/faiss/AutoTune.cpp +93 -80
  12. data/vendor/faiss/faiss/Clustering.cpp +39 -240
  13. data/vendor/faiss/faiss/Clustering.h +6 -0
  14. data/vendor/faiss/faiss/IVFlib.cpp +41 -21
  15. data/vendor/faiss/faiss/Index.cpp +6 -5
  16. data/vendor/faiss/faiss/Index.h +5 -5
  17. data/vendor/faiss/faiss/Index2Layer.cpp +37 -53
  18. data/vendor/faiss/faiss/IndexAdditiveQuantizer.cpp +49 -37
  19. data/vendor/faiss/faiss/IndexAdditiveQuantizerFastScan.cpp +36 -34
  20. data/vendor/faiss/faiss/IndexAdditiveQuantizerFastScan.h +4 -1
  21. data/vendor/faiss/faiss/IndexBinary.cpp +5 -3
  22. data/vendor/faiss/faiss/IndexBinary.h +4 -4
  23. data/vendor/faiss/faiss/IndexBinaryFlat.cpp +1 -1
  24. data/vendor/faiss/faiss/IndexBinaryFlat.h +1 -1
  25. data/vendor/faiss/faiss/IndexBinaryFromFloat.cpp +4 -4
  26. data/vendor/faiss/faiss/IndexBinaryHNSW.cpp +88 -97
  27. data/vendor/faiss/faiss/IndexBinaryHNSW.h +9 -3
  28. data/vendor/faiss/faiss/IndexBinaryHash.cpp +45 -236
  29. data/vendor/faiss/faiss/IndexBinaryHash.h +6 -6
  30. data/vendor/faiss/faiss/IndexBinaryIVF.cpp +89 -417
  31. data/vendor/faiss/faiss/IndexFastScan.cpp +72 -109
  32. data/vendor/faiss/faiss/IndexFastScan.h +25 -23
  33. data/vendor/faiss/faiss/IndexFlat.cpp +27 -20
  34. data/vendor/faiss/faiss/IndexFlat.h +21 -18
  35. data/vendor/faiss/faiss/IndexFlatCodes.cpp +42 -19
  36. data/vendor/faiss/faiss/IndexHNSW.cpp +374 -206
  37. data/vendor/faiss/faiss/IndexHNSW.h +16 -2
  38. data/vendor/faiss/faiss/IndexIDMap.cpp +25 -21
  39. data/vendor/faiss/faiss/IndexIDMap.h +9 -7
  40. data/vendor/faiss/faiss/IndexIVF.cpp +467 -364
  41. data/vendor/faiss/faiss/IndexIVF.h +33 -12
  42. data/vendor/faiss/faiss/IndexIVFAdditiveQuantizer.cpp +79 -76
  43. data/vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.cpp +96 -93
  44. data/vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.h +4 -1
  45. data/vendor/faiss/faiss/IndexIVFFastScan.cpp +357 -238
  46. data/vendor/faiss/faiss/IndexIVFFastScan.h +42 -41
  47. data/vendor/faiss/faiss/IndexIVFFlat.cpp +39 -69
  48. data/vendor/faiss/faiss/IndexIVFFlat.h +32 -0
  49. data/vendor/faiss/faiss/IndexIVFFlatPanorama.cpp +56 -33
  50. data/vendor/faiss/faiss/IndexIVFFlatPanorama.h +3 -1
  51. data/vendor/faiss/faiss/IndexIVFIndependentQuantizer.cpp +18 -15
  52. data/vendor/faiss/faiss/IndexIVFPQ.cpp +73 -846
  53. data/vendor/faiss/faiss/IndexIVFPQFastScan.cpp +151 -121
  54. data/vendor/faiss/faiss/IndexIVFPQFastScan.h +3 -0
  55. data/vendor/faiss/faiss/IndexIVFPQR.cpp +23 -20
  56. data/vendor/faiss/faiss/IndexIVFRaBitQ.cpp +30 -52
  57. data/vendor/faiss/faiss/IndexIVFRaBitQ.h +2 -1
  58. data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.cpp +475 -476
  59. data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.h +248 -93
  60. data/vendor/faiss/faiss/IndexIVFSpectralHash.cpp +41 -127
  61. data/vendor/faiss/faiss/IndexIVFSpectralHash.h +1 -1
  62. data/vendor/faiss/faiss/IndexLSH.cpp +36 -19
  63. data/vendor/faiss/faiss/IndexLattice.cpp +13 -13
  64. data/vendor/faiss/faiss/IndexNNDescent.cpp +36 -21
  65. data/vendor/faiss/faiss/IndexNNDescent.h +2 -2
  66. data/vendor/faiss/faiss/IndexNSG.cpp +38 -23
  67. data/vendor/faiss/faiss/IndexNeuralNetCodec.cpp +31 -11
  68. data/vendor/faiss/faiss/IndexPQ.cpp +128 -221
  69. data/vendor/faiss/faiss/IndexPQ.h +3 -2
  70. data/vendor/faiss/faiss/IndexPQFastScan.cpp +20 -14
  71. data/vendor/faiss/faiss/IndexPQFastScan.h +3 -0
  72. data/vendor/faiss/faiss/IndexPreTransform.cpp +25 -18
  73. data/vendor/faiss/faiss/IndexPreTransform.h +1 -1
  74. data/vendor/faiss/faiss/IndexRaBitQ.cpp +11 -36
  75. data/vendor/faiss/faiss/IndexRaBitQ.h +2 -1
  76. data/vendor/faiss/faiss/IndexRaBitQFastScan.cpp +41 -277
  77. data/vendor/faiss/faiss/IndexRaBitQFastScan.h +183 -27
  78. data/vendor/faiss/faiss/IndexRefine.cpp +30 -25
  79. data/vendor/faiss/faiss/IndexRefine.h +4 -4
  80. data/vendor/faiss/faiss/IndexReplicas.cpp +6 -6
  81. data/vendor/faiss/faiss/IndexRowwiseMinMax.cpp +15 -14
  82. data/vendor/faiss/faiss/IndexRowwiseMinMax.h +1 -1
  83. data/vendor/faiss/faiss/IndexScalarQuantizer.cpp +150 -20
  84. data/vendor/faiss/faiss/IndexScalarQuantizer.h +10 -0
  85. data/vendor/faiss/faiss/IndexShards.cpp +10 -9
  86. data/vendor/faiss/faiss/IndexShardsIVF.cpp +21 -15
  87. data/vendor/faiss/faiss/MatrixStats.cpp +5 -4
  88. data/vendor/faiss/faiss/MetaIndexes.cpp +19 -17
  89. data/vendor/faiss/faiss/MetaIndexes.h +1 -1
  90. data/vendor/faiss/faiss/MetricType.h +14 -7
  91. data/vendor/faiss/faiss/SuperKMeans.cpp +656 -0
  92. data/vendor/faiss/faiss/SuperKMeans.h +97 -0
  93. data/vendor/faiss/faiss/VectorTransform.cpp +237 -149
  94. data/vendor/faiss/faiss/VectorTransform.h +16 -16
  95. data/vendor/faiss/faiss/build.cpp +23 -0
  96. data/vendor/faiss/faiss/build.h +15 -0
  97. data/vendor/faiss/faiss/clone_index.cpp +48 -47
  98. data/vendor/faiss/faiss/cppcontrib/SaDecodeKernels.h +1 -1
  99. data/vendor/faiss/faiss/cppcontrib/sa_decode/Level2-avx2-inl.h +47 -47
  100. data/vendor/faiss/faiss/cppcontrib/sa_decode/Level2-inl.h +11 -0
  101. data/vendor/faiss/faiss/cppcontrib/sa_decode/Level2-neon-inl.h +902 -12
  102. data/vendor/faiss/faiss/cppcontrib/sa_decode/PQ-avx2-inl.h +38 -38
  103. data/vendor/faiss/faiss/cppcontrib/sa_decode/PQ-inl.h +11 -0
  104. data/vendor/faiss/faiss/cppcontrib/sa_decode/PQ-neon-inl.h +702 -10
  105. data/vendor/faiss/faiss/factory_tools.cpp +9 -0
  106. data/vendor/faiss/faiss/gpu/GpuIndexCagra.h +6 -5
  107. data/vendor/faiss/faiss/gpu/GpuResources.h +3 -2
  108. data/vendor/faiss/faiss/gpu/StandardGpuResources.cpp +15 -16
  109. data/vendor/faiss/faiss/gpu/StandardGpuResources.h +5 -4
  110. data/vendor/faiss/faiss/gpu/test/TestGpuIndexFlat.cpp +46 -0
  111. data/vendor/faiss/faiss/gpu/test/TestGpuIndexIVFFlat.cpp +56 -0
  112. data/vendor/faiss/faiss/gpu/test/TestGpuIndexIVFPQ.cpp +78 -1
  113. data/vendor/faiss/faiss/gpu/test/TestUtils.cpp +72 -0
  114. data/vendor/faiss/faiss/gpu/test/TestUtils.h +23 -0
  115. data/vendor/faiss/faiss/gpu/utils/CuvsFilterConvert.h +1 -1
  116. data/vendor/faiss/faiss/gpu/utils/CuvsUtils.h +21 -10
  117. data/vendor/faiss/faiss/gpu_metal/GpuIndexFlat.h +22 -0
  118. data/vendor/faiss/faiss/gpu_metal/MetalCloner.h +35 -0
  119. data/vendor/faiss/faiss/gpu_metal/MetalDistance.h +87 -0
  120. data/vendor/faiss/faiss/gpu_metal/MetalFlatKernels.h +40 -0
  121. data/vendor/faiss/faiss/gpu_metal/MetalIndex.h +58 -0
  122. data/vendor/faiss/faiss/gpu_metal/MetalIndexFlat.h +65 -0
  123. data/vendor/faiss/faiss/gpu_metal/MetalIndexIVFFlat.h +181 -0
  124. data/vendor/faiss/faiss/gpu_metal/MetalKernels.h +111 -0
  125. data/vendor/faiss/faiss/gpu_metal/MetalPythonBridge.h +45 -0
  126. data/vendor/faiss/faiss/gpu_metal/MetalResources.h +79 -0
  127. data/vendor/faiss/faiss/gpu_metal/StandardMetalResources.h +35 -0
  128. data/vendor/faiss/faiss/gpu_metal/impl/MetalIVFFlat.h +193 -0
  129. data/vendor/faiss/faiss/impl/AdSampling.cpp +103 -0
  130. data/vendor/faiss/faiss/impl/AdSampling.h +35 -0
  131. data/vendor/faiss/faiss/impl/AdditiveQuantizer.cpp +29 -25
  132. data/vendor/faiss/faiss/impl/AdditiveQuantizer.h +1 -0
  133. data/vendor/faiss/faiss/impl/AuxIndexStructures.cpp +10 -9
  134. data/vendor/faiss/faiss/impl/AuxIndexStructures.h +3 -0
  135. data/vendor/faiss/faiss/impl/ClusteringHelpers.cpp +244 -0
  136. data/vendor/faiss/faiss/impl/ClusteringHelpers.h +94 -0
  137. data/vendor/faiss/faiss/impl/ClusteringInitialization.cpp +16 -16
  138. data/vendor/faiss/faiss/impl/CodePacker.cpp +3 -3
  139. data/vendor/faiss/faiss/impl/CodePackerRaBitQ.cpp +1 -1
  140. data/vendor/faiss/faiss/impl/DistanceComputer.h +8 -8
  141. data/vendor/faiss/faiss/impl/FaissAssert.h +6 -3
  142. data/vendor/faiss/faiss/impl/FaissException.h +50 -3
  143. data/vendor/faiss/faiss/impl/HNSW.cpp +639 -507
  144. data/vendor/faiss/faiss/impl/HNSW.h +61 -44
  145. data/vendor/faiss/faiss/impl/IDSelector.cpp +15 -11
  146. data/vendor/faiss/faiss/impl/IDSelector.h +8 -8
  147. data/vendor/faiss/faiss/impl/InvertedListScannerStats.h +26 -0
  148. data/vendor/faiss/faiss/impl/LocalSearchQuantizer.cpp +82 -77
  149. data/vendor/faiss/faiss/impl/NNDescent.cpp +62 -25
  150. data/vendor/faiss/faiss/impl/NNDescent.h +6 -2
  151. data/vendor/faiss/faiss/impl/NSG.cpp +53 -32
  152. data/vendor/faiss/faiss/impl/NSG.h +4 -4
  153. data/vendor/faiss/faiss/impl/Panorama.cpp +23 -6
  154. data/vendor/faiss/faiss/impl/Panorama.h +269 -87
  155. data/vendor/faiss/faiss/impl/PdxLayout.cpp +93 -0
  156. data/vendor/faiss/faiss/impl/PdxLayout.h +41 -0
  157. data/vendor/faiss/faiss/impl/PolysemousTraining.cpp +46 -32
  158. data/vendor/faiss/faiss/impl/PolysemousTraining.h +3 -3
  159. data/vendor/faiss/faiss/impl/ProductAdditiveQuantizer.cpp +35 -35
  160. data/vendor/faiss/faiss/impl/ProductQuantizer-inl.h +21 -16
  161. data/vendor/faiss/faiss/impl/ProductQuantizer.cpp +55 -25
  162. data/vendor/faiss/faiss/impl/Quantizer.h +2 -2
  163. data/vendor/faiss/faiss/impl/RaBitQUtils.cpp +55 -49
  164. data/vendor/faiss/faiss/impl/RaBitQUtils.h +65 -0
  165. data/vendor/faiss/faiss/impl/RaBitQuantizer.cpp +302 -283
  166. data/vendor/faiss/faiss/impl/ResidualQuantizer.cpp +26 -23
  167. data/vendor/faiss/faiss/impl/ResidualQuantizer.h +1 -1
  168. data/vendor/faiss/faiss/impl/ResultHandler.h +100 -75
  169. data/vendor/faiss/faiss/impl/ScalarQuantizer.cpp +318 -7
  170. data/vendor/faiss/faiss/impl/ScalarQuantizer.h +77 -1
  171. data/vendor/faiss/faiss/impl/ThreadedIndex-inl.h +14 -11
  172. data/vendor/faiss/faiss/impl/VisitedTable.cpp +10 -10
  173. data/vendor/faiss/faiss/impl/VisitedTable.h +70 -28
  174. data/vendor/faiss/faiss/impl/approx_topk/approx_topk.h +276 -0
  175. data/vendor/faiss/faiss/impl/approx_topk/avx2.cpp +68 -0
  176. data/vendor/faiss/faiss/{utils → impl}/approx_topk/generic.h +15 -8
  177. data/vendor/faiss/faiss/impl/approx_topk/neon.cpp +68 -0
  178. data/vendor/faiss/faiss/impl/approx_topk/rq_beam_search_tab-inl.h +169 -0
  179. data/vendor/faiss/faiss/impl/approx_topk/rq_beam_search_tab.h +117 -0
  180. data/vendor/faiss/faiss/impl/approx_topk/simdlib256-inl.h +146 -0
  181. data/vendor/faiss/faiss/impl/binary_hamming/IndexBinaryHNSW_impl.h +73 -0
  182. data/vendor/faiss/faiss/impl/binary_hamming/IndexBinaryHash_impl.h +270 -0
  183. data/vendor/faiss/faiss/impl/binary_hamming/IndexBinaryIVF_impl.h +460 -0
  184. data/vendor/faiss/faiss/impl/binary_hamming/IndexIVFSpectralHash_impl.h +159 -0
  185. data/vendor/faiss/faiss/impl/binary_hamming/IndexPQ_impl.h +92 -0
  186. data/vendor/faiss/faiss/impl/binary_hamming/avx2.cpp +26 -0
  187. data/vendor/faiss/faiss/impl/binary_hamming/avx512.cpp +26 -0
  188. data/vendor/faiss/faiss/impl/binary_hamming/dispatch.h +143 -0
  189. data/vendor/faiss/faiss/impl/binary_hamming/neon.cpp +26 -0
  190. data/vendor/faiss/faiss/impl/binary_hamming/rvv.cpp +26 -0
  191. data/vendor/faiss/faiss/impl/expanded_scanners.h +8 -3
  192. data/vendor/faiss/faiss/impl/{FastScanDistancePostProcessing.h → fast_scan/FastScanDistancePostProcessing.h} +13 -6
  193. data/vendor/faiss/faiss/impl/{LookupTableScaler.h → fast_scan/LookupTableScaler.h} +16 -5
  194. data/vendor/faiss/faiss/impl/fast_scan/accumulate_loops.h +237 -0
  195. data/vendor/faiss/faiss/impl/fast_scan/accumulate_loops_512.h +185 -0
  196. data/vendor/faiss/faiss/impl/fast_scan/decompose_qbs.h +229 -0
  197. data/vendor/faiss/faiss/impl/fast_scan/dispatching.h +270 -0
  198. data/vendor/faiss/faiss/impl/{pq4_fast_scan.cpp → fast_scan/fast_scan.cpp} +169 -2
  199. data/vendor/faiss/faiss/impl/fast_scan/fast_scan.h +341 -0
  200. data/vendor/faiss/faiss/impl/fast_scan/impl-avx2.cpp +36 -0
  201. data/vendor/faiss/faiss/impl/fast_scan/impl-avx512.cpp +40 -0
  202. data/vendor/faiss/faiss/impl/fast_scan/impl-neon.cpp +120 -0
  203. data/vendor/faiss/faiss/impl/fast_scan/impl-riscv.cpp +104 -0
  204. data/vendor/faiss/faiss/impl/fast_scan/kernels_simd256.h +213 -0
  205. data/vendor/faiss/faiss/impl/{pq4_fast_scan_search_qbs.cpp → fast_scan/kernels_simd512.h} +26 -356
  206. data/vendor/faiss/faiss/impl/fast_scan/rabitq_dispatching.h +90 -0
  207. data/vendor/faiss/faiss/impl/fast_scan/rabitq_result_handler.h +108 -0
  208. data/vendor/faiss/faiss/impl/{simd_result_handlers.h → fast_scan/simd_result_handlers.h} +282 -134
  209. data/vendor/faiss/faiss/impl/hnsw/LockVector.cpp +54 -0
  210. data/vendor/faiss/faiss/impl/hnsw/LockVector.h +64 -0
  211. data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.cpp +83 -0
  212. data/vendor/faiss/faiss/impl/hnsw/MinimaxHeap.h +113 -0
  213. data/vendor/faiss/faiss/impl/hnsw/avx2.cpp +150 -0
  214. data/vendor/faiss/faiss/impl/hnsw/avx512.cpp +142 -0
  215. data/vendor/faiss/faiss/impl/index_read.cpp +1227 -79
  216. data/vendor/faiss/faiss/impl/index_read_utils.h +1 -1
  217. data/vendor/faiss/faiss/impl/index_write.cpp +96 -13
  218. data/vendor/faiss/faiss/impl/io.cpp +6 -6
  219. data/vendor/faiss/faiss/impl/io_macros.h +58 -16
  220. data/vendor/faiss/faiss/impl/kmeans1d.cpp +10 -10
  221. data/vendor/faiss/faiss/impl/lattice_Zn.cpp +37 -23
  222. data/vendor/faiss/faiss/impl/lattice_Zn.h +6 -6
  223. data/vendor/faiss/faiss/impl/mapped_io.cpp +6 -6
  224. data/vendor/faiss/faiss/impl/platform_macros.h +15 -4
  225. data/vendor/faiss/faiss/impl/pq_code_distance/IVFPQScanner_impl.h +549 -0
  226. data/vendor/faiss/faiss/impl/pq_code_distance/IVFPQ_QueryTables.cpp +245 -0
  227. data/vendor/faiss/faiss/impl/pq_code_distance/IVFPQ_QueryTables.h +105 -0
  228. data/vendor/faiss/faiss/impl/pq_code_distance/PQDistanceComputer_impl.h +106 -0
  229. data/vendor/faiss/faiss/impl/pq_code_distance/avx2.cpp +23 -0
  230. data/vendor/faiss/faiss/impl/pq_code_distance/avx512.cpp +23 -0
  231. data/vendor/faiss/faiss/impl/pq_code_distance/neon.cpp +23 -0
  232. data/vendor/faiss/faiss/impl/pq_code_distance/{pq_code_distance-avx2.cpp → pq_code_distance-avx2.h} +9 -13
  233. data/vendor/faiss/faiss/impl/pq_code_distance/{pq_code_distance-avx512.cpp → pq_code_distance-avx512.h} +9 -57
  234. data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-generic.cpp +45 -107
  235. data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-generic.h +96 -0
  236. data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-inl.h +274 -5
  237. data/vendor/faiss/faiss/impl/pq_code_distance/pq_code_distance-sve.cpp +10 -7
  238. data/vendor/faiss/faiss/impl/pq_code_distance/pq_scan_impl.h +105 -0
  239. data/vendor/faiss/faiss/impl/pq_code_distance/rvv.cpp +70 -0
  240. data/vendor/faiss/faiss/impl/residual_quantizer_encode_steps.cpp +311 -477
  241. data/vendor/faiss/faiss/impl/residual_quantizer_encode_steps.h +1 -1
  242. data/vendor/faiss/faiss/impl/scalar_quantizer/codecs.h +1 -1
  243. data/vendor/faiss/faiss/impl/scalar_quantizer/distance_computers.h +9 -2
  244. data/vendor/faiss/faiss/impl/scalar_quantizer/quantizers.h +419 -19
  245. data/vendor/faiss/faiss/impl/scalar_quantizer/scanners.h +27 -1
  246. data/vendor/faiss/faiss/impl/scalar_quantizer/similarities.h +3 -3
  247. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx2.cpp +387 -2
  248. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-impl.h +553 -0
  249. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512-spr.cpp +559 -0
  250. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-avx512.cpp +341 -2
  251. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-dispatch.h +425 -3
  252. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-neon.cpp +290 -2
  253. data/vendor/faiss/faiss/impl/scalar_quantizer/sq-rvv.cpp +337 -0
  254. data/vendor/faiss/faiss/impl/scalar_quantizer/training.cpp +192 -8
  255. data/vendor/faiss/faiss/impl/scalar_quantizer/training.h +12 -0
  256. data/vendor/faiss/faiss/impl/simd_dispatch.h +157 -66
  257. data/vendor/faiss/faiss/impl/simdlib/simdlib.h +57 -0
  258. data/vendor/faiss/faiss/{utils → impl/simdlib}/simdlib_avx2.h +264 -172
  259. data/vendor/faiss/faiss/impl/simdlib/simdlib_avx512.h +414 -0
  260. data/vendor/faiss/faiss/impl/simdlib/simdlib_dispatch.h +44 -0
  261. data/vendor/faiss/faiss/{utils → impl/simdlib}/simdlib_emulated.h +231 -166
  262. data/vendor/faiss/faiss/{utils → impl/simdlib}/simdlib_neon.h +270 -218
  263. data/vendor/faiss/faiss/{utils → impl/simdlib}/simdlib_ppc64.h +201 -160
  264. data/vendor/faiss/faiss/impl/svs_io.cpp +12 -3
  265. data/vendor/faiss/faiss/impl/svs_io.h +8 -2
  266. data/vendor/faiss/faiss/index_factory.cpp +90 -18
  267. data/vendor/faiss/faiss/index_io.h +40 -0
  268. data/vendor/faiss/faiss/invlists/BlockInvertedLists.cpp +66 -16
  269. data/vendor/faiss/faiss/invlists/DirectMap.cpp +28 -15
  270. data/vendor/faiss/faiss/invlists/DirectMap.h +4 -3
  271. data/vendor/faiss/faiss/invlists/InvertedLists.cpp +170 -86
  272. data/vendor/faiss/faiss/invlists/InvertedLists.h +88 -25
  273. data/vendor/faiss/faiss/invlists/InvertedListsIOHook.cpp +4 -4
  274. data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.cpp +13 -13
  275. data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.h +1 -1
  276. data/vendor/faiss/faiss/svs/IndexSVSFaissUtils.h +1 -1
  277. data/vendor/faiss/faiss/svs/IndexSVSFlat.cpp +2 -2
  278. data/vendor/faiss/faiss/svs/IndexSVSIVF.cpp +350 -0
  279. data/vendor/faiss/faiss/svs/IndexSVSIVF.h +128 -0
  280. data/vendor/faiss/faiss/svs/IndexSVSIVFLVQ.cpp +40 -0
  281. data/vendor/faiss/faiss/svs/IndexSVSIVFLVQ.h +43 -0
  282. data/vendor/faiss/faiss/svs/IndexSVSIVFLeanVec.cpp +225 -0
  283. data/vendor/faiss/faiss/svs/IndexSVSIVFLeanVec.h +71 -0
  284. data/vendor/faiss/faiss/svs/IndexSVSVamana.cpp +142 -21
  285. data/vendor/faiss/faiss/svs/IndexSVSVamana.h +33 -7
  286. data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.cpp +3 -2
  287. data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.h +2 -1
  288. data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.cpp +77 -27
  289. data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.h +10 -4
  290. data/vendor/faiss/faiss/utils/Heap.cpp +10 -10
  291. data/vendor/faiss/faiss/utils/NeuralNet.cpp +47 -36
  292. data/vendor/faiss/faiss/utils/NeuralNet.h +1 -1
  293. data/vendor/faiss/faiss/utils/approx_topk_hamming/approx_topk_hamming.h +10 -4
  294. data/vendor/faiss/faiss/utils/bf16.h +34 -0
  295. data/vendor/faiss/faiss/utils/distances.cpp +390 -560
  296. data/vendor/faiss/faiss/utils/distances.h +20 -1
  297. data/vendor/faiss/faiss/utils/distances_dispatch.h +117 -37
  298. data/vendor/faiss/faiss/utils/distances_fused/avx512.cpp +8 -7
  299. data/vendor/faiss/faiss/utils/distances_fused/distances_fused.cpp +33 -14
  300. data/vendor/faiss/faiss/utils/distances_fused/distances_fused.h +12 -1
  301. data/vendor/faiss/faiss/utils/distances_fused/simdlib_based.cpp +16 -293
  302. data/vendor/faiss/faiss/utils/distances_fused/simdlib_based_neon.cpp +57 -0
  303. data/vendor/faiss/faiss/utils/distances_fused/simdlib_kernel-inl.h +290 -0
  304. data/vendor/faiss/faiss/utils/distances_simd.cpp +5 -178
  305. data/vendor/faiss/faiss/utils/extra_distances.cpp +9 -8
  306. data/vendor/faiss/faiss/utils/extra_distances.h +32 -6
  307. data/vendor/faiss/faiss/utils/hamming-inl.h +13 -11
  308. data/vendor/faiss/faiss/utils/hamming.cpp +66 -517
  309. data/vendor/faiss/faiss/utils/hamming.h +92 -2
  310. data/vendor/faiss/faiss/utils/hamming_distance/common.h +287 -10
  311. data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx2.cpp +16 -0
  312. data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx512.cpp +15 -0
  313. data/vendor/faiss/faiss/utils/hamming_distance/hamming_avx512_spr.cpp +15 -0
  314. data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx2.h +142 -0
  315. data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512.h +210 -0
  316. data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-avx512_spr.h +171 -0
  317. data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-generic.h +368 -0
  318. data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-neon.h +322 -0
  319. data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer-rvv.h +39 -0
  320. data/vendor/faiss/faiss/utils/hamming_distance/hamming_computer.h +146 -0
  321. data/vendor/faiss/faiss/utils/hamming_distance/hamming_impl.h +481 -0
  322. data/vendor/faiss/faiss/utils/hamming_distance/hamming_neon.cpp +15 -0
  323. data/vendor/faiss/faiss/utils/hamming_distance/hamming_rvv.cpp +15 -0
  324. data/vendor/faiss/faiss/utils/partitioning.cpp +66 -989
  325. data/vendor/faiss/faiss/utils/partitioning.h +31 -0
  326. data/vendor/faiss/faiss/utils/popcount.h +29 -0
  327. data/vendor/faiss/faiss/utils/pq_code_distance.h +2 -2
  328. data/vendor/faiss/faiss/utils/prefetch.h +2 -2
  329. data/vendor/faiss/faiss/utils/quantize_lut.cpp +30 -30
  330. data/vendor/faiss/faiss/utils/quantize_lut.h +1 -1
  331. data/vendor/faiss/faiss/utils/rabitq_simd.h +57 -536
  332. data/vendor/faiss/faiss/utils/random.cpp +6 -6
  333. data/vendor/faiss/faiss/utils/simd_impl/IVFFlatScanner-inl.h +51 -0
  334. data/vendor/faiss/faiss/utils/simd_impl/distances_aarch64.cpp +5 -1
  335. data/vendor/faiss/faiss/utils/simd_impl/distances_arm_sve.cpp +213 -4
  336. data/vendor/faiss/faiss/utils/simd_impl/distances_autovec-inl.h +163 -10
  337. data/vendor/faiss/faiss/utils/simd_impl/distances_avx2.cpp +250 -4
  338. data/vendor/faiss/faiss/utils/simd_impl/distances_avx512.cpp +7 -4
  339. data/vendor/faiss/faiss/utils/simd_impl/distances_rvv.cpp +189 -0
  340. data/vendor/faiss/faiss/utils/simd_impl/distances_simdlib256.h +195 -0
  341. data/vendor/faiss/faiss/utils/simd_impl/distances_sse-inl.h +2 -1
  342. data/vendor/faiss/faiss/utils/{distances_fused/simdlib_based.h → simd_impl/exhaustive_L2sqr_blas_cmax.h} +5 -10
  343. data/vendor/faiss/faiss/utils/simd_impl/hamming_impl.h +481 -0
  344. data/vendor/faiss/faiss/utils/simd_impl/partitioning_avx2.cpp +14 -0
  345. data/vendor/faiss/faiss/utils/simd_impl/partitioning_neon.cpp +14 -0
  346. data/vendor/faiss/faiss/utils/simd_impl/partitioning_simdlib256.h +1031 -0
  347. data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx2.cpp +355 -0
  348. data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512.cpp +477 -0
  349. data/vendor/faiss/faiss/utils/simd_impl/rabitq_avx512_spr.cpp +343 -0
  350. data/vendor/faiss/faiss/utils/simd_impl/rabitq_neon.cpp +55 -0
  351. data/vendor/faiss/faiss/utils/simd_impl/rabitq_rvv.cpp +55 -0
  352. data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_dispatch.h +32 -0
  353. data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels.h +43 -0
  354. data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels_avx2.cpp +57 -0
  355. data/vendor/faiss/faiss/utils/simd_impl/super_kmeans_kernels_avx512.cpp +45 -0
  356. data/vendor/faiss/faiss/utils/simd_levels.cpp +29 -7
  357. data/vendor/faiss/faiss/utils/simd_levels.h +93 -1
  358. data/vendor/faiss/faiss/utils/sorting.cpp +48 -36
  359. data/vendor/faiss/faiss/utils/utils.cpp +5 -5
  360. data/vendor/faiss/faiss/utils/utils.h +3 -3
  361. metadata +129 -34
  362. data/vendor/faiss/faiss/impl/RaBitQStats.cpp +0 -29
  363. data/vendor/faiss/faiss/impl/RaBitQStats.h +0 -56
  364. data/vendor/faiss/faiss/impl/pq4_fast_scan.h +0 -224
  365. data/vendor/faiss/faiss/impl/pq4_fast_scan_search_1.cpp +0 -230
  366. data/vendor/faiss/faiss/utils/approx_topk/approx_topk.h +0 -84
  367. data/vendor/faiss/faiss/utils/approx_topk/avx2-inl.h +0 -196
  368. data/vendor/faiss/faiss/utils/approx_topk/mode.h +0 -34
  369. data/vendor/faiss/faiss/utils/distances_fused/avx512.h +0 -36
  370. data/vendor/faiss/faiss/utils/extra_distances-inl.h +0 -235
  371. data/vendor/faiss/faiss/utils/hamming_distance/avx2-inl.h +0 -462
  372. data/vendor/faiss/faiss/utils/hamming_distance/avx512-inl.h +0 -490
  373. data/vendor/faiss/faiss/utils/hamming_distance/generic-inl.h +0 -449
  374. data/vendor/faiss/faiss/utils/hamming_distance/hamdis-inl.h +0 -87
  375. data/vendor/faiss/faiss/utils/hamming_distance/neon-inl.h +0 -524
  376. data/vendor/faiss/faiss/utils/simdlib.h +0 -42
  377. data/vendor/faiss/faiss/utils/simdlib_avx512.h +0 -365
  378. /data/ext/faiss/{utils_rb.h → utils.h} +0 -0
@@ -10,6 +10,8 @@
10
10
 
11
11
  #include <faiss/impl/io_macros.h>
12
12
 
13
+ #include <cinttypes>
14
+ #include <cmath>
13
15
  #include <cstdio>
14
16
  #include <cstdlib>
15
17
  #include <cstring>
@@ -56,6 +58,9 @@
56
58
  #ifdef FAISS_ENABLE_SVS
57
59
  #include <faiss/impl/svs_io.h>
58
60
  #include <faiss/svs/IndexSVSFlat.h>
61
+ #include <faiss/svs/IndexSVSIVF.h>
62
+ #include <faiss/svs/IndexSVSIVFLVQ.h>
63
+ #include <faiss/svs/IndexSVSIVFLeanVec.h>
59
64
  #include <faiss/svs/IndexSVSVamana.h>
60
65
  #include <faiss/svs/IndexSVSVamanaLVQ.h>
61
66
  #include <faiss/svs/IndexSVSVamanaLeanVec.h>
@@ -78,6 +83,68 @@
78
83
 
79
84
  namespace faiss {
80
85
 
86
+ namespace {
87
+ size_t deserialization_loop_limit_ = 0;
88
+ size_t deserialization_vector_byte_limit_ = uint64_t{1} << 40; // 1 TB
89
+ size_t deserialization_lattice_r2_limit_ = 0;
90
+
91
+ #ifdef FAISS_ENABLE_SVS
92
+ // Read and validate an SVSStorageKind from the stream. Centralizes the
93
+ // [0, SVS_count) range check so every SVS read site rejects out-of-range
94
+ // values uniformly at the deserialization boundary, instead of letting
95
+ // to_svs_storage_kind() surface the failure later from inside an SVS
96
+ // runtime load.
97
+ SVSStorageKind read_svs_storage_kind(IOReader* f) {
98
+ int sk;
99
+ READ1(sk);
100
+ FAISS_THROW_IF_NOT_FMT(
101
+ sk >= 0 && sk < static_cast<int>(SVS_count),
102
+ "invalid SVS storage_kind=%d (must be in [0, %d))",
103
+ sk,
104
+ static_cast<int>(SVS_count));
105
+ return static_cast<SVSStorageKind>(sk);
106
+ }
107
+ #endif // FAISS_ENABLE_SVS
108
+ } // namespace
109
+
110
+ size_t get_deserialization_loop_limit() {
111
+ return deserialization_loop_limit_;
112
+ }
113
+
114
+ void set_deserialization_loop_limit(size_t value) {
115
+ deserialization_loop_limit_ = value;
116
+ }
117
+
118
+ size_t get_deserialization_vector_byte_limit() {
119
+ return deserialization_vector_byte_limit_;
120
+ }
121
+
122
+ void set_deserialization_vector_byte_limit(size_t value) {
123
+ deserialization_vector_byte_limit_ = value;
124
+ }
125
+
126
+ size_t get_deserialization_lattice_r2_limit() {
127
+ return deserialization_lattice_r2_limit_;
128
+ }
129
+
130
+ void set_deserialization_lattice_r2_limit(size_t value) {
131
+ deserialization_lattice_r2_limit_ = value;
132
+ }
133
+
134
+ #define FAISS_CHECK_DESERIALIZATION_LOOP_LIMIT(val, field_name) \
135
+ do { \
136
+ auto limit_ = get_deserialization_loop_limit(); \
137
+ if (limit_ > 0) { \
138
+ FAISS_THROW_IF_NOT_FMT( \
139
+ static_cast<size_t>(val) <= limit_, \
140
+ "%s=%zd exceeds deserialization_loop_limit" \
141
+ " of %zd", \
142
+ field_name, \
143
+ static_cast<size_t>(val), \
144
+ limit_); \
145
+ } \
146
+ } while (0)
147
+
81
148
  /*************************************************************
82
149
  * Mmap-ing and viewing facilities
83
150
  **************************************************************/
@@ -221,7 +288,7 @@ static void read_index_header(Index& idx, IOReader* f) {
221
288
  idx_t dummy;
222
289
  READ1(dummy);
223
290
  READ1(dummy);
224
- READ1(idx.is_trained);
291
+ READ1_BOOL(idx.is_trained);
225
292
  int metric_type_int;
226
293
  READ1(metric_type_int);
227
294
  idx.metric_type = metric_type_from_int(metric_type_int);
@@ -249,7 +316,7 @@ std::unique_ptr<VectorTransform> read_VectorTransform_up(IOReader* f) {
249
316
  if (h == fourcc("Pcam")) {
250
317
  READ1(pca->epsilon);
251
318
  }
252
- READ1(pca->random_rotation);
319
+ READ1_BOOL(pca->random_rotation);
253
320
  if (h != fourcc("PCAm")) {
254
321
  READ1(pca->balanced_bins);
255
322
  }
@@ -265,11 +332,12 @@ std::unique_ptr<VectorTransform> read_VectorTransform_up(IOReader* f) {
265
332
  } else if (h == fourcc("LTra")) {
266
333
  lt = std::make_unique<LinearTransform>();
267
334
  }
268
- READ1(lt->have_bias);
335
+ READ1_BOOL(lt->have_bias);
269
336
  READVECTOR(lt->A);
270
337
  READVECTOR(lt->b);
271
- FAISS_THROW_IF_NOT(lt->A.size() >= lt->d_in * lt->d_out);
272
- FAISS_THROW_IF_NOT(!lt->have_bias || lt->b.size() >= lt->d_out);
338
+ FAISS_THROW_IF_NOT(
339
+ lt->A.size() >= size_t(lt->d_in) * size_t(lt->d_out));
340
+ FAISS_THROW_IF_NOT(!lt->have_bias || lt->b.size() >= size_t(lt->d_out));
273
341
  lt->set_is_orthonormal();
274
342
  vt = std::move(lt);
275
343
  } else if (h == fourcc("RmDT")) {
@@ -288,7 +356,7 @@ std::unique_ptr<VectorTransform> read_VectorTransform_up(IOReader* f) {
288
356
  auto itqt = std::make_unique<ITQTransform>();
289
357
 
290
358
  READVECTOR(itqt->mean);
291
- READ1(itqt->do_pca);
359
+ READ1_BOOL(itqt->do_pca);
292
360
  {
293
361
  // Read, dereference, discard.
294
362
  auto sub_vt = read_VectorTransform_up(f);
@@ -317,11 +385,104 @@ std::unique_ptr<VectorTransform> read_VectorTransform_up(IOReader* f) {
317
385
  }
318
386
  READ1(vt->d_in);
319
387
  READ1(vt->d_out);
320
- READ1(vt->is_trained);
388
+ READ1_BOOL(vt->is_trained);
389
+ FAISS_THROW_IF_NOT_FMT(
390
+ vt->d_in >= 0,
391
+ "invalid VectorTransform d_in=%d (must be >= 0)",
392
+ vt->d_in);
393
+ FAISS_THROW_IF_NOT_FMT(
394
+ vt->d_out >= 0,
395
+ "invalid VectorTransform d_out=%d (must be >= 0)",
396
+ vt->d_out);
397
+ {
398
+ size_t dim_product = mul_no_overflow(
399
+ vt->d_in, vt->d_out, "VectorTransform d_in * d_out");
400
+ FAISS_THROW_IF_NOT_MSG(
401
+ dim_product <=
402
+ get_deserialization_vector_byte_limit() / sizeof(float),
403
+ "VectorTransform d_in * d_out would exceed "
404
+ "deserialization vector byte limit");
405
+ }
321
406
  if (h == fourcc("HRot")) {
407
+ FAISS_THROW_IF_NOT_FMT(
408
+ vt->d_out > 0 && (vt->d_out & (vt->d_out - 1)) == 0,
409
+ "invalid HadamardRotation d_out=%d (must be a power of 2 > 0)",
410
+ vt->d_out);
411
+ FAISS_THROW_IF_NOT_FMT(
412
+ vt->d_out >= vt->d_in,
413
+ "invalid HadamardRotation d_out=%d < d_in=%d",
414
+ vt->d_out,
415
+ vt->d_in);
416
+ FAISS_THROW_IF_NOT_FMT(
417
+ static_cast<size_t>(vt->d_out) <=
418
+ get_deserialization_vector_byte_limit() /
419
+ (3 * sizeof(float)),
420
+ "HadamardRotation d_out=%d would exceed deserialization byte limit",
421
+ vt->d_out);
322
422
  auto* hr = dynamic_cast<HadamardRotation*>(vt.get());
423
+ FAISS_THROW_IF_NOT_MSG(hr, "dynamic_cast to HadamardRotation failed");
424
+ FAISS_THROW_IF_NOT_FMT(
425
+ vt->d_in > 0,
426
+ "invalid HadamardRotation d_in=%d (must be > 0)",
427
+ vt->d_in);
428
+ size_t p = 1;
429
+ while (p < static_cast<size_t>(vt->d_in)) {
430
+ p <<= 1;
431
+ }
432
+ FAISS_THROW_IF_NOT_FMT(
433
+ static_cast<size_t>(vt->d_out) == p,
434
+ "invalid HadamardRotation d_out %d for d_in %d"
435
+ " (d_out must be the smallest power of 2 >= d_in)",
436
+ vt->d_out,
437
+ vt->d_in);
438
+ size_t byte_limit = get_deserialization_vector_byte_limit();
439
+ FAISS_THROW_IF_NOT_MSG(
440
+ p <= byte_limit / (3 * sizeof(float)),
441
+ "HadamardRotation d_out exceeds deserialization byte limit");
323
442
  hr->init(hr->seed);
324
443
  }
444
+ if (h == fourcc("RmDT")) {
445
+ auto* rdt = dynamic_cast<RemapDimensionsTransform*>(vt.get());
446
+ FAISS_THROW_IF_NOT_MSG(
447
+ rdt, "dynamic_cast to RemapDimensionsTransform failed");
448
+ FAISS_THROW_IF_NOT_FMT(
449
+ static_cast<int>(rdt->map.size()) >= rdt->d_out,
450
+ "RemapDimensionsTransform map size %d < d_out %d",
451
+ (int)rdt->map.size(),
452
+ rdt->d_out);
453
+ }
454
+ if (h == fourcc("VNrm")) {
455
+ FAISS_THROW_IF_NOT_FMT(
456
+ vt->d_in == vt->d_out,
457
+ "NormalizationTransform requires d_in == d_out, "
458
+ "got d_in=%d d_out=%d",
459
+ vt->d_in,
460
+ vt->d_out);
461
+ }
462
+ if (h == fourcc("VCnt")) {
463
+ auto* ct = dynamic_cast<CenteringTransform*>(vt.get());
464
+ FAISS_THROW_IF_NOT_MSG(ct, "dynamic_cast to CenteringTransform failed");
465
+ FAISS_THROW_IF_NOT_FMT(
466
+ static_cast<int>(ct->mean.size()) >= ct->d_in,
467
+ "CenteringTransform mean size %d < d_in %d",
468
+ (int)ct->mean.size(),
469
+ ct->d_in);
470
+ FAISS_THROW_IF_NOT_FMT(
471
+ vt->d_in == vt->d_out,
472
+ "CenteringTransform requires d_in == d_out, "
473
+ "got d_in=%d d_out=%d",
474
+ vt->d_in,
475
+ vt->d_out);
476
+ }
477
+ if (h == fourcc("Viqt")) {
478
+ auto* itqt = dynamic_cast<ITQTransform*>(vt.get());
479
+ FAISS_THROW_IF_NOT_MSG(itqt, "dynamic_cast to ITQTransform failed");
480
+ FAISS_THROW_IF_NOT_FMT(
481
+ static_cast<int>(itqt->mean.size()) >= itqt->d_in,
482
+ "ITQTransform mean size %d < d_in %d",
483
+ (int)itqt->mean.size(),
484
+ itqt->d_in);
485
+ }
325
486
  return vt;
326
487
  }
327
488
 
@@ -357,33 +518,121 @@ static void read_ArrayInvertedLists_sizes(
357
518
  }
358
519
  }
359
520
 
521
+ bool index_read_warn_on_null_invlists = true;
522
+
360
523
  std::unique_ptr<InvertedLists> read_InvertedLists_up(
361
524
  IOReader* f,
362
525
  int io_flags) {
363
526
  uint32_t h;
364
527
  READ1(h);
365
528
  if (h == fourcc("il00")) {
366
- fprintf(stderr,
367
- "read_InvertedLists:"
368
- " WARN! inverted lists not stored with IVF object\n");
529
+ if (index_read_warn_on_null_invlists) {
530
+ fprintf(stderr,
531
+ "read_InvertedLists:"
532
+ " WARN! inverted lists not stored with IVF object\n");
533
+ }
369
534
  return nullptr;
370
535
  } else if (h == fourcc("ilpn") && !(io_flags & IO_FLAG_SKIP_IVF_DATA)) {
371
536
  size_t nlist, code_size, n_levels;
372
537
  READ1(nlist);
538
+ FAISS_CHECK_DESERIALIZATION_LOOP_LIMIT(nlist, "ilpn nlist");
373
539
  READ1(code_size);
374
540
  READ1(n_levels);
541
+ FAISS_THROW_IF_NOT_FMT(
542
+ n_levels > 0, "invalid ilpn n_levels %zd", n_levels);
543
+ constexpr size_t bs = Panorama::kDefaultBatchSize;
375
544
  auto ailp = std::make_unique<ArrayInvertedListsPanorama>(
376
- nlist, code_size, n_levels);
545
+ nlist, code_size, n_levels, bs);
377
546
  std::vector<size_t> sizes(nlist);
378
547
  read_ArrayInvertedLists_sizes(f, sizes);
548
+ // Do resize + read in a single pass per list. See the matching
549
+ // comment in the `ilar` branch below for rationale.
550
+ size_t byte_limit = get_deserialization_vector_byte_limit();
379
551
  for (size_t i = 0; i < nlist; i++) {
552
+ size_t n = sizes[i];
553
+ FAISS_THROW_IF_NOT_FMT(
554
+ n <= byte_limit / sizeof(idx_t),
555
+ "inverted list %zu ids size %zu exceeds "
556
+ "deserialization byte limit",
557
+ i,
558
+ n);
559
+ ailp->ids[i].resize(n);
560
+ size_t num_elems = ((n + bs - 1) / bs) * bs;
561
+ size_t codes_bytes = mul_no_overflow(
562
+ num_elems, code_size, "inverted list codes");
563
+ FAISS_THROW_IF_NOT_FMT(
564
+ codes_bytes <= byte_limit,
565
+ "inverted list %zu codes size %zu exceeds "
566
+ "deserialization byte limit",
567
+ i,
568
+ codes_bytes);
569
+ ailp->codes[i].resize(codes_bytes);
570
+ size_t cum_sums_count = mul_no_overflow(
571
+ num_elems,
572
+ add_no_overflow(
573
+ n_levels, 1, "inverted list cum_sums n_levels"),
574
+ "inverted list cum_sums");
575
+ FAISS_THROW_IF_NOT_FMT(
576
+ cum_sums_count <= byte_limit / sizeof(ailp->cum_sums[0][0]),
577
+ "inverted list %zu cum_sums size %zu exceeds "
578
+ "deserialization byte limit",
579
+ i,
580
+ cum_sums_count);
581
+ ailp->cum_sums[i].resize(cum_sums_count);
582
+ if (n > 0) {
583
+ read_vector_with_known_size(
584
+ ailp->codes[i], f, ailp->codes[i].size());
585
+ read_vector_with_known_size(ailp->ids[i], f, n);
586
+ read_vector_with_known_size(
587
+ ailp->cum_sums[i], f, ailp->cum_sums[i].size());
588
+ }
589
+ }
590
+ return ailp;
591
+ } else if (h == fourcc("ilp2") && !(io_flags & IO_FLAG_SKIP_IVF_DATA)) {
592
+ size_t nlist, code_size, n_levels, bs;
593
+ READ1(nlist);
594
+ FAISS_CHECK_DESERIALIZATION_LOOP_LIMIT(nlist, "ilp2 nlist");
595
+ READ1(code_size);
596
+ READ1(n_levels);
597
+ READ1(bs);
598
+ FAISS_THROW_IF_NOT_FMT(
599
+ n_levels > 0, "invalid ilp2 n_levels %zd", n_levels);
600
+ FAISS_THROW_IF_NOT_FMT(bs > 0, "invalid ilp2 batch_size %zd", bs);
601
+ auto ailp = std::make_unique<ArrayInvertedListsPanorama>(
602
+ nlist, code_size, n_levels, bs);
603
+ std::vector<size_t> sizes(nlist);
604
+ read_ArrayInvertedLists_sizes(f, sizes);
605
+ size_t byte_limit = get_deserialization_vector_byte_limit();
606
+ for (size_t i = 0; i < nlist; i++) {
607
+ FAISS_THROW_IF_NOT_FMT(
608
+ sizes[i] <= byte_limit / sizeof(idx_t),
609
+ "inverted list %zu ids size %zu exceeds "
610
+ "deserialization byte limit",
611
+ i,
612
+ sizes[i]);
380
613
  ailp->ids[i].resize(sizes[i]);
381
- size_t num_elems =
382
- ((sizes[i] + ArrayInvertedListsPanorama::kBatchSize - 1) /
383
- ArrayInvertedListsPanorama::kBatchSize) *
384
- ArrayInvertedListsPanorama::kBatchSize;
385
- ailp->codes[i].resize(num_elems * code_size);
386
- ailp->cum_sums[i].resize(num_elems * (n_levels + 1));
614
+ size_t num_elems = ((sizes[i] + bs - 1) / bs) * bs;
615
+ size_t codes_bytes = mul_no_overflow(
616
+ num_elems, code_size, "inverted list codes");
617
+ FAISS_THROW_IF_NOT_FMT(
618
+ codes_bytes <= byte_limit,
619
+ "inverted list %zu codes size %zu exceeds "
620
+ "deserialization byte limit",
621
+ i,
622
+ codes_bytes);
623
+ ailp->codes[i].resize(codes_bytes);
624
+ size_t cum_sums_count = mul_no_overflow(
625
+ num_elems,
626
+ add_no_overflow(
627
+ n_levels, 1, "inverted list cum_sums n_levels"),
628
+ "inverted list cum_sums");
629
+ FAISS_THROW_IF_NOT_FMT(
630
+ cum_sums_count <= byte_limit / sizeof(ailp->cum_sums[0][0]),
631
+ "inverted list %zu cum_sums size %zu exceeds "
632
+ "deserialization byte limit",
633
+ i,
634
+ cum_sums_count);
635
+ ailp->cum_sums[i].resize(cum_sums_count);
387
636
  }
388
637
  for (size_t i = 0; i < nlist; i++) {
389
638
  size_t n = sizes[i];
@@ -399,18 +648,36 @@ std::unique_ptr<InvertedLists> read_InvertedLists_up(
399
648
  } else if (h == fourcc("ilar") && !(io_flags & IO_FLAG_SKIP_IVF_DATA)) {
400
649
  auto ails = std::make_unique<ArrayInvertedLists>(0, 0);
401
650
  READ1(ails->nlist);
651
+ FAISS_CHECK_DESERIALIZATION_LOOP_LIMIT(ails->nlist, "ilar nlist");
402
652
  READ1(ails->code_size);
403
653
  ails->ids.resize(ails->nlist);
404
654
  ails->codes.resize(ails->nlist);
405
655
  std::vector<size_t> sizes(ails->nlist);
406
656
  read_ArrayInvertedLists_sizes(f, sizes);
407
- for (size_t i = 0; i < ails->nlist; i++) {
408
- ails->ids[i].resize(sizes[i]);
409
- ails->codes[i].resize(mul_no_overflow(
410
- sizes[i], ails->code_size, "inverted list codes"));
411
- }
412
- for (size_t i = 0; i < ails->nlist; i++) {
413
- size_t n = ails->ids[i].size();
657
+ // Resize + read in a single pass per list so that each list's
658
+ // heap allocation is released by the mmap view-substitution
659
+ // before the next list is allocated. This bounds peak heap to
660
+ // one list's worth of memory, which matters for large IVF
661
+ // indexes (hundreds of GB) under IO_FLAG_MMAP_IFC.
662
+ size_t ilar_byte_limit = get_deserialization_vector_byte_limit();
663
+ for (size_t i = 0; i < sizes.size(); i++) {
664
+ size_t n = sizes[i];
665
+ FAISS_THROW_IF_NOT_FMT(
666
+ n <= ilar_byte_limit / sizeof(idx_t),
667
+ "inverted list %zu ids size %zu exceeds "
668
+ "deserialization byte limit",
669
+ i,
670
+ n);
671
+ ails->ids[i].resize(n);
672
+ size_t codes_bytes =
673
+ mul_no_overflow(n, ails->code_size, "inverted list codes");
674
+ FAISS_THROW_IF_NOT_FMT(
675
+ codes_bytes <= ilar_byte_limit,
676
+ "inverted list %zu codes size %zu exceeds "
677
+ "deserialization byte limit",
678
+ i,
679
+ codes_bytes);
680
+ ails->codes[i].resize(codes_bytes);
414
681
  if (n > 0) {
415
682
  read_vector_with_known_size(
416
683
  ails->codes[i],
@@ -429,6 +696,7 @@ std::unique_ptr<InvertedLists> read_InvertedLists_up(
429
696
  int h2 = (io_flags & 0xffff0000) | (fourcc("il__") & 0x0000ffff);
430
697
  size_t nlist, code_size;
431
698
  READ1(nlist);
699
+ FAISS_CHECK_DESERIALIZATION_LOOP_LIMIT(nlist, "ilar skip nlist");
432
700
  READ1(code_size);
433
701
  std::vector<size_t> sizes(nlist);
434
702
  read_ArrayInvertedLists_sizes(f, sizes);
@@ -463,20 +731,47 @@ void read_ProductQuantizer(ProductQuantizer* pq, IOReader* f) {
463
731
  READ1(pq->nbits);
464
732
  FAISS_THROW_IF_NOT_FMT(
465
733
  pq->M > 0, "invalid ProductQuantizer M=%zd (must be > 0)", pq->M);
734
+ FAISS_THROW_IF_NOT_FMT(
735
+ pq->nbits <= 24, "invalid ProductQuantizer nbits=%zd", pq->nbits);
736
+ {
737
+ size_t ksub = size_t{1} << pq->nbits;
738
+ size_t n = mul_no_overflow(pq->d, ksub, "PQ centroids");
739
+ FAISS_THROW_IF_NOT_MSG(
740
+ n < get_deserialization_vector_byte_limit() / sizeof(float),
741
+ "PQ centroids allocation would exceed deserialization byte limit");
742
+ // Per-subquantizer tables (e.g. IVFPQ residual norms, search-time
743
+ // distance tables) are sized M * ksub.
744
+ size_t m_ksub = mul_no_overflow(pq->M, ksub, "PQ M*ksub");
745
+ FAISS_THROW_IF_NOT_MSG(
746
+ m_ksub <
747
+ get_deserialization_vector_byte_limit() / sizeof(float),
748
+ "PQ M*ksub allocation would exceed deserialization byte limit");
749
+ }
466
750
  pq->set_derived_values();
467
751
  READVECTOR(pq->centroids);
752
+ FAISS_THROW_IF_NOT_FMT(
753
+ pq->centroids.size() == pq->d * pq->ksub,
754
+ "ProductQuantizer centroids size %zu != d * ksub (%zu * %zu = %zu)",
755
+ pq->centroids.size(),
756
+ pq->d,
757
+ pq->ksub,
758
+ pq->d * pq->ksub);
468
759
  }
469
760
 
470
761
  static void read_ResidualQuantizer_old(ResidualQuantizer& rq, IOReader* f) {
471
762
  READ1(rq.d);
763
+ FAISS_THROW_IF_NOT_FMT(
764
+ rq.d > 0, "invalid AdditiveQuantizer d %zd, must be > 0", rq.d);
472
765
  READ1(rq.M);
766
+ FAISS_THROW_IF_NOT_FMT(
767
+ rq.M > 0, "invalid AdditiveQuantizer M %zd, must be > 0", rq.M);
473
768
  READVECTOR(rq.nbits);
474
769
  FAISS_THROW_IF_NOT_FMT(
475
770
  rq.nbits.size() == rq.M,
476
771
  "ResidualQuantizer nbits size %zd != M %zd",
477
772
  rq.nbits.size(),
478
773
  rq.M);
479
- READ1(rq.is_trained);
774
+ READ1_BOOL(rq.is_trained);
480
775
  READ1(rq.train_type);
481
776
  READ1(rq.max_beam_size);
482
777
  READVECTOR(rq.codebooks);
@@ -488,9 +783,13 @@ static void read_ResidualQuantizer_old(ResidualQuantizer& rq, IOReader* f) {
488
783
 
489
784
  static void read_AdditiveQuantizer(AdditiveQuantizer& aq, IOReader* f) {
490
785
  READ1(aq.d);
786
+ FAISS_THROW_IF_NOT_FMT(
787
+ aq.d > 0, "invalid AdditiveQuantizer d %zd, must be > 0", aq.d);
491
788
  READ1(aq.M);
789
+ FAISS_THROW_IF_NOT_FMT(
790
+ aq.M > 0, "invalid AdditiveQuantizer M %zd, must be > 0", aq.M);
492
791
  READVECTOR(aq.nbits);
493
- READ1(aq.is_trained);
792
+ READ1_BOOL(aq.is_trained);
494
793
  READVECTOR(aq.codebooks);
495
794
  FAISS_THROW_IF_NOT_FMT(
496
795
  aq.nbits.size() == aq.M,
@@ -515,6 +814,97 @@ static void read_AdditiveQuantizer(AdditiveQuantizer& aq, IOReader* f) {
515
814
  }
516
815
 
517
816
  aq.set_derived_values();
817
+
818
+ // Sanity-check codebooks size without knowing the effective dimension.
819
+ // codebooks stores effective_d * total_codebook_size floats, so its
820
+ // size must be a positive multiple of total_codebook_size.
821
+ if (aq.total_codebook_size > 0) {
822
+ FAISS_THROW_IF_NOT_FMT(
823
+ aq.codebooks.size() >= aq.total_codebook_size &&
824
+ aq.codebooks.size() % aq.total_codebook_size == 0,
825
+ "AdditiveQuantizer codebooks size %zd is not a positive "
826
+ "multiple of total_codebook_size %zd",
827
+ aq.codebooks.size(),
828
+ aq.total_codebook_size);
829
+ }
830
+ }
831
+
832
+ // Validate that the codebooks vector is large enough for the given
833
+ // effective dimension. For a standalone AdditiveQuantizer the effective
834
+ // dimension equals aq.d. For a ProductAdditiveQuantizer the codebooks
835
+ // are sized for d_sub = d / nsplits, so callers pass that instead.
836
+ static void validate_codebooks_size(
837
+ const AdditiveQuantizer& aq,
838
+ size_t effective_d) {
839
+ size_t required = mul_no_overflow(
840
+ effective_d, aq.total_codebook_size, "codebooks validation");
841
+ FAISS_THROW_IF_NOT_FMT(
842
+ aq.codebooks.size() >= required,
843
+ "AdditiveQuantizer codebooks size %zd too small for "
844
+ "d=%zd * total_codebook_size=%zd",
845
+ aq.codebooks.size(),
846
+ effective_d,
847
+ aq.total_codebook_size);
848
+ }
849
+
850
+ // Validate FastScan fields shared by all FastScan index types.
851
+ // M, ksub, bbs must be positive; bbs must be 32-aligned; M2 must be
852
+ // roundup(M, 2); and ksub * M / ksub * M2 must not overflow.
853
+ static void validate_fastscan_fields(
854
+ size_t M,
855
+ size_t M2,
856
+ size_t ksub,
857
+ int bbs,
858
+ const char* index_type) {
859
+ FAISS_THROW_IF_NOT_FMT(
860
+ M > 0 && ksub > 0,
861
+ "%s: invalid quantizer state (M=%zd, ksub=%zd, must be > 0)",
862
+ index_type,
863
+ M,
864
+ ksub);
865
+ FAISS_THROW_IF_NOT_FMT(
866
+ bbs > 0 && bbs % 32 == 0,
867
+ "%s: invalid bbs=%d (must be > 0 and a multiple of 32)",
868
+ index_type,
869
+ bbs);
870
+ size_t expected_M2 = (M + 1) & ~static_cast<size_t>(1); // roundup(M, 2)
871
+ FAISS_THROW_IF_NOT_FMT(
872
+ M2 == expected_M2,
873
+ "%s: invalid M2=%zd (expected roundup(M=%zd, 2) = %zd)",
874
+ index_type,
875
+ M2,
876
+ M,
877
+ expected_M2);
878
+ mul_no_overflow(ksub, M, index_type);
879
+ mul_no_overflow(ksub, M2, index_type);
880
+ }
881
+
882
+ // Validate that the AdditiveQuantizer dimension matches the index header
883
+ // dimension. compute_LUT() treats codebooks as a (d, total_codebook_size)
884
+ // matrix and query vectors are sized for idx_d, so a mismatch leads to
885
+ // out-of-bounds reads.
886
+ static void validate_aq_dimension_match(
887
+ const AdditiveQuantizer& aq,
888
+ int idx_d,
889
+ const char* index_type) {
890
+ FAISS_THROW_IF_NOT_FMT(
891
+ aq.d == static_cast<size_t>(idx_d),
892
+ "%s: AdditiveQuantizer d=%zd does not match index d=%d",
893
+ index_type,
894
+ aq.d,
895
+ idx_d);
896
+ }
897
+
898
+ static void validate_code_size_match(
899
+ size_t stored,
900
+ size_t expected,
901
+ const char* index_type) {
902
+ FAISS_THROW_IF_NOT_FMT(
903
+ stored == expected,
904
+ "%s code_size mismatch: stored %zd vs derived %zd",
905
+ index_type,
906
+ stored,
907
+ expected);
518
908
  }
519
909
 
520
910
  static void read_ResidualQuantizer(
@@ -522,8 +912,29 @@ static void read_ResidualQuantizer(
522
912
  IOReader* f,
523
913
  int io_flags) {
524
914
  read_AdditiveQuantizer(rq, f);
915
+ validate_codebooks_size(rq, rq.d);
525
916
  READ1(rq.train_type);
526
917
  READ1(rq.max_beam_size);
918
+ FAISS_THROW_IF_NOT_FMT(
919
+ rq.max_beam_size > 0,
920
+ "invalid max_beam_size %d, must be > 0",
921
+ rq.max_beam_size);
922
+ {
923
+ // Validate that the key allocation driven by max_beam_size
924
+ // (beam_size * M * sizeof(int32_t)) fits within the byte limit.
925
+ size_t beam_alloc = mul_no_overflow(
926
+ static_cast<size_t>(rq.max_beam_size),
927
+ rq.M,
928
+ "max_beam_size * M");
929
+ beam_alloc = mul_no_overflow(
930
+ beam_alloc, sizeof(int32_t), "max_beam_size * M * elem");
931
+ FAISS_THROW_IF_NOT_FMT(
932
+ beam_alloc < get_deserialization_vector_byte_limit(),
933
+ "max_beam_size %d * M %zd would exceed "
934
+ "deserialization vector byte limit",
935
+ rq.max_beam_size,
936
+ rq.M);
937
+ }
527
938
  if ((rq.train_type & ResidualQuantizer::Skip_codebook_tables) ||
528
939
  (io_flags & IO_FLAG_SKIP_PRECOMPUTE_TABLE)) {
529
940
  // don't precompute the tables
@@ -534,6 +945,7 @@ static void read_ResidualQuantizer(
534
945
 
535
946
  static void read_LocalSearchQuantizer(LocalSearchQuantizer& lsq, IOReader* f) {
536
947
  read_AdditiveQuantizer(lsq, f);
948
+ validate_codebooks_size(lsq, lsq.d);
537
949
  READ1(lsq.K);
538
950
  READ1(lsq.train_iters);
539
951
  READ1(lsq.encode_ils_iters);
@@ -552,6 +964,17 @@ static void read_ProductAdditiveQuantizer(
552
964
  IOReader* f) {
553
965
  read_AdditiveQuantizer(paq, f);
554
966
  READ1(paq.nsplits);
967
+ FAISS_THROW_IF_NOT_FMT(
968
+ paq.nsplits > 0,
969
+ "invalid ProductAdditiveQuantizer nsplits %zd (must be > 0)",
970
+ paq.nsplits);
971
+ FAISS_CHECK_DESERIALIZATION_LOOP_LIMIT(paq.nsplits, "nsplits");
972
+ FAISS_THROW_IF_NOT_FMT(
973
+ paq.d % paq.nsplits == 0,
974
+ "ProductAdditiveQuantizer d=%zd not divisible by nsplits=%zd",
975
+ paq.d,
976
+ paq.nsplits);
977
+ validate_codebooks_size(paq, paq.d / paq.nsplits);
555
978
  }
556
979
 
557
980
  static void read_ProductResidualQuantizer(
@@ -560,9 +983,19 @@ static void read_ProductResidualQuantizer(
560
983
  int io_flags) {
561
984
  read_ProductAdditiveQuantizer(prq, f);
562
985
 
986
+ size_t d_sub = prq.d / prq.nsplits;
563
987
  for (size_t i = 0; i < prq.nsplits; i++) {
564
988
  auto rq = std::make_unique<ResidualQuantizer>();
565
989
  read_ResidualQuantizer(*rq, f, io_flags);
990
+ FAISS_THROW_IF_NOT_FMT(
991
+ rq->d == d_sub,
992
+ "ProductResidualQuantizer sub-quantizer %zd has d=%zd, "
993
+ "expected d_sub=%zd (d=%zd / nsplits=%zd)",
994
+ i,
995
+ rq->d,
996
+ d_sub,
997
+ prq.d,
998
+ prq.nsplits);
566
999
  prq.quantizers.push_back(rq.release());
567
1000
  }
568
1001
  }
@@ -572,21 +1005,137 @@ static void read_ProductLocalSearchQuantizer(
572
1005
  IOReader* f) {
573
1006
  read_ProductAdditiveQuantizer(plsq, f);
574
1007
 
1008
+ size_t d_sub = plsq.d / plsq.nsplits;
575
1009
  for (size_t i = 0; i < plsq.nsplits; i++) {
576
1010
  auto lsq = std::make_unique<LocalSearchQuantizer>();
577
1011
  read_LocalSearchQuantizer(*lsq, f);
1012
+ FAISS_THROW_IF_NOT_FMT(
1013
+ lsq->d == d_sub,
1014
+ "ProductLocalSearchQuantizer sub-quantizer %zd has d=%zd, "
1015
+ "expected d_sub=%zd (d=%zd / nsplits=%zd)",
1016
+ i,
1017
+ lsq->d,
1018
+ d_sub,
1019
+ plsq.d,
1020
+ plsq.nsplits);
578
1021
  plsq.quantizers.push_back(lsq.release());
579
1022
  }
580
1023
  }
581
1024
 
582
- void read_ScalarQuantizer(ScalarQuantizer* ivsc, IOReader* f) {
583
- READ1(ivsc->qtype);
1025
+ void read_ScalarQuantizer(
1026
+ ScalarQuantizer* ivsc,
1027
+ IOReader* f,
1028
+ const Index& idx) {
1029
+ int qtype_int;
1030
+ READ1(qtype_int);
1031
+ FAISS_THROW_IF_NOT_FMT(
1032
+ qtype_int >= ScalarQuantizer::QT_8bit &&
1033
+ qtype_int < ScalarQuantizer::QT_count,
1034
+ "invalid ScalarQuantizer qtype %d",
1035
+ qtype_int);
1036
+ ivsc->qtype = static_cast<ScalarQuantizer::QuantizerType>(qtype_int);
584
1037
  READ1(ivsc->rangestat);
585
1038
  READ1(ivsc->rangestat_arg);
586
1039
  READ1(ivsc->d);
587
1040
  READ1(ivsc->code_size);
1041
+ FAISS_THROW_IF_NOT_FMT(
1042
+ static_cast<size_t>(idx.d) == ivsc->d,
1043
+ "ScalarQuantizer d %zu != index header d %d",
1044
+ ivsc->d,
1045
+ idx.d);
588
1046
  READVECTOR(ivsc->trained);
1047
+ // Populate bits/code_size before the validation block uses ivsc->bits.
589
1048
  ivsc->set_derived_sizes();
1049
+ // Validate trained vector size matches the quantizer type and dimension.
1050
+ // UNIFORM/NON_UNIFORM qtypes require training data; other qtypes
1051
+ // (fp16, bf16, 8bit_direct*) need none.
1052
+ // An untrained index (is_trained == false) legitimately has
1053
+ // trained.size() == 0, so we allow that case.
1054
+ {
1055
+ size_t expected = 0;
1056
+ switch (ivsc->qtype) {
1057
+ case ScalarQuantizer::QT_4bit_uniform:
1058
+ case ScalarQuantizer::QT_8bit_uniform:
1059
+ expected = 2;
1060
+ break;
1061
+ case ScalarQuantizer::QT_4bit:
1062
+ case ScalarQuantizer::QT_8bit:
1063
+ case ScalarQuantizer::QT_6bit:
1064
+ expected = 2 * ivsc->d;
1065
+ break;
1066
+ case ScalarQuantizer::QT_fp16:
1067
+ case ScalarQuantizer::QT_bf16:
1068
+ case ScalarQuantizer::QT_8bit_direct:
1069
+ case ScalarQuantizer::QT_8bit_direct_signed:
1070
+ case ScalarQuantizer::QT_0bit:
1071
+ case ScalarQuantizer::QT_count:
1072
+ expected = 0;
1073
+ break;
1074
+ case ScalarQuantizer::QT_1bit_tqmse:
1075
+ expected = 2 + 1; // 2^bits centroids + (2^bits - 1) boundaries
1076
+ break;
1077
+ case ScalarQuantizer::QT_2bit_tqmse:
1078
+ expected = 4 + 3;
1079
+ break;
1080
+ case ScalarQuantizer::QT_3bit_tqmse:
1081
+ expected = 8 + 7;
1082
+ break;
1083
+ case ScalarQuantizer::QT_4bit_tqmse:
1084
+ expected = 16 + 15;
1085
+ break;
1086
+ case ScalarQuantizer::QT_8bit_tqmse:
1087
+ expected = 256 + 255;
1088
+ break;
1089
+ case ScalarQuantizer::QT_2bit_tq:
1090
+ case ScalarQuantizer::QT_3bit_tq:
1091
+ case ScalarQuantizer::QT_4bit_tq:
1092
+ case ScalarQuantizer::QT_5bit_tq: {
1093
+ // k centroids + (k-1) boundaries + 3 extra (seed + qjl_type)
1094
+ size_t mse_bits = ivsc->bits - 1;
1095
+ size_t k = size_t(1) << mse_bits;
1096
+ expected = k + (k - 1) + 3;
1097
+ break;
1098
+ }
1099
+ }
1100
+ if (ivsc->trained.empty() && expected > 0) {
1101
+ // Empty trained is only valid for untrained indices.
1102
+ FAISS_THROW_IF_NOT_FMT(
1103
+ !idx.is_trained,
1104
+ "ScalarQuantizer trained size 0 != expected %zu "
1105
+ "for qtype %d, d %zu (index is marked as trained)",
1106
+ expected,
1107
+ (int)ivsc->qtype,
1108
+ ivsc->d);
1109
+ } else {
1110
+ FAISS_THROW_IF_NOT_FMT(
1111
+ ivsc->trained.size() == expected,
1112
+ "ScalarQuantizer trained size %zu != expected %zu "
1113
+ "for qtype %d, d %zu",
1114
+ ivsc->trained.size(),
1115
+ expected,
1116
+ (int)ivsc->qtype,
1117
+ ivsc->d);
1118
+ if (expected > 0) {
1119
+ FAISS_THROW_IF_NOT_MSG(
1120
+ idx.is_trained,
1121
+ "ScalarQuantizer has training data but "
1122
+ "index header is_trained is false");
1123
+ }
1124
+ }
1125
+ }
1126
+
1127
+ // TurboQ full types: extract seed and qjl_type from trained,
1128
+ // regenerate projection matrix.
1129
+ if (ScalarQuantizer::TurboQuantRefine::is_turboq_full(ivsc->qtype) &&
1130
+ ivsc->trained.size() >= 3) {
1131
+ size_t n = ivsc->trained.size();
1132
+ ivsc->turboq_refine.qjl_type =
1133
+ static_cast<uint8_t>(ivsc->trained[n - 1]);
1134
+ ivsc->turboq_refine.seed =
1135
+ ScalarQuantizer::TurboQuantRefine::unpack_seed(
1136
+ ivsc->trained[n - 3], ivsc->trained[n - 2]);
1137
+ ivsc->turboq_refine.init_projection(ivsc->d);
1138
+ }
590
1139
  }
591
1140
 
592
1141
  static void validate_HNSW(const HNSW& hnsw) {
@@ -723,11 +1272,14 @@ static void read_HNSW(HNSW& hnsw, IOReader* f) {
723
1272
  static void read_NSG(NSG& nsg, IOReader* f) {
724
1273
  READ1(nsg.ntotal);
725
1274
  READ1(nsg.R);
1275
+ FAISS_CHECK_DESERIALIZATION_LOOP_LIMIT(nsg.ntotal, "nsg.ntotal");
1276
+ FAISS_CHECK_DESERIALIZATION_LOOP_LIMIT(nsg.R, "nsg.R");
1277
+ FAISS_THROW_IF_NOT_FMT(nsg.R > 0, "invalid NSG R %d (must be > 0)", nsg.R);
726
1278
  READ1(nsg.L);
727
1279
  READ1(nsg.C);
728
1280
  READ1(nsg.search_L);
729
1281
  READ1(nsg.enterpoint);
730
- READ1(nsg.is_built);
1282
+ READ1_BOOL(nsg.is_built);
731
1283
 
732
1284
  FAISS_THROW_IF_NOT_FMT(
733
1285
  nsg.ntotal >= 0, "invalid NSG ntotal %d", nsg.ntotal);
@@ -779,12 +1331,32 @@ static void read_NNDescent(NNDescent& nnd, IOReader* f) {
779
1331
  READ1(nnd.iter);
780
1332
  READ1(nnd.search_L);
781
1333
  READ1(nnd.random_seed);
782
- READ1(nnd.has_built);
1334
+ READ1_BOOL(nnd.has_built);
783
1335
 
784
1336
  FAISS_THROW_IF_NOT_FMT(
785
1337
  nnd.ntotal >= 0, "invalid NNDescent ntotal %d", nnd.ntotal);
786
1338
 
787
1339
  READVECTOR(nnd.final_graph);
1340
+ // Validate neighbor IDs in the graph
1341
+ if (nnd.has_built && nnd.K > 0 && nnd.ntotal > 0) {
1342
+ FAISS_THROW_IF_NOT_FMT(
1343
+ nnd.final_graph.size() == (size_t)nnd.ntotal * (size_t)nnd.K,
1344
+ "NNDescent final_graph size %zu != ntotal * K (%d * %d = %zu)",
1345
+ nnd.final_graph.size(),
1346
+ nnd.ntotal,
1347
+ nnd.K,
1348
+ (size_t)nnd.ntotal * (size_t)nnd.K);
1349
+ for (size_t i = 0; i < nnd.final_graph.size(); i++) {
1350
+ int id = nnd.final_graph[i];
1351
+ FAISS_THROW_IF_NOT_FMT(
1352
+ id >= -1 && id < nnd.ntotal,
1353
+ "NNDescent final_graph[%zu] = %d out of range "
1354
+ "[-1, %d)",
1355
+ i,
1356
+ id,
1357
+ nnd.ntotal);
1358
+ }
1359
+ }
788
1360
  }
789
1361
 
790
1362
  std::unique_ptr<ProductQuantizer> read_ProductQuantizer_up(const char* fname) {
@@ -809,6 +1381,7 @@ ProductQuantizer* read_ProductQuantizer(IOReader* reader) {
809
1381
  static void read_RaBitQuantizer(
810
1382
  RaBitQuantizer& rabitq,
811
1383
  IOReader* f,
1384
+ int expected_d,
812
1385
  bool multi_bit = true) {
813
1386
  READ1(rabitq.d);
814
1387
  READ1(rabitq.code_size);
@@ -821,6 +1394,12 @@ static void read_RaBitQuantizer(
821
1394
  } else {
822
1395
  rabitq.nb_bits = 1;
823
1396
  }
1397
+
1398
+ FAISS_THROW_IF_NOT_FMT(
1399
+ rabitq.d == static_cast<size_t>(expected_d),
1400
+ "RaBitQuantizer dimension mismatch: rabitq.d=%zu vs index d=%d",
1401
+ rabitq.d,
1402
+ expected_d);
824
1403
  }
825
1404
 
826
1405
  void read_direct_map(DirectMap* dm, IOReader* f) {
@@ -845,6 +1424,7 @@ void read_ivf_header(
845
1424
  std::vector<std::vector<idx_t>>* ids) {
846
1425
  read_index_header(*ivf, f);
847
1426
  READ1(ivf->nlist);
1427
+ FAISS_CHECK_DESERIALIZATION_LOOP_LIMIT(ivf->nlist, "nlist");
848
1428
  READ1(ivf->nprobe);
849
1429
  ivf->quantizer = read_index(f);
850
1430
  ivf->own_fields = true;
@@ -890,7 +1470,7 @@ static std::unique_ptr<IndexIVFPQ> read_ivfpq(
890
1470
 
891
1471
  std::vector<std::vector<idx_t>> ids;
892
1472
  read_ivf_header(ivpq.get(), f, legacy ? &ids : nullptr);
893
- READ1(ivpq->by_residual);
1473
+ READ1_BOOL(ivpq->by_residual);
894
1474
  READ1(ivpq->code_size);
895
1475
  read_ProductQuantizer(&ivpq->pq, f);
896
1476
 
@@ -915,6 +1495,20 @@ static std::unique_ptr<IndexIVFPQ> read_ivfpq(
915
1495
  read_ProductQuantizer(&ivfpqr->refine_pq, f);
916
1496
  READVECTOR(ivfpqr->refine_codes);
917
1497
  READ1(ivfpqr->k_factor);
1498
+ // k_factor multiplies k to size search-time allocations
1499
+ // (n * k * k_factor labels + distances). Defaults are 1
1500
+ // (IndexRefine) and 4 (IndexIVFPQR); AutoTune explores
1501
+ // powers-of-two up to 64. Cap at 1000 to leave ample
1502
+ // headroom beyond any known usage while still blocking
1503
+ // OOM from crafted files (same cap as beam_factor in
1504
+ // ResidualCoarseQuantizer).
1505
+ FAISS_THROW_IF_NOT_FMT(
1506
+ std::isfinite(ivfpqr->k_factor) &&
1507
+ ivfpqr->k_factor >= 1.0f &&
1508
+ ivfpqr->k_factor <= 1000.0f,
1509
+ "k_factor %.6g out of valid range [1, 1000]"
1510
+ " for IndexIVFPQR",
1511
+ ivfpqr->k_factor);
918
1512
  }
919
1513
  }
920
1514
  return ivpq;
@@ -945,7 +1539,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
945
1539
  d, n_levels, batch_size);
946
1540
  }
947
1541
  READ1(idxp->ntotal);
948
- READ1(idxp->is_trained);
1542
+ READ1_BOOL(idxp->is_trained);
949
1543
  READVECTOR(idxp->codes);
950
1544
  READVECTOR(idxp->cum_sums);
951
1545
  idxp->verbose = false;
@@ -970,11 +1564,15 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
970
1564
  auto idxl = std::make_unique<IndexLSH>();
971
1565
  read_index_header(*idxl, f);
972
1566
  READ1(idxl->nbits);
973
- READ1(idxl->rotate_data);
974
- READ1(idxl->train_thresholds);
1567
+ READ1_BOOL(idxl->rotate_data);
1568
+ READ1_BOOL(idxl->train_thresholds);
975
1569
  READVECTOR(idxl->thresholds);
976
1570
  int code_size_i;
977
1571
  READ1(code_size_i);
1572
+ FAISS_THROW_IF_NOT_FMT(
1573
+ code_size_i >= 0,
1574
+ "IndexLSH invalid code_size %d (must be >= 0)",
1575
+ code_size_i);
978
1576
  idxl->code_size = code_size_i;
979
1577
  if (h == fourcc("IxHE")) {
980
1578
  FAISS_THROW_IF_NOT_FMT(
@@ -985,6 +1583,8 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
985
1583
  // leak
986
1584
  idxl->code_size *= 8;
987
1585
  }
1586
+ validate_code_size_match(
1587
+ idxl->code_size, (idxl->nbits + 7) / 8, "IndexLSH");
988
1588
  {
989
1589
  // Read, dereference, discard.
990
1590
  auto sub_vt = read_VectorTransform_up(f);
@@ -1007,9 +1607,11 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1007
1607
  read_ProductQuantizer(&idxp->pq, f);
1008
1608
  idxp->code_size = idxp->pq.code_size;
1009
1609
  read_vector(idxp->codes, f);
1610
+ FAISS_THROW_IF_NOT(
1611
+ idxp->codes.size() == idxp->ntotal * idxp->code_size);
1010
1612
  if (h == fourcc("IxPo") || h == fourcc("IxPq")) {
1011
1613
  READ1(idxp->search_type);
1012
- READ1(idxp->encode_signs);
1614
+ READ1_BOOL(idxp->encode_signs);
1013
1615
  READ1(idxp->polysemous_ht);
1014
1616
  }
1015
1617
  // Old versions of PQ all had metric_type set to INNER_PRODUCT
@@ -1027,40 +1629,102 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1027
1629
  } else {
1028
1630
  read_ResidualQuantizer(idxr->rq, f, io_flags);
1029
1631
  }
1632
+ validate_aq_dimension_match(
1633
+ idxr->rq, idxr->d, "IndexResidualQuantizer");
1030
1634
  READ1(idxr->code_size);
1635
+ validate_code_size_match(
1636
+ idxr->code_size, idxr->rq.code_size, "IndexResidualQuantizer");
1031
1637
  read_vector(idxr->codes, f);
1638
+ FAISS_THROW_IF_NOT(
1639
+ idxr->codes.size() == idxr->ntotal * idxr->code_size);
1032
1640
  idx = std::move(idxr);
1033
1641
  } else if (h == fourcc("IxLS")) {
1034
1642
  auto idxr = std::make_unique<IndexLocalSearchQuantizer>();
1035
1643
  read_index_header(*idxr, f);
1036
1644
  read_LocalSearchQuantizer(idxr->lsq, f);
1645
+ validate_aq_dimension_match(
1646
+ idxr->lsq, idxr->d, "IndexLocalSearchQuantizer");
1037
1647
  READ1(idxr->code_size);
1648
+ validate_code_size_match(
1649
+ idxr->code_size,
1650
+ idxr->lsq.code_size,
1651
+ "IndexLocalSearchQuantizer");
1038
1652
  read_vector(idxr->codes, f);
1653
+ FAISS_THROW_IF_NOT(
1654
+ idxr->codes.size() == idxr->ntotal * idxr->code_size);
1039
1655
  idx = std::move(idxr);
1040
1656
  } else if (h == fourcc("IxPR")) {
1041
1657
  auto idxpr = std::make_unique<IndexProductResidualQuantizer>();
1042
1658
  read_index_header(*idxpr, f);
1043
1659
  read_ProductResidualQuantizer(idxpr->prq, f, io_flags);
1660
+ validate_aq_dimension_match(
1661
+ idxpr->prq, idxpr->d, "IndexProductResidualQuantizer");
1044
1662
  READ1(idxpr->code_size);
1663
+ validate_code_size_match(
1664
+ idxpr->code_size,
1665
+ idxpr->prq.code_size,
1666
+ "IndexProductResidualQuantizer");
1045
1667
  read_vector(idxpr->codes, f);
1668
+ FAISS_THROW_IF_NOT(
1669
+ idxpr->codes.size() == idxpr->ntotal * idxpr->code_size);
1046
1670
  idx = std::move(idxpr);
1047
1671
  } else if (h == fourcc("IxPL")) {
1048
1672
  auto idxpl = std::make_unique<IndexProductLocalSearchQuantizer>();
1049
1673
  read_index_header(*idxpl, f);
1050
1674
  read_ProductLocalSearchQuantizer(idxpl->plsq, f);
1675
+ validate_aq_dimension_match(
1676
+ idxpl->plsq, idxpl->d, "IndexProductLocalSearchQuantizer");
1051
1677
  READ1(idxpl->code_size);
1678
+ validate_code_size_match(
1679
+ idxpl->code_size,
1680
+ idxpl->plsq.code_size,
1681
+ "IndexProductLocalSearchQuantizer");
1052
1682
  read_vector(idxpl->codes, f);
1683
+ FAISS_THROW_IF_NOT(
1684
+ idxpl->codes.size() == idxpl->ntotal * idxpl->code_size);
1053
1685
  idx = std::move(idxpl);
1054
1686
  } else if (h == fourcc("ImRQ")) {
1055
1687
  auto idxr = std::make_unique<ResidualCoarseQuantizer>();
1056
1688
  read_index_header(*idxr, f);
1057
1689
  read_ResidualQuantizer(idxr->rq, f, io_flags);
1690
+ validate_aq_dimension_match(
1691
+ idxr->rq, idxr->d, "ResidualCoarseQuantizer");
1058
1692
  READ1(idxr->beam_factor);
1059
1693
  if (io_flags & IO_FLAG_SKIP_PRECOMPUTE_TABLE) {
1060
1694
  // then we force the beam factor to -1
1061
1695
  // which skips the table precomputation.
1062
1696
  idxr->beam_factor = -1;
1063
1697
  }
1698
+ FAISS_THROW_IF_NOT_MSG(
1699
+ static_cast<size_t>(idxr->ntotal) <
1700
+ get_deserialization_vector_byte_limit() / sizeof(float),
1701
+ "ResidualCoarseQuantizer centroid norms allocation would "
1702
+ "exceed deserialization byte limit");
1703
+ // Validate beam_factor to prevent overflow in search() where
1704
+ // beam_size = int(k * beam_factor) and allocations scale with it.
1705
+ if (idxr->beam_factor > 0) {
1706
+ FAISS_THROW_IF_NOT_FMT(
1707
+ idxr->beam_factor <= 1000.0f,
1708
+ "beam_factor %.6g is too large (max 1000)",
1709
+ idxr->beam_factor);
1710
+ }
1711
+ // Validate ntotal against byte limit: search() allocates
1712
+ // O(ntotal * M) when beam_size is capped to ntotal.
1713
+ {
1714
+ size_t ntotal_alloc = mul_no_overflow(
1715
+ static_cast<size_t>(idxr->ntotal),
1716
+ idxr->rq.M,
1717
+ "ntotal * M");
1718
+ ntotal_alloc = mul_no_overflow(
1719
+ ntotal_alloc, sizeof(int32_t), "ntotal * M * elem");
1720
+ FAISS_THROW_IF_NOT_FMT(
1721
+ ntotal_alloc < get_deserialization_vector_byte_limit(),
1722
+ "ResidualCoarseQuantizer ntotal %" PRId64
1723
+ " * M %zd would exceed "
1724
+ "deserialization vector byte limit",
1725
+ idxr->ntotal,
1726
+ idxr->rq.M);
1727
+ }
1064
1728
  idxr->set_beam_factor(idxr->beam_factor);
1065
1729
  idx = std::move(idxr);
1066
1730
  } else if (
@@ -1095,10 +1759,13 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1095
1759
  read_ProductResidualQuantizer(
1096
1760
  *(ProductResidualQuantizer*)idxaqfs->aq, f, io_flags);
1097
1761
  }
1762
+ validate_aq_dimension_match(
1763
+ *idxaqfs->aq, idxaqfs->d, "IndexAdditiveQuantizerFastScan");
1098
1764
 
1099
1765
  READ1(idxaqfs->implem);
1100
1766
  READ1(idxaqfs->bbs);
1101
1767
  READ1(idxaqfs->qbs);
1768
+ FAISS_THROW_IF_NOT_MSG(idxaqfs->qbs >= 0, "qbs must be non-negative");
1102
1769
 
1103
1770
  READ1(idxaqfs->M);
1104
1771
  READ1(idxaqfs->nbits);
@@ -1107,11 +1774,19 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1107
1774
  READ1(idxaqfs->ntotal2);
1108
1775
  READ1(idxaqfs->M2);
1109
1776
 
1110
- READ1(idxaqfs->rescale_norm);
1777
+ READ1_BOOL(idxaqfs->rescale_norm);
1111
1778
  READ1(idxaqfs->norm_scale);
1112
1779
  READ1(idxaqfs->max_train_points);
1113
1780
 
1114
1781
  READVECTOR(idxaqfs->codes);
1782
+
1783
+ validate_fastscan_fields(
1784
+ idxaqfs->M,
1785
+ idxaqfs->M2,
1786
+ idxaqfs->ksub,
1787
+ idxaqfs->bbs,
1788
+ "IndexAdditiveQuantizerFastScan");
1789
+
1115
1790
  idx = std::move(idxaqfs);
1116
1791
  } else if (
1117
1792
  h == fourcc("IVLf") || h == fourcc("IVRf") || h == fourcc("NPLf") ||
@@ -1146,11 +1821,14 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1146
1821
  read_ProductResidualQuantizer(
1147
1822
  *(ProductResidualQuantizer*)ivaqfs->aq, f, io_flags);
1148
1823
  }
1824
+ validate_aq_dimension_match(
1825
+ *ivaqfs->aq, ivaqfs->d, "IndexIVFAdditiveQuantizerFastScan");
1149
1826
 
1150
- READ1(ivaqfs->by_residual);
1827
+ READ1_BOOL(ivaqfs->by_residual);
1151
1828
  READ1(ivaqfs->implem);
1152
1829
  READ1(ivaqfs->bbs);
1153
1830
  READ1(ivaqfs->qbs);
1831
+ FAISS_THROW_IF_NOT_MSG(ivaqfs->qbs >= 0, "qbs must be non-negative");
1154
1832
 
1155
1833
  READ1(ivaqfs->M);
1156
1834
  READ1(ivaqfs->nbits);
@@ -1159,12 +1837,20 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1159
1837
  READ1(ivaqfs->qbs2);
1160
1838
  READ1(ivaqfs->M2);
1161
1839
 
1162
- READ1(ivaqfs->rescale_norm);
1840
+ READ1_BOOL(ivaqfs->rescale_norm);
1163
1841
  READ1(ivaqfs->norm_scale);
1164
1842
  READ1(ivaqfs->max_train_points);
1165
1843
 
1166
1844
  read_InvertedLists(*ivaqfs, f, io_flags);
1167
1845
  ivaqfs->init_code_packer();
1846
+
1847
+ validate_fastscan_fields(
1848
+ ivaqfs->M,
1849
+ ivaqfs->M2,
1850
+ ivaqfs->ksub,
1851
+ ivaqfs->bbs,
1852
+ "IndexIVFAdditiveQuantizerFastScan");
1853
+
1168
1854
  idx = std::move(ivaqfs);
1169
1855
  } else if (h == fourcc("IvFl") || h == fourcc("IvFL")) { // legacy
1170
1856
  auto ivfl = std::make_unique<IndexIVFFlat>();
@@ -1198,7 +1884,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1198
1884
  "invalid IVFFlatDedup instances table size: %zd "
1199
1885
  "(must be even)",
1200
1886
  tab.size());
1201
- for (long i = 0; i < tab.size(); i += 2) {
1887
+ for (size_t i = 0; i < tab.size(); i += 2) {
1202
1888
  std::pair<idx_t, idx_t> pair(tab[i], tab[i + 1]);
1203
1889
  ivfl->instances.insert(pair);
1204
1890
  }
@@ -1210,6 +1896,15 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1210
1896
  read_ivf_header(ivfp.get(), f);
1211
1897
  ivfp->code_size = ivfp->d * sizeof(float);
1212
1898
  READ1(ivfp->n_levels);
1899
+ ivfp->batch_size = Panorama::kDefaultBatchSize;
1900
+ read_InvertedLists(*ivfp, f, io_flags);
1901
+ idx = std::move(ivfp);
1902
+ } else if (h == fourcc("IwP2")) {
1903
+ auto ivfp = std::make_unique<IndexIVFFlatPanorama>();
1904
+ read_ivf_header(ivfp.get(), f);
1905
+ ivfp->code_size = ivfp->d * sizeof(float);
1906
+ READ1(ivfp->n_levels);
1907
+ READ1(ivfp->batch_size);
1213
1908
  read_InvertedLists(*ivfp, f, io_flags);
1214
1909
  idx = std::move(ivfp);
1215
1910
  } else if (h == fourcc("IwFl")) {
@@ -1221,7 +1916,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1221
1916
  } else if (h == fourcc("IxSQ")) {
1222
1917
  auto idxs = std::make_unique<IndexScalarQuantizer>();
1223
1918
  read_index_header(*idxs, f);
1224
- read_ScalarQuantizer(&idxs->sq, f);
1919
+ read_ScalarQuantizer(&idxs->sq, f, *idxs);
1225
1920
  read_vector(idxs->codes, f);
1226
1921
  idxs->code_size = idxs->sq.code_size;
1227
1922
  idx = std::move(idxs);
@@ -1240,6 +1935,24 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1240
1935
  nsq);
1241
1936
  FAISS_THROW_IF_NOT_FMT(
1242
1937
  r2 > 0, "invalid IndexLattice r2 %d (must be > 0)", r2);
1938
+ {
1939
+ // ZnSphereCodecRec constructor populates a decode cache
1940
+ // whose build cost grows polynomially in r2. The
1941
+ // in-codec memory cap (lattice_Zn.cpp) bounds the cache
1942
+ // size but not the CPU cost of building it, so for small
1943
+ // dsq the cap permits enough decode() iterations to far
1944
+ // exceed reasonable load-time budgets. Callers that
1945
+ // operate on untrusted index payloads can opt in to a
1946
+ // tighter bound via set_deserialization_lattice_r2_limit;
1947
+ // the default of 0 preserves existing behavior.
1948
+ auto limit_ = get_deserialization_lattice_r2_limit();
1949
+ FAISS_THROW_IF_NOT_FMT(
1950
+ limit_ == 0 || static_cast<size_t>(r2) <= limit_,
1951
+ "IndexLattice r2=%d exceeds "
1952
+ "deserialization_lattice_r2_limit of %zd",
1953
+ r2,
1954
+ limit_);
1955
+ }
1243
1956
  int dsq = d / nsq;
1244
1957
  FAISS_THROW_IF_NOT_FMT(
1245
1958
  dsq >= 2 && (dsq & (dsq - 1)) == 0,
@@ -1248,6 +1961,27 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1248
1961
  nsq,
1249
1962
  dsq);
1250
1963
  auto idxl = std::make_unique<IndexLattice>(d, nsq, scale_nbit, r2);
1964
+ // IndexLattice is a lossy compressor: code_size should be
1965
+ // smaller than the uncompressed vector (d floats). A corrupt
1966
+ // scale_nbit can overflow the total_nbit computation, producing
1967
+ // a code_size that wraps to a huge value.
1968
+ {
1969
+ size_t max_code_size = mul_no_overflow(
1970
+ static_cast<size_t>(d),
1971
+ sizeof(float),
1972
+ "IndexLattice uncompressed vector size");
1973
+ FAISS_THROW_IF_NOT_FMT(
1974
+ idxl->code_size <= max_code_size,
1975
+ "IndexLattice code_size %zd exceeds uncompressed "
1976
+ "vector size %zd (likely corrupt scale_nbit=%d, "
1977
+ "d=%d, nsq=%d, r2=%d)",
1978
+ idxl->code_size,
1979
+ max_code_size,
1980
+ scale_nbit,
1981
+ d,
1982
+ nsq,
1983
+ r2);
1984
+ }
1251
1985
  read_index_header(*idxl, f);
1252
1986
  READVECTOR(idxl->trained);
1253
1987
  idx = std::move(idxl);
@@ -1255,21 +1989,25 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1255
1989
  auto ivsc = std::make_unique<IndexIVFScalarQuantizer>();
1256
1990
  std::vector<std::vector<idx_t>> ids;
1257
1991
  read_ivf_header(ivsc.get(), f, &ids);
1258
- read_ScalarQuantizer(&ivsc->sq, f);
1992
+ read_ScalarQuantizer(&ivsc->sq, f, *ivsc);
1259
1993
  READ1(ivsc->code_size);
1994
+ validate_code_size_match(
1995
+ ivsc->code_size, ivsc->sq.code_size, "IndexIVFScalarQuantizer");
1260
1996
  ArrayInvertedLists* ail = set_array_invlist(ivsc.get(), ids);
1261
- for (int i = 0; i < ivsc->nlist; i++)
1997
+ for (size_t i = 0; i < ivsc->nlist; i++)
1262
1998
  READVECTOR(ail->codes[i]);
1263
1999
  idx = std::move(ivsc);
1264
2000
  } else if (h == fourcc("IwSQ") || h == fourcc("IwSq")) {
1265
2001
  auto ivsc = std::make_unique<IndexIVFScalarQuantizer>();
1266
2002
  read_ivf_header(ivsc.get(), f);
1267
- read_ScalarQuantizer(&ivsc->sq, f);
2003
+ read_ScalarQuantizer(&ivsc->sq, f, *ivsc);
1268
2004
  READ1(ivsc->code_size);
2005
+ validate_code_size_match(
2006
+ ivsc->code_size, ivsc->sq.code_size, "IndexIVFScalarQuantizer");
1269
2007
  if (h == fourcc("IwSQ")) {
1270
2008
  ivsc->by_residual = true;
1271
2009
  } else {
1272
- READ1(ivsc->by_residual);
2010
+ READ1_BOOL(ivsc->by_residual);
1273
2011
  }
1274
2012
  read_InvertedLists(*ivsc, f, io_flags);
1275
2013
  idx = std::move(ivsc);
@@ -1302,7 +2040,13 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1302
2040
  read_ProductResidualQuantizer(
1303
2041
  *(ProductResidualQuantizer*)iva->aq, f, io_flags);
1304
2042
  }
1305
- READ1(iva->by_residual);
2043
+ validate_aq_dimension_match(
2044
+ *iva->aq, iva->d, "IndexIVFAdditiveQuantizer");
2045
+ validate_code_size_match(
2046
+ iva->code_size,
2047
+ iva->aq->code_size,
2048
+ "IndexIVFAdditiveQuantizer");
2049
+ READ1_BOOL(iva->by_residual);
1306
2050
  READ1(iva->use_precomputed_table);
1307
2051
  read_InvertedLists(*iva, f, io_flags);
1308
2052
  idx = std::move(iva);
@@ -1329,7 +2073,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1329
2073
  read_index_header(*indep, f);
1330
2074
  indep->quantizer = read_index(f, io_flags);
1331
2075
  bool has_vt;
1332
- READ1(has_vt);
2076
+ READ1_BOOL(has_vt);
1333
2077
  if (has_vt) {
1334
2078
  indep->vt = read_VectorTransform(f);
1335
2079
  }
@@ -1337,6 +2081,18 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1337
2081
  indep->index_ivf = dynamic_cast<IndexIVF*>(ivf_idx.get());
1338
2082
  FAISS_THROW_IF_NOT(indep->index_ivf);
1339
2083
  ivf_idx.release();
2084
+ if (indep->vt) {
2085
+ FAISS_THROW_IF_NOT_FMT(
2086
+ indep->vt->d_in == indep->d,
2087
+ "IndexIVFIndependentQuantizer: vt->d_in (%d) != index d (%d)",
2088
+ indep->vt->d_in,
2089
+ indep->d);
2090
+ FAISS_THROW_IF_NOT_FMT(
2091
+ indep->vt->d_out == indep->index_ivf->d,
2092
+ "IndexIVFIndependentQuantizer: vt->d_out (%d) != index_ivf->d (%d)",
2093
+ indep->vt->d_out,
2094
+ indep->index_ivf->d);
2095
+ }
1340
2096
  if (auto index_ivfpq = dynamic_cast<IndexIVFPQ*>(indep->index_ivf)) {
1341
2097
  READ1(index_ivfpq->use_precomputed_table);
1342
2098
  }
@@ -1351,10 +2107,46 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1351
2107
  } else {
1352
2108
  READ1(nt);
1353
2109
  }
2110
+ FAISS_THROW_IF_NOT_FMT(
2111
+ nt >= 0,
2112
+ "invalid VectorTransform chain length %d (must be >= 0)",
2113
+ nt);
2114
+ FAISS_CHECK_DESERIALIZATION_LOOP_LIMIT(
2115
+ nt, "VectorTransform chain length");
1354
2116
  for (int i = 0; i < nt; i++) {
1355
2117
  ixpt->chain.push_back(read_VectorTransform(f));
1356
2118
  }
1357
2119
  ixpt->index = read_index(f, io_flags);
2120
+ // Validate transform chain dimension consistency:
2121
+ // chain[0].d_in must equal the outer index d, consecutive
2122
+ // transforms must have matching d_out/d_in, and the last
2123
+ // transform's d_out must equal the sub-index d.
2124
+ if (nt > 0) {
2125
+ FAISS_THROW_IF_NOT_FMT(
2126
+ ixpt->chain[0]->d_in == ixpt->d,
2127
+ "IndexPreTransform chain[0] d_in=%d != index d=%d",
2128
+ ixpt->chain[0]->d_in,
2129
+ ixpt->d);
2130
+ for (int i = 1; i < nt; i++) {
2131
+ FAISS_THROW_IF_NOT_FMT(
2132
+ ixpt->chain[i]->d_in == ixpt->chain[i - 1]->d_out,
2133
+ "IndexPreTransform chain[%d] d_in=%d != "
2134
+ "chain[%d] d_out=%d",
2135
+ i,
2136
+ ixpt->chain[i]->d_in,
2137
+ i - 1,
2138
+ ixpt->chain[i - 1]->d_out);
2139
+ }
2140
+ if (ixpt->index) {
2141
+ FAISS_THROW_IF_NOT_FMT(
2142
+ ixpt->chain[nt - 1]->d_out == ixpt->index->d,
2143
+ "IndexPreTransform chain[%d] d_out=%d "
2144
+ "!= sub-index d=%d",
2145
+ nt - 1,
2146
+ ixpt->chain[nt - 1]->d_out,
2147
+ ixpt->index->d);
2148
+ }
2149
+ }
1358
2150
  idx = std::move(ixpt);
1359
2151
  } else if (h == fourcc("Imiq")) {
1360
2152
  auto imiq = std::make_unique<MultiIndexQuantizer>();
@@ -1367,6 +2159,12 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1367
2159
  auto base = read_index_up(f, io_flags);
1368
2160
  auto refine = read_index_up(f, io_flags);
1369
2161
  READ1(idxrf->k_factor);
2162
+ // Same rationale as IndexIVFPQR k_factor above.
2163
+ FAISS_THROW_IF_NOT_FMT(
2164
+ std::isfinite(idxrf->k_factor) && idxrf->k_factor >= 1.0f &&
2165
+ idxrf->k_factor <= 1000.0f,
2166
+ "k_factor %.6g out of valid range [1, 1000] for IndexRefine",
2167
+ idxrf->k_factor);
1370
2168
  if (h == fourcc("IxRP")) {
1371
2169
  // then make a RefineFlatPanorama with it
1372
2170
  auto idxrf_new = std::make_unique<IndexRefinePanorama>();
@@ -1390,8 +2188,21 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1390
2188
  : std::make_unique<IndexIDMap>();
1391
2189
  read_index_header(*idxmap, f);
1392
2190
  idxmap->index = read_index(f, io_flags);
2191
+ FAISS_THROW_IF_NOT_MSG(idxmap->index, "IndexIDMap inner index is null");
1393
2192
  idxmap->own_fields = true;
1394
2193
  READVECTOR(idxmap->id_map);
2194
+ FAISS_THROW_IF_NOT_FMT(
2195
+ idxmap->id_map.size() == idxmap->ntotal,
2196
+ "IndexIDMap id_map size (%" PRId64
2197
+ ") does not match ntotal (%" PRId64 ")",
2198
+ int64_t(idxmap->id_map.size()),
2199
+ idxmap->ntotal);
2200
+ FAISS_THROW_IF_NOT_FMT(
2201
+ idxmap->index->ntotal == idxmap->ntotal,
2202
+ "IndexIDMap inner index ntotal (%" PRId64
2203
+ ") does not match IndexIDMap ntotal (%" PRId64 ")",
2204
+ idxmap->index->ntotal,
2205
+ idxmap->ntotal);
1395
2206
  if (is_map2) {
1396
2207
  static_cast<IndexIDMap2*>(idxmap.get())->construct_rev_map();
1397
2208
  }
@@ -1400,12 +2211,21 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1400
2211
  auto idxp = std::make_unique<Index2Layer>();
1401
2212
  read_index_header(*idxp, f);
1402
2213
  idxp->q1.quantizer = read_index(f, io_flags);
2214
+ idxp->q1.own_fields = true;
1403
2215
  READ1(idxp->q1.nlist);
1404
2216
  READ1(idxp->q1.quantizer_trains_alone);
1405
2217
  read_ProductQuantizer(&idxp->pq, f);
1406
2218
  READ1(idxp->code_size_1);
1407
2219
  READ1(idxp->code_size_2);
1408
2220
  READ1(idxp->code_size);
2221
+ validate_code_size_match(
2222
+ idxp->code_size_2,
2223
+ idxp->pq.code_size,
2224
+ "Index2Layer code_size_2");
2225
+ validate_code_size_match(
2226
+ idxp->code_size,
2227
+ idxp->code_size_1 + idxp->code_size_2,
2228
+ "Index2Layer");
1409
2229
  read_vector(idxp->codes, f);
1410
2230
  idx = std::move(idxp);
1411
2231
  } else if (
@@ -1438,6 +2258,9 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1438
2258
  if (h == fourcc("IHfP")) {
1439
2259
  auto idx_panorama =
1440
2260
  dynamic_cast<IndexHNSWFlatPanorama*>(idxhnsw.get());
2261
+ FAISS_THROW_IF_NOT_MSG(
2262
+ idx_panorama,
2263
+ "dynamic_cast to IndexHNSWFlatPanorama failed");
1441
2264
  size_t nlevels;
1442
2265
  READ1(nlevels);
1443
2266
  const_cast<size_t&>(idx_panorama->num_panorama_levels) = nlevels;
@@ -1446,9 +2269,11 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1446
2269
  READVECTOR(idx_panorama->cum_sums);
1447
2270
  }
1448
2271
  if (h == fourcc("IHNc") || h == fourcc("IHc2")) {
1449
- READ1(idxhnsw->keep_max_size_level0);
2272
+ READ1_BOOL(idxhnsw->keep_max_size_level0);
1450
2273
  auto idx_hnsw_cagra = dynamic_cast<IndexHNSWCagra*>(idxhnsw.get());
1451
- READ1(idx_hnsw_cagra->base_level_only);
2274
+ FAISS_THROW_IF_NOT_MSG(
2275
+ idx_hnsw_cagra, "dynamic_cast to IndexHNSWCagra failed");
2276
+ READ1_BOOL(idx_hnsw_cagra->base_level_only);
1452
2277
  READ1(idx_hnsw_cagra->num_base_level_search_entrypoints);
1453
2278
  if (h == fourcc("IHc2")) {
1454
2279
  READ1(idx_hnsw_cagra->numeric_type_);
@@ -1457,11 +2282,49 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1457
2282
  }
1458
2283
  }
1459
2284
  read_HNSW(idxhnsw->hnsw, f);
2285
+ // Cross-check HNSW graph size against index header ntotal
2286
+ FAISS_THROW_IF_NOT_FMT(
2287
+ idxhnsw->hnsw.levels.size() == (size_t)idxhnsw->ntotal,
2288
+ "HNSW levels size %zu != index ntotal %" PRId64,
2289
+ idxhnsw->hnsw.levels.size(),
2290
+ idxhnsw->ntotal);
1460
2291
  idxhnsw->hnsw.is_panorama = (h == fourcc("IHfP"));
2292
+ // `HNSW::is_similarity` is intentionally not serialized, so we
2293
+ // re-derive it here from the persisted metric type. Without this,
2294
+ // a saved IP/similarity index would come back configured as a
2295
+ // distance index and silently produce wrong rankings on search.
2296
+ idxhnsw->hnsw.is_similarity =
2297
+ is_similarity_metric(idxhnsw->metric_type);
1461
2298
  idxhnsw->storage = read_index(f, io_flags);
1462
2299
  idxhnsw->own_fields = idxhnsw->storage != nullptr;
2300
+ // Cross-check storage ntotal and d against index
2301
+ if (idxhnsw->storage) {
2302
+ FAISS_THROW_IF_NOT_FMT(
2303
+ idxhnsw->storage->ntotal == idxhnsw->ntotal,
2304
+ "HNSW storage ntotal %" PRId64 " != index ntotal %" PRId64,
2305
+ idxhnsw->storage->ntotal,
2306
+ idxhnsw->ntotal);
2307
+ FAISS_THROW_IF_NOT_FMT(
2308
+ idxhnsw->storage->d == idxhnsw->d,
2309
+ "HNSW storage d %d != index d %d",
2310
+ idxhnsw->storage->d,
2311
+ idxhnsw->d);
2312
+ }
2313
+ if (h == fourcc("IHN2")) {
2314
+ FAISS_THROW_IF_NOT_MSG(
2315
+ idxhnsw->storage,
2316
+ "IndexHNSW2Level requires non-null storage");
2317
+ FAISS_THROW_IF_NOT_MSG(
2318
+ dynamic_cast<Index2Layer*>(idxhnsw->storage) ||
2319
+ dynamic_cast<IndexIVFPQ*>(idxhnsw->storage),
2320
+ "IndexHNSW2Level storage must be Index2Layer or IndexIVFPQ");
2321
+ }
1463
2322
  if (h == fourcc("IHNp") && !(io_flags & IO_FLAG_PQ_SKIP_SDC_TABLE)) {
1464
- dynamic_cast<IndexPQ*>(idxhnsw->storage)->pq.compute_sdc_table();
2323
+ auto* storage_pq = dynamic_cast<IndexPQ*>(idxhnsw->storage);
2324
+ FAISS_THROW_IF_NOT_MSG(
2325
+ storage_pq,
2326
+ "dynamic_cast to IndexPQ failed for HNSW storage");
2327
+ storage_pq->pq.compute_sdc_table();
1465
2328
  }
1466
2329
  idx = std::move(idxhnsw);
1467
2330
  } else if (
@@ -1484,15 +2347,58 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1484
2347
  READ1(idxnsg->nndescent_L);
1485
2348
  READ1(idxnsg->nndescent_iter);
1486
2349
  read_NSG(idxnsg->nsg, f);
2350
+ // Cross-check NSG graph ntotal against index header ntotal
2351
+ if (idxnsg->nsg.is_built) {
2352
+ FAISS_THROW_IF_NOT_FMT(
2353
+ idxnsg->nsg.ntotal == idxnsg->ntotal,
2354
+ "NSG ntotal %d != index ntotal %" PRId64,
2355
+ idxnsg->nsg.ntotal,
2356
+ idxnsg->ntotal);
2357
+ }
1487
2358
  idxnsg->storage = read_index(f, io_flags);
1488
2359
  idxnsg->own_fields = true;
2360
+ // Cross-check storage ntotal and d against index
2361
+ if (idxnsg->storage) {
2362
+ FAISS_THROW_IF_NOT_FMT(
2363
+ idxnsg->storage->ntotal == idxnsg->ntotal,
2364
+ "NSG storage ntotal %" PRId64 " != index ntotal %" PRId64,
2365
+ idxnsg->storage->ntotal,
2366
+ idxnsg->ntotal);
2367
+ FAISS_THROW_IF_NOT_FMT(
2368
+ idxnsg->storage->d == idxnsg->d,
2369
+ "NSG storage d %d != index d %d",
2370
+ idxnsg->storage->d,
2371
+ idxnsg->d);
2372
+ }
1489
2373
  idx = std::move(idxnsg);
1490
2374
  } else if (h == fourcc("INNf")) {
1491
2375
  auto idxnnd = std::make_unique<IndexNNDescentFlat>();
1492
2376
  read_index_header(*idxnnd, f);
1493
2377
  read_NNDescent(idxnnd->nndescent, f);
2378
+ // Cross-check NNDescent ntotal against index header ntotal
2379
+ if (idxnnd->nndescent.has_built) {
2380
+ FAISS_THROW_IF_NOT_FMT(
2381
+ idxnnd->nndescent.ntotal == idxnnd->ntotal,
2382
+ "NNDescent ntotal %d != index ntotal %" PRId64,
2383
+ idxnnd->nndescent.ntotal,
2384
+ idxnnd->ntotal);
2385
+ }
1494
2386
  idxnnd->storage = read_index(f, io_flags);
1495
2387
  idxnnd->own_fields = true;
2388
+ // Cross-check storage ntotal and d against index
2389
+ if (idxnnd->storage) {
2390
+ FAISS_THROW_IF_NOT_FMT(
2391
+ idxnnd->storage->ntotal == idxnnd->ntotal,
2392
+ "NNDescent storage ntotal %" PRId64
2393
+ " != index ntotal %" PRId64,
2394
+ idxnnd->storage->ntotal,
2395
+ idxnnd->ntotal);
2396
+ FAISS_THROW_IF_NOT_FMT(
2397
+ idxnnd->storage->d == idxnnd->d,
2398
+ "NNDescent storage d %d != index d %d",
2399
+ idxnnd->storage->d,
2400
+ idxnnd->d);
2401
+ }
1496
2402
  idx = std::move(idxnnd);
1497
2403
  } else if (h == fourcc("IPfs")) {
1498
2404
  auto idxpqfs = std::make_unique<IndexPQFastScan>();
@@ -1501,6 +2407,7 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1501
2407
  READ1(idxpqfs->implem);
1502
2408
  READ1(idxpqfs->bbs);
1503
2409
  READ1(idxpqfs->qbs);
2410
+ FAISS_THROW_IF_NOT_MSG(idxpqfs->qbs >= 0, "qbs must be non-negative");
1504
2411
  READ1(idxpqfs->ntotal2);
1505
2412
  READ1(idxpqfs->M2);
1506
2413
  READVECTOR(idxpqfs->codes);
@@ -1511,12 +2418,19 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1511
2418
  idxpqfs->ksub = (1 << pq.nbits);
1512
2419
  idxpqfs->code_size = pq.code_size;
1513
2420
 
2421
+ validate_fastscan_fields(
2422
+ idxpqfs->M,
2423
+ idxpqfs->M2,
2424
+ idxpqfs->ksub,
2425
+ idxpqfs->bbs,
2426
+ "IndexPQFastScan");
2427
+
1514
2428
  idx = std::move(idxpqfs);
1515
2429
 
1516
2430
  } else if (h == fourcc("IwPf")) {
1517
2431
  auto ivpq = std::make_unique<IndexIVFPQFastScan>();
1518
2432
  read_ivf_header(ivpq.get(), f);
1519
- READ1(ivpq->by_residual);
2433
+ READ1_BOOL(ivpq->by_residual);
1520
2434
  READ1(ivpq->code_size);
1521
2435
  READ1(ivpq->bbs);
1522
2436
  READ1(ivpq->M2);
@@ -1533,6 +2447,9 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1533
2447
  ivpq->code_size = pq.code_size;
1534
2448
  ivpq->init_code_packer();
1535
2449
 
2450
+ validate_fastscan_fields(
2451
+ ivpq->M, ivpq->M2, ivpq->ksub, ivpq->bbs, "IndexIVFPQFastScan");
2452
+
1536
2453
  idx = std::move(ivpq);
1537
2454
  } else if (h == fourcc("IRMf")) {
1538
2455
  auto imm = std::make_unique<IndexRowwiseMinMax>();
@@ -1557,9 +2474,13 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1557
2474
 
1558
2475
  auto idxqfs = std::make_unique<IndexRaBitQFastScan>();
1559
2476
  read_index_header(*idxqfs, f);
1560
- read_RaBitQuantizer(idxqfs->rabitq, f, true);
2477
+ read_RaBitQuantizer(idxqfs->rabitq, f, idxqfs->d, true);
1561
2478
  READVECTOR(idxqfs->center);
1562
2479
  READ1(idxqfs->qb);
2480
+ FAISS_THROW_IF_NOT_FMT(
2481
+ idxqfs->qb > 0 && idxqfs->qb <= 8,
2482
+ "invalid RaBitQ qb=%d (must be in [1, 8])",
2483
+ idxqfs->qb);
1563
2484
 
1564
2485
  std::vector<uint8_t> legacy_flat_storage;
1565
2486
  if (is_legacy) {
@@ -1577,6 +2498,13 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1577
2498
  idxqfs->nbits = nbits_fastscan;
1578
2499
  idxqfs->ksub = (1 << nbits_fastscan);
1579
2500
 
2501
+ validate_fastscan_fields(
2502
+ idxqfs->M,
2503
+ idxqfs->M2,
2504
+ idxqfs->ksub,
2505
+ idxqfs->bbs,
2506
+ "IndexRaBitQFastScan");
2507
+
1580
2508
  READVECTOR(idxqfs->codes);
1581
2509
 
1582
2510
  if (is_legacy) {
@@ -1602,12 +2530,19 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1602
2530
 
1603
2531
  idx = std::move(idxqfs);
1604
2532
  } else if (h == fourcc("Ixrq")) {
2533
+ // Ixrq = original single-bit format
1605
2534
  auto idxq = std::make_unique<IndexRaBitQ>();
1606
2535
  read_index_header(*idxq, f);
1607
- read_RaBitQuantizer(idxq->rabitq, f, false);
2536
+ read_RaBitQuantizer(idxq->rabitq, f, idxq->d, false);
1608
2537
  READVECTOR(idxq->codes);
1609
2538
  READVECTOR(idxq->center);
1610
2539
  READ1(idxq->qb);
2540
+ // qb=0: Not quantized - direct distance computation on given float32s.
2541
+ // qb>0 && qb<=8: Scalar-quantized with qb bits of precision.
2542
+ FAISS_THROW_IF_NOT_FMT(
2543
+ idxq->qb <= 8,
2544
+ "invalid RaBitQ qb=%d (must be in [0, 8])",
2545
+ idxq->qb);
1611
2546
 
1612
2547
  // rabitq.nb_bits is already set to 1 by read_RaBitQuantizer
1613
2548
  idxq->code_size = idxq->rabitq.code_size;
@@ -1616,20 +2551,33 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1616
2551
  // Ixrr = multi-bit format (new)
1617
2552
  auto idxq = std::make_unique<IndexRaBitQ>();
1618
2553
  read_index_header(*idxq, f);
1619
- read_RaBitQuantizer(idxq->rabitq, f, true); // Reads nb_bits from file
2554
+ read_RaBitQuantizer(
2555
+ idxq->rabitq, f, idxq->d, true); // Reads nb_bits from file
1620
2556
  READVECTOR(idxq->codes);
1621
2557
  READVECTOR(idxq->center);
1622
2558
  READ1(idxq->qb);
2559
+ // qb=0: Not quantized - direct distance computation on given float32s.
2560
+ // qb>0 && qb<=8: Scalar-quantized with qb bits of precision.
2561
+ FAISS_THROW_IF_NOT_FMT(
2562
+ idxq->qb <= 8,
2563
+ "invalid RaBitQ qb=%d (must be in [0, 8])",
2564
+ idxq->qb);
1623
2565
 
1624
2566
  idxq->code_size = idxq->rabitq.code_size;
1625
2567
  idx = std::move(idxq);
1626
2568
  } else if (h == fourcc("Iwrq")) {
1627
2569
  auto ivrq = std::make_unique<IndexIVFRaBitQ>();
1628
2570
  read_ivf_header(ivrq.get(), f);
1629
- read_RaBitQuantizer(ivrq->rabitq, f, false);
2571
+ read_RaBitQuantizer(ivrq->rabitq, f, ivrq->d, false);
1630
2572
  READ1(ivrq->code_size);
1631
- READ1(ivrq->by_residual);
2573
+ READ1_BOOL(ivrq->by_residual);
1632
2574
  READ1(ivrq->qb);
2575
+ // qb=0: Not quantized - direct distance computation on given float32s.
2576
+ // qb>0 && qb<=8: Scalar-quantized with qb bits of precision.
2577
+ FAISS_THROW_IF_NOT_FMT(
2578
+ ivrq->qb <= 8,
2579
+ "invalid RaBitQ qb=%d (must be in [0, 8])",
2580
+ ivrq->qb);
1633
2581
 
1634
2582
  // rabitq.nb_bits is already set to 1 by read_RaBitQuantizer
1635
2583
  // Update rabitq to match nb_bits
@@ -1642,10 +2590,17 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1642
2590
  // Iwrr = multi-bit format (new)
1643
2591
  auto ivrq = std::make_unique<IndexIVFRaBitQ>();
1644
2592
  read_ivf_header(ivrq.get(), f);
1645
- read_RaBitQuantizer(ivrq->rabitq, f, true); // Reads nb_bits from file
2593
+ read_RaBitQuantizer(
2594
+ ivrq->rabitq, f, ivrq->d, true); // Reads nb_bits from file
1646
2595
  READ1(ivrq->code_size);
1647
- READ1(ivrq->by_residual);
2596
+ READ1_BOOL(ivrq->by_residual);
1648
2597
  READ1(ivrq->qb);
2598
+ // qb=0: Not quantized - direct distance computation on given float32s.
2599
+ // qb>0 && qb<=8: Scalar-quantized with qb bits of precision.
2600
+ FAISS_THROW_IF_NOT_FMT(
2601
+ ivrq->qb <= 8,
2602
+ "invalid RaBitQ qb=%d (must be in [0, 8])",
2603
+ ivrq->qb);
1649
2604
 
1650
2605
  // Update rabitq to match nb_bits
1651
2606
  ivrq->rabitq.code_size =
@@ -1656,13 +2611,14 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1656
2611
  }
1657
2612
  #ifdef FAISS_ENABLE_SVS
1658
2613
  else if (
1659
- h == fourcc("ILVQ") || h == fourcc("ISVL") || h == fourcc("ISVD")) {
2614
+ h == fourcc("ILVQ") || h == fourcc("ISVL") || h == fourcc("ISVD") ||
2615
+ h == fourcc("ISV2")) {
1660
2616
  std::unique_ptr<IndexSVSVamana> svs;
1661
2617
  if (h == fourcc("ILVQ")) {
1662
2618
  svs = std::make_unique<IndexSVSVamanaLVQ>();
1663
2619
  } else if (h == fourcc("ISVL")) {
1664
2620
  svs = std::make_unique<IndexSVSVamanaLeanVec>();
1665
- } else if (h == fourcc("ISVD")) {
2621
+ } else if (h == fourcc("ISVD") || h == fourcc("ISV2")) {
1666
2622
  svs = std::make_unique<IndexSVSVamana>();
1667
2623
  }
1668
2624
 
@@ -1674,42 +2630,112 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1674
2630
  READ1(svs->construction_window_size);
1675
2631
  READ1(svs->max_candidate_pool_size);
1676
2632
  READ1(svs->prune_to);
1677
- READ1(svs->use_full_search_history);
1678
- READ1(svs->storage_kind);
2633
+ READ1_BOOL(svs->use_full_search_history);
2634
+
2635
+ svs->storage_kind = read_svs_storage_kind(f);
2636
+ READ1_BOOL(svs->is_static);
2637
+
1679
2638
  if (h == fourcc("ISVL")) {
1680
- READ1(dynamic_cast<IndexSVSVamanaLeanVec*>(svs.get())->leanvec_d);
2639
+ auto* leanvec = dynamic_cast<IndexSVSVamanaLeanVec*>(svs.get());
2640
+ FAISS_THROW_IF_NOT_MSG(
2641
+ leanvec, "dynamic_cast to IndexSVSVamanaLeanVec failed");
2642
+ READ1(leanvec->leanvec_d);
1681
2643
  }
1682
2644
 
1683
2645
  bool initialized;
1684
- READ1(initialized);
2646
+ READ1_BOOL(initialized);
1685
2647
  if (initialized) {
1686
- faiss::svs_io::ReaderStreambuf rbuf(f);
2648
+ faiss::svs_io::ReaderStreambuf rbuf(
2649
+ f, get_deserialization_vector_byte_limit());
1687
2650
  std::istream is(&rbuf);
1688
2651
  svs->deserialize_impl(is);
1689
2652
  }
1690
2653
  if (h == fourcc("ISVL")) {
1691
2654
  bool trained;
1692
- READ1(trained);
2655
+ READ1_BOOL(trained);
1693
2656
  if (trained) {
1694
- faiss::svs_io::ReaderStreambuf rbuf(f);
2657
+ faiss::svs_io::ReaderStreambuf rbuf(
2658
+ f, get_deserialization_vector_byte_limit());
1695
2659
  std::istream is(&rbuf);
1696
- dynamic_cast<IndexSVSVamanaLeanVec*>(svs.get())
1697
- ->deserialize_training_data(is);
2660
+ auto* leanvec = dynamic_cast<IndexSVSVamanaLeanVec*>(svs.get());
2661
+ FAISS_THROW_IF_NOT_MSG(
2662
+ leanvec,
2663
+ "dynamic_cast to IndexSVSVamanaLeanVec failed");
2664
+ leanvec->deserialize_training_data(is);
1698
2665
  }
1699
2666
  }
2667
+ if (h == fourcc("ISV2")) {
2668
+ READVECTOR(svs->stored_vectors);
2669
+ } else {
2670
+ svs->stored_vectors_valid = false;
2671
+ }
1700
2672
  idx = std::move(svs);
1701
2673
  } else if (h == fourcc("ISVF")) {
1702
2674
  auto svs = std::make_unique<IndexSVSFlat>();
1703
2675
  read_index_header(*svs, f);
1704
2676
 
1705
2677
  bool initialized;
1706
- READ1(initialized);
2678
+ READ1_BOOL(initialized);
1707
2679
  if (initialized) {
1708
- faiss::svs_io::ReaderStreambuf rbuf(f);
2680
+ faiss::svs_io::ReaderStreambuf rbuf(
2681
+ f, get_deserialization_vector_byte_limit());
1709
2682
  std::istream is(&rbuf);
1710
2683
  svs->deserialize_impl(is);
1711
2684
  }
1712
2685
  idx = std::move(svs);
2686
+ } else if (
2687
+ h == fourcc("ISIQ") || h == fourcc("ISIL") || h == fourcc("ISID")) {
2688
+ std::unique_ptr<IndexSVSIVF> svs_ivf;
2689
+ if (h == fourcc("ISIQ")) {
2690
+ svs_ivf = std::make_unique<IndexSVSIVFLVQ>();
2691
+ } else if (h == fourcc("ISIL")) {
2692
+ svs_ivf = std::make_unique<IndexSVSIVFLeanVec>();
2693
+ } else if (h == fourcc("ISID")) {
2694
+ svs_ivf = std::make_unique<IndexSVSIVF>();
2695
+ }
2696
+
2697
+ read_index_header(*svs_ivf, f);
2698
+ READ1(svs_ivf->num_centroids);
2699
+ READ1(svs_ivf->minibatch_size);
2700
+ READ1(svs_ivf->num_iterations);
2701
+ READ1_BOOL(svs_ivf->is_hierarchical);
2702
+ READ1(svs_ivf->training_fraction);
2703
+ READ1(svs_ivf->hierarchical_level1_clusters);
2704
+ READ1(svs_ivf->seed);
2705
+ READ1(svs_ivf->n_probes);
2706
+ READ1(svs_ivf->k_reorder);
2707
+ READ1(svs_ivf->num_threads);
2708
+ READ1(svs_ivf->intra_query_threads);
2709
+ svs_ivf->storage_kind = read_svs_storage_kind(f);
2710
+ READ1_BOOL(svs_ivf->is_static);
2711
+ if (h == fourcc("ISIL")) {
2712
+ auto* leanvec = dynamic_cast<IndexSVSIVFLeanVec*>(svs_ivf.get());
2713
+ FAISS_THROW_IF_NOT_MSG(
2714
+ leanvec, "dynamic_cast to IndexSVSIVFLeanVec failed");
2715
+ READ1(leanvec->leanvec_d);
2716
+ }
2717
+
2718
+ bool initialized;
2719
+ READ1_BOOL(initialized);
2720
+ if (initialized) {
2721
+ faiss::svs_io::ReaderStreambuf rbuf(f);
2722
+ std::istream is(&rbuf);
2723
+ svs_ivf->deserialize_impl(is);
2724
+ }
2725
+ if (h == fourcc("ISIL")) {
2726
+ bool trained;
2727
+ READ1_BOOL(trained);
2728
+ if (trained) {
2729
+ faiss::svs_io::ReaderStreambuf rbuf(f);
2730
+ std::istream is(&rbuf);
2731
+ auto* leanvec =
2732
+ dynamic_cast<IndexSVSIVFLeanVec*>(svs_ivf.get());
2733
+ FAISS_THROW_IF_NOT_MSG(
2734
+ leanvec, "dynamic_cast to IndexSVSIVFLeanVec failed");
2735
+ leanvec->deserialize_training_data(is);
2736
+ }
2737
+ }
2738
+ idx = std::move(svs_ivf);
1713
2739
  }
1714
2740
  #endif // FAISS_ENABLE_SVS
1715
2741
  else if (h == fourcc("Iwrn") || h == fourcc("Iwrf")) {
@@ -1719,15 +2745,19 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1719
2745
 
1720
2746
  auto ivrqfs = std::make_unique<IndexIVFRaBitQFastScan>();
1721
2747
  read_ivf_header(ivrqfs.get(), f);
1722
- read_RaBitQuantizer(ivrqfs->rabitq, f);
1723
- READ1(ivrqfs->by_residual);
2748
+ read_RaBitQuantizer(ivrqfs->rabitq, f, ivrqfs->d);
2749
+ READ1_BOOL(ivrqfs->by_residual);
1724
2750
  READ1(ivrqfs->code_size);
1725
2751
  READ1(ivrqfs->bbs);
1726
2752
  READ1(ivrqfs->qbs2);
1727
2753
  READ1(ivrqfs->M2);
1728
2754
  READ1(ivrqfs->implem);
1729
2755
  READ1(ivrqfs->qb);
1730
- READ1(ivrqfs->centered);
2756
+ FAISS_THROW_IF_NOT_FMT(
2757
+ ivrqfs->qb > 0 && ivrqfs->qb <= 8,
2758
+ "invalid RaBitQ qb=%d (must be in [1, 8])",
2759
+ ivrqfs->qb);
2760
+ READ1_BOOL(ivrqfs->centered);
1731
2761
 
1732
2762
  std::vector<uint8_t> legacy_flat_storage;
1733
2763
  if (is_legacy) {
@@ -1741,6 +2771,13 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
1741
2771
  ivrqfs->nbits = nbits_fastscan;
1742
2772
  ivrqfs->ksub = (1 << nbits_fastscan);
1743
2773
 
2774
+ validate_fastscan_fields(
2775
+ ivrqfs->M,
2776
+ ivrqfs->M2,
2777
+ ivrqfs->ksub,
2778
+ ivrqfs->bbs,
2779
+ "IndexIVFRaBitQFastScan");
2780
+
1744
2781
  read_InvertedLists(*ivrqfs, f, io_flags);
1745
2782
  ivrqfs->init_code_packer();
1746
2783
 
@@ -1847,12 +2884,19 @@ static void read_index_binary_header(IndexBinary& idx, IOReader* f) {
1847
2884
  READ1(idx.d);
1848
2885
  READ1(idx.code_size);
1849
2886
  READ1(idx.ntotal);
1850
- READ1(idx.is_trained);
2887
+ READ1_BOOL(idx.is_trained);
1851
2888
  int metric_type_int;
1852
2889
  READ1(metric_type_int);
1853
2890
  idx.metric_type = metric_type_from_int(metric_type_int);
1854
2891
  FAISS_THROW_IF_NOT_FMT(
1855
- idx.d >= 0, "invalid binary index dimension %d", idx.d);
2892
+ idx.d > 0 && idx.d % 8 == 0,
2893
+ "invalid binary index dimension %d (must be > 0 and a multiple of 8)",
2894
+ idx.d);
2895
+ FAISS_THROW_IF_NOT_FMT(
2896
+ idx.code_size == idx.d / 8,
2897
+ "binary index code_size=%d does not match d/8=%d",
2898
+ (int)idx.code_size,
2899
+ idx.d / 8);
1856
2900
  FAISS_THROW_IF_NOT_FMT(
1857
2901
  idx.ntotal >= 0,
1858
2902
  "invalid binary index ntotal %" PRId64,
@@ -1866,6 +2910,7 @@ static void read_binary_ivf_header(
1866
2910
  std::vector<std::vector<idx_t>>* ids = nullptr) {
1867
2911
  read_index_binary_header(ivf, f);
1868
2912
  READ1(ivf.nlist);
2913
+ FAISS_CHECK_DESERIALIZATION_LOOP_LIMIT(ivf.nlist, "nlist");
1869
2914
  READ1(ivf.nprobe);
1870
2915
  ivf.quantizer = read_index_binary(f);
1871
2916
  ivf.own_fields = true;
@@ -1880,11 +2925,25 @@ static void read_binary_ivf_header(
1880
2925
  static void read_binary_hash_invlists(
1881
2926
  IndexBinaryHash::InvertedListMap& invlists,
1882
2927
  int b,
2928
+ size_t code_size,
1883
2929
  IOReader* f) {
1884
2930
  size_t sz;
1885
2931
  READ1(sz);
2932
+ FAISS_CHECK_DESERIALIZATION_LOOP_LIMIT(sz, "binary hash invlists sz");
1886
2933
  int il_nbit = 0;
1887
2934
  READ1(il_nbit);
2935
+ FAISS_THROW_IF_NOT_FMT(
2936
+ il_nbit >= 0,
2937
+ "invalid binary hash invlists il_nbit=%d (must be >= 0)",
2938
+ il_nbit);
2939
+ if (sz > 0) {
2940
+ FAISS_THROW_IF_NOT_FMT(
2941
+ il_nbit > 0,
2942
+ "invalid binary hash invlists il_nbit=%d for sz=%zd "
2943
+ "(must be > 0 when entries exist)",
2944
+ il_nbit,
2945
+ sz);
2946
+ }
1888
2947
  // buffer for bitstrings
1889
2948
  size_t bits_per_entry = (size_t)b + (size_t)il_nbit;
1890
2949
  size_t total_bits =
@@ -1909,6 +2968,13 @@ static void read_binary_hash_invlists(
1909
2968
  READVECTOR(il.ids);
1910
2969
  FAISS_THROW_IF_NOT(il.ids.size() == ilsz);
1911
2970
  READVECTOR(il.vecs);
2971
+ FAISS_THROW_IF_NOT_FMT(
2972
+ il.vecs.size() == il.ids.size() * code_size,
2973
+ "binary hash invlists: vecs size %zu != ids size %zu * "
2974
+ "code_size %zu",
2975
+ il.vecs.size(),
2976
+ il.ids.size(),
2977
+ code_size);
1912
2978
  }
1913
2979
  }
1914
2980
 
@@ -1921,6 +2987,7 @@ static void read_binary_multi_hash_map(
1921
2987
  size_t sz;
1922
2988
  READ1(id_bits);
1923
2989
  READ1(sz);
2990
+ FAISS_CHECK_DESERIALIZATION_LOOP_LIMIT(sz, "multi hash map sz");
1924
2991
  std::vector<uint8_t> buf;
1925
2992
  READVECTOR(buf);
1926
2993
  size_t nbit = add_no_overflow(
@@ -1930,12 +2997,28 @@ static void read_binary_multi_hash_map(
1930
2997
  FAISS_THROW_IF_NOT(buf.size() == (nbit + 7) / 8);
1931
2998
  BitstringReader rd(buf.data(), buf.size());
1932
2999
  map.reserve(sz);
3000
+ size_t total_ids = 0;
1933
3001
  for (size_t i = 0; i < sz; i++) {
1934
3002
  uint64_t hash = rd.read(b);
1935
3003
  uint64_t ilsz = rd.read(id_bits);
3004
+ FAISS_THROW_IF_NOT_FMT(
3005
+ ilsz <= ntotal - total_ids,
3006
+ "multi hash map: ilsz=%zu at entry %zu would exceed "
3007
+ "ntotal=%zu (already read %zu ids)",
3008
+ (size_t)ilsz,
3009
+ i,
3010
+ ntotal,
3011
+ total_ids);
3012
+ total_ids += ilsz;
1936
3013
  auto& il = map[hash];
1937
3014
  for (size_t j = 0; j < ilsz; j++) {
1938
- il.push_back(rd.read(id_bits));
3015
+ uint64_t id = rd.read(id_bits);
3016
+ FAISS_THROW_IF_NOT_FMT(
3017
+ id < ntotal,
3018
+ "multi hash map: id=%zu >= ntotal=%zu",
3019
+ (size_t)id,
3020
+ ntotal);
3021
+ il.push_back(id);
1939
3022
  }
1940
3023
  }
1941
3024
  }
@@ -1960,25 +3043,53 @@ std::unique_ptr<IndexBinary> read_index_binary_up(IOReader* f, int io_flags) {
1960
3043
  read_index_binary_header(*idxff, f);
1961
3044
  idxff->own_fields = true;
1962
3045
  idxff->index = read_index(f, io_flags);
3046
+ FAISS_THROW_IF_NOT_MSG(
3047
+ idxff->index, "IndexBinaryFromFloat inner index is null");
1963
3048
  idx = std::move(idxff);
1964
3049
  } else if (h == fourcc("IBHf")) {
1965
3050
  auto idxhnsw = std::make_unique<IndexBinaryHNSW>();
1966
3051
  read_index_binary_header(*idxhnsw, f);
1967
3052
  read_HNSW(idxhnsw->hnsw, f);
1968
3053
  idxhnsw->hnsw.is_panorama = false;
3054
+ FAISS_THROW_IF_NOT_FMT(
3055
+ idxhnsw->hnsw.levels.size() == (size_t)idxhnsw->ntotal,
3056
+ "IndexBinaryHNSW HNSW levels size %zu != ntotal %" PRId64,
3057
+ idxhnsw->hnsw.levels.size(),
3058
+ idxhnsw->ntotal);
1969
3059
  idxhnsw->storage = read_index_binary(f, io_flags);
1970
3060
  idxhnsw->own_fields = true;
3061
+ FAISS_THROW_IF_NOT_MSG(
3062
+ idxhnsw->storage &&
3063
+ dynamic_cast<IndexBinaryFlat*>(idxhnsw->storage) !=
3064
+ nullptr,
3065
+ "IndexBinaryHNSW requires IndexBinaryFlat storage");
3066
+ FAISS_THROW_IF_NOT_MSG(
3067
+ idxhnsw->storage->ntotal == idxhnsw->ntotal,
3068
+ "IndexBinaryHNSW storage ntotal mismatch");
1971
3069
  idx = std::move(idxhnsw);
1972
3070
  } else if (h == fourcc("IBHc")) {
1973
3071
  auto idxhnsw = std::make_unique<IndexBinaryHNSWCagra>();
1974
3072
  read_index_binary_header(*idxhnsw, f);
1975
- READ1(idxhnsw->keep_max_size_level0);
1976
- READ1(idxhnsw->base_level_only);
3073
+ READ1_BOOL(idxhnsw->keep_max_size_level0);
3074
+ READ1_BOOL(idxhnsw->base_level_only);
1977
3075
  READ1(idxhnsw->num_base_level_search_entrypoints);
1978
3076
  read_HNSW(idxhnsw->hnsw, f);
1979
3077
  idxhnsw->hnsw.is_panorama = false;
3078
+ FAISS_THROW_IF_NOT_FMT(
3079
+ idxhnsw->hnsw.levels.size() == (size_t)idxhnsw->ntotal,
3080
+ "IndexBinaryHNSWCagra HNSW levels size %zu != ntotal %" PRId64,
3081
+ idxhnsw->hnsw.levels.size(),
3082
+ idxhnsw->ntotal);
1980
3083
  idxhnsw->storage = read_index_binary(f, io_flags);
1981
3084
  idxhnsw->own_fields = true;
3085
+ FAISS_THROW_IF_NOT_MSG(
3086
+ idxhnsw->storage &&
3087
+ dynamic_cast<IndexBinaryFlat*>(idxhnsw->storage) !=
3088
+ nullptr,
3089
+ "IndexBinaryHNSWCagra requires IndexBinaryFlat storage");
3090
+ FAISS_THROW_IF_NOT_MSG(
3091
+ idxhnsw->storage->ntotal == idxhnsw->ntotal,
3092
+ "IndexBinaryHNSWCagra storage ntotal mismatch");
1982
3093
  idx = std::move(idxhnsw);
1983
3094
  } else if (h == fourcc("IBMp") || h == fourcc("IBM2")) {
1984
3095
  bool is_map2 = h == fourcc("IBM2");
@@ -1989,6 +3100,18 @@ std::unique_ptr<IndexBinary> read_index_binary_up(IOReader* f, int io_flags) {
1989
3100
  idxmap->index = read_index_binary(f, io_flags);
1990
3101
  idxmap->own_fields = true;
1991
3102
  READVECTOR(idxmap->id_map);
3103
+ FAISS_THROW_IF_NOT_FMT(
3104
+ idxmap->id_map.size() == idxmap->ntotal,
3105
+ "IndexBinaryIDMap id_map size (%" PRId64
3106
+ ") does not match ntotal (%" PRId64 ")",
3107
+ int64_t(idxmap->id_map.size()),
3108
+ idxmap->ntotal);
3109
+ FAISS_THROW_IF_NOT_FMT(
3110
+ idxmap->index->ntotal == idxmap->ntotal,
3111
+ "IndexBinaryIDMap inner index ntotal (%" PRId64
3112
+ ") does not match IndexBinaryIDMap ntotal (%" PRId64 ")",
3113
+ idxmap->index->ntotal,
3114
+ idxmap->ntotal);
1992
3115
  if (is_map2) {
1993
3116
  static_cast<IndexBinaryIDMap2*>(idxmap.get())->construct_rev_map();
1994
3117
  }
@@ -1997,8 +3120,17 @@ std::unique_ptr<IndexBinary> read_index_binary_up(IOReader* f, int io_flags) {
1997
3120
  auto idxh = std::make_unique<IndexBinaryHash>();
1998
3121
  read_index_binary_header(*idxh, f);
1999
3122
  READ1(idxh->b);
3123
+ FAISS_THROW_IF_NOT_FMT(
3124
+ idxh->b > 0,
3125
+ "invalid IndexBinaryHash b=%d (must be > 0)",
3126
+ idxh->b);
3127
+ FAISS_THROW_IF_NOT_FMT(
3128
+ static_cast<size_t>(idxh->b) <= idxh->code_size * 8,
3129
+ "IndexBinaryHash b=%d exceeds code_size=%d bits",
3130
+ idxh->b,
3131
+ idxh->code_size);
2000
3132
  READ1(idxh->nflip);
2001
- read_binary_hash_invlists(idxh->invlists, idxh->b, f);
3133
+ read_binary_hash_invlists(idxh->invlists, idxh->b, idxh->code_size, f);
2002
3134
  idx = std::move(idxh);
2003
3135
  } else if (h == fourcc("IBHm")) {
2004
3136
  auto idxmh = std::make_unique<IndexBinaryMultiHash>();
@@ -2010,7 +3142,23 @@ std::unique_ptr<IndexBinary> read_index_binary_up(IOReader* f, int io_flags) {
2010
3142
  storage_idx.release();
2011
3143
  idxmh->own_fields = true;
2012
3144
  READ1(idxmh->b);
3145
+ FAISS_THROW_IF_NOT_FMT(
3146
+ idxmh->b > 0,
3147
+ "invalid IndexBinaryMultiHash b=%d (must be > 0)",
3148
+ idxmh->b);
2013
3149
  READ1(idxmh->nhash);
3150
+ FAISS_THROW_IF_NOT_FMT(
3151
+ idxmh->nhash > 0,
3152
+ "invalid IndexBinaryMultiHash nhash %d (must be > 0)",
3153
+ idxmh->nhash);
3154
+ FAISS_CHECK_DESERIALIZATION_LOOP_LIMIT(idxmh->nhash, "nhash");
3155
+ FAISS_THROW_IF_NOT_FMT(
3156
+ mul_no_overflow(idxmh->nhash, idxmh->b, "nhash * b") <=
3157
+ mul_no_overflow(idxmh->code_size, 8, "code_size * 8"),
3158
+ "IndexBinaryMultiHash nhash=%d * b=%d exceeds code_size=%d bits",
3159
+ idxmh->nhash,
3160
+ idxmh->b,
3161
+ idxmh->code_size);
2014
3162
  READ1(idxmh->nflip);
2015
3163
  idxmh->maps.resize(idxmh->nhash);
2016
3164
  for (int i = 0; i < idxmh->nhash; i++) {