faiss 0.4.3 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +10 -0
  3. data/README.md +2 -0
  4. data/ext/faiss/index.cpp +33 -6
  5. data/ext/faiss/index_binary.cpp +17 -4
  6. data/ext/faiss/kmeans.cpp +6 -6
  7. data/lib/faiss/version.rb +1 -1
  8. data/vendor/faiss/faiss/AutoTune.cpp +2 -3
  9. data/vendor/faiss/faiss/AutoTune.h +1 -1
  10. data/vendor/faiss/faiss/Clustering.cpp +2 -2
  11. data/vendor/faiss/faiss/Clustering.h +2 -2
  12. data/vendor/faiss/faiss/IVFlib.cpp +26 -51
  13. data/vendor/faiss/faiss/IVFlib.h +1 -1
  14. data/vendor/faiss/faiss/Index.cpp +11 -0
  15. data/vendor/faiss/faiss/Index.h +34 -11
  16. data/vendor/faiss/faiss/Index2Layer.cpp +1 -1
  17. data/vendor/faiss/faiss/Index2Layer.h +2 -2
  18. data/vendor/faiss/faiss/IndexAdditiveQuantizer.cpp +1 -0
  19. data/vendor/faiss/faiss/IndexAdditiveQuantizerFastScan.cpp +9 -4
  20. data/vendor/faiss/faiss/IndexAdditiveQuantizerFastScan.h +5 -1
  21. data/vendor/faiss/faiss/IndexBinary.h +7 -7
  22. data/vendor/faiss/faiss/IndexBinaryFromFloat.h +1 -1
  23. data/vendor/faiss/faiss/IndexBinaryHNSW.cpp +8 -2
  24. data/vendor/faiss/faiss/IndexBinaryHNSW.h +1 -1
  25. data/vendor/faiss/faiss/IndexBinaryHash.cpp +3 -3
  26. data/vendor/faiss/faiss/IndexBinaryHash.h +5 -5
  27. data/vendor/faiss/faiss/IndexBinaryIVF.cpp +7 -6
  28. data/vendor/faiss/faiss/IndexFastScan.cpp +125 -49
  29. data/vendor/faiss/faiss/IndexFastScan.h +102 -7
  30. data/vendor/faiss/faiss/IndexFlat.cpp +374 -4
  31. data/vendor/faiss/faiss/IndexFlat.h +81 -1
  32. data/vendor/faiss/faiss/IndexHNSW.cpp +93 -2
  33. data/vendor/faiss/faiss/IndexHNSW.h +58 -2
  34. data/vendor/faiss/faiss/IndexIDMap.cpp +14 -13
  35. data/vendor/faiss/faiss/IndexIDMap.h +6 -6
  36. data/vendor/faiss/faiss/IndexIVF.cpp +1 -1
  37. data/vendor/faiss/faiss/IndexIVF.h +5 -5
  38. data/vendor/faiss/faiss/IndexIVFAdditiveQuantizer.cpp +1 -1
  39. data/vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.cpp +9 -3
  40. data/vendor/faiss/faiss/IndexIVFAdditiveQuantizerFastScan.h +3 -1
  41. data/vendor/faiss/faiss/IndexIVFFastScan.cpp +176 -90
  42. data/vendor/faiss/faiss/IndexIVFFastScan.h +173 -18
  43. data/vendor/faiss/faiss/IndexIVFFlat.cpp +1 -0
  44. data/vendor/faiss/faiss/IndexIVFFlatPanorama.cpp +251 -0
  45. data/vendor/faiss/faiss/IndexIVFFlatPanorama.h +64 -0
  46. data/vendor/faiss/faiss/IndexIVFPQ.cpp +3 -1
  47. data/vendor/faiss/faiss/IndexIVFPQ.h +1 -1
  48. data/vendor/faiss/faiss/IndexIVFPQFastScan.cpp +134 -2
  49. data/vendor/faiss/faiss/IndexIVFPQFastScan.h +7 -1
  50. data/vendor/faiss/faiss/IndexIVFRaBitQ.cpp +99 -8
  51. data/vendor/faiss/faiss/IndexIVFRaBitQ.h +4 -1
  52. data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.cpp +828 -0
  53. data/vendor/faiss/faiss/IndexIVFRaBitQFastScan.h +252 -0
  54. data/vendor/faiss/faiss/IndexIVFSpectralHash.cpp +1 -1
  55. data/vendor/faiss/faiss/IndexIVFSpectralHash.h +1 -1
  56. data/vendor/faiss/faiss/IndexNNDescent.cpp +1 -1
  57. data/vendor/faiss/faiss/IndexNSG.cpp +1 -1
  58. data/vendor/faiss/faiss/IndexNeuralNetCodec.h +1 -1
  59. data/vendor/faiss/faiss/IndexPQ.cpp +4 -1
  60. data/vendor/faiss/faiss/IndexPQ.h +1 -1
  61. data/vendor/faiss/faiss/IndexPQFastScan.cpp +6 -2
  62. data/vendor/faiss/faiss/IndexPQFastScan.h +5 -1
  63. data/vendor/faiss/faiss/IndexPreTransform.cpp +14 -0
  64. data/vendor/faiss/faiss/IndexPreTransform.h +9 -0
  65. data/vendor/faiss/faiss/IndexRaBitQ.cpp +96 -13
  66. data/vendor/faiss/faiss/IndexRaBitQ.h +11 -2
  67. data/vendor/faiss/faiss/IndexRaBitQFastScan.cpp +731 -0
  68. data/vendor/faiss/faiss/IndexRaBitQFastScan.h +175 -0
  69. data/vendor/faiss/faiss/IndexRefine.cpp +49 -0
  70. data/vendor/faiss/faiss/IndexRefine.h +17 -0
  71. data/vendor/faiss/faiss/IndexShards.cpp +1 -1
  72. data/vendor/faiss/faiss/MatrixStats.cpp +3 -3
  73. data/vendor/faiss/faiss/MetricType.h +1 -1
  74. data/vendor/faiss/faiss/VectorTransform.h +2 -2
  75. data/vendor/faiss/faiss/clone_index.cpp +5 -1
  76. data/vendor/faiss/faiss/gpu/GpuCloner.cpp +1 -1
  77. data/vendor/faiss/faiss/gpu/GpuClonerOptions.h +3 -1
  78. data/vendor/faiss/faiss/gpu/GpuIndex.h +11 -11
  79. data/vendor/faiss/faiss/gpu/GpuIndexBinaryCagra.h +1 -1
  80. data/vendor/faiss/faiss/gpu/GpuIndexBinaryFlat.h +1 -1
  81. data/vendor/faiss/faiss/gpu/GpuIndexCagra.h +11 -7
  82. data/vendor/faiss/faiss/gpu/StandardGpuResources.cpp +1 -1
  83. data/vendor/faiss/faiss/gpu/perf/IndexWrapper-inl.h +2 -0
  84. data/vendor/faiss/faiss/gpu/test/TestGpuIcmEncoder.cpp +7 -0
  85. data/vendor/faiss/faiss/gpu/test/TestGpuIndexIVFFlat.cpp +1 -1
  86. data/vendor/faiss/faiss/impl/AdditiveQuantizer.cpp +1 -1
  87. data/vendor/faiss/faiss/impl/AdditiveQuantizer.h +1 -1
  88. data/vendor/faiss/faiss/impl/AuxIndexStructures.cpp +2 -2
  89. data/vendor/faiss/faiss/impl/AuxIndexStructures.h +1 -1
  90. data/vendor/faiss/faiss/impl/CodePacker.h +2 -2
  91. data/vendor/faiss/faiss/impl/DistanceComputer.h +77 -6
  92. data/vendor/faiss/faiss/impl/FastScanDistancePostProcessing.h +53 -0
  93. data/vendor/faiss/faiss/impl/HNSW.cpp +295 -16
  94. data/vendor/faiss/faiss/impl/HNSW.h +35 -6
  95. data/vendor/faiss/faiss/impl/IDSelector.cpp +2 -2
  96. data/vendor/faiss/faiss/impl/IDSelector.h +4 -4
  97. data/vendor/faiss/faiss/impl/LocalSearchQuantizer.cpp +4 -4
  98. data/vendor/faiss/faiss/impl/LocalSearchQuantizer.h +1 -1
  99. data/vendor/faiss/faiss/impl/LookupTableScaler.h +1 -1
  100. data/vendor/faiss/faiss/impl/NNDescent.cpp +1 -1
  101. data/vendor/faiss/faiss/impl/NNDescent.h +2 -2
  102. data/vendor/faiss/faiss/impl/NSG.cpp +1 -1
  103. data/vendor/faiss/faiss/impl/Panorama.cpp +193 -0
  104. data/vendor/faiss/faiss/impl/Panorama.h +204 -0
  105. data/vendor/faiss/faiss/impl/PanoramaStats.cpp +33 -0
  106. data/vendor/faiss/faiss/impl/PanoramaStats.h +38 -0
  107. data/vendor/faiss/faiss/impl/PolysemousTraining.cpp +5 -5
  108. data/vendor/faiss/faiss/impl/ProductAdditiveQuantizer.cpp +1 -1
  109. data/vendor/faiss/faiss/impl/ProductAdditiveQuantizer.h +1 -1
  110. data/vendor/faiss/faiss/impl/ProductQuantizer-inl.h +2 -0
  111. data/vendor/faiss/faiss/impl/ProductQuantizer.h +1 -1
  112. data/vendor/faiss/faiss/impl/RaBitQStats.cpp +29 -0
  113. data/vendor/faiss/faiss/impl/RaBitQStats.h +56 -0
  114. data/vendor/faiss/faiss/impl/RaBitQUtils.cpp +294 -0
  115. data/vendor/faiss/faiss/impl/RaBitQUtils.h +330 -0
  116. data/vendor/faiss/faiss/impl/RaBitQuantizer.cpp +304 -223
  117. data/vendor/faiss/faiss/impl/RaBitQuantizer.h +72 -4
  118. data/vendor/faiss/faiss/impl/RaBitQuantizerMultiBit.cpp +362 -0
  119. data/vendor/faiss/faiss/impl/RaBitQuantizerMultiBit.h +112 -0
  120. data/vendor/faiss/faiss/impl/ResidualQuantizer.h +1 -1
  121. data/vendor/faiss/faiss/impl/ResultHandler.h +4 -4
  122. data/vendor/faiss/faiss/impl/ScalarQuantizer.cpp +7 -10
  123. data/vendor/faiss/faiss/impl/ScalarQuantizer.h +2 -4
  124. data/vendor/faiss/faiss/impl/ThreadedIndex-inl.h +7 -4
  125. data/vendor/faiss/faiss/impl/index_read.cpp +238 -10
  126. data/vendor/faiss/faiss/impl/index_write.cpp +212 -19
  127. data/vendor/faiss/faiss/impl/io.cpp +2 -2
  128. data/vendor/faiss/faiss/impl/io.h +4 -4
  129. data/vendor/faiss/faiss/impl/kmeans1d.cpp +1 -1
  130. data/vendor/faiss/faiss/impl/kmeans1d.h +1 -1
  131. data/vendor/faiss/faiss/impl/lattice_Zn.h +2 -2
  132. data/vendor/faiss/faiss/impl/mapped_io.cpp +2 -2
  133. data/vendor/faiss/faiss/impl/mapped_io.h +4 -3
  134. data/vendor/faiss/faiss/impl/maybe_owned_vector.h +8 -1
  135. data/vendor/faiss/faiss/impl/platform_macros.h +12 -0
  136. data/vendor/faiss/faiss/impl/pq4_fast_scan.cpp +30 -4
  137. data/vendor/faiss/faiss/impl/pq4_fast_scan.h +14 -8
  138. data/vendor/faiss/faiss/impl/pq4_fast_scan_search_qbs.cpp +5 -6
  139. data/vendor/faiss/faiss/impl/simd_result_handlers.h +55 -11
  140. data/vendor/faiss/faiss/impl/svs_io.cpp +86 -0
  141. data/vendor/faiss/faiss/impl/svs_io.h +67 -0
  142. data/vendor/faiss/faiss/impl/zerocopy_io.h +1 -1
  143. data/vendor/faiss/faiss/index_factory.cpp +217 -8
  144. data/vendor/faiss/faiss/index_factory.h +1 -1
  145. data/vendor/faiss/faiss/index_io.h +1 -1
  146. data/vendor/faiss/faiss/invlists/BlockInvertedLists.h +1 -1
  147. data/vendor/faiss/faiss/invlists/DirectMap.cpp +1 -1
  148. data/vendor/faiss/faiss/invlists/InvertedLists.cpp +115 -1
  149. data/vendor/faiss/faiss/invlists/InvertedLists.h +46 -0
  150. data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.cpp +1 -1
  151. data/vendor/faiss/faiss/invlists/OnDiskInvertedLists.h +1 -1
  152. data/vendor/faiss/faiss/svs/IndexSVSFaissUtils.h +261 -0
  153. data/vendor/faiss/faiss/svs/IndexSVSFlat.cpp +117 -0
  154. data/vendor/faiss/faiss/svs/IndexSVSFlat.h +66 -0
  155. data/vendor/faiss/faiss/svs/IndexSVSVamana.cpp +245 -0
  156. data/vendor/faiss/faiss/svs/IndexSVSVamana.h +137 -0
  157. data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.cpp +39 -0
  158. data/vendor/faiss/faiss/svs/IndexSVSVamanaLVQ.h +42 -0
  159. data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.cpp +149 -0
  160. data/vendor/faiss/faiss/svs/IndexSVSVamanaLeanVec.h +58 -0
  161. data/vendor/faiss/faiss/utils/AlignedTable.h +1 -1
  162. data/vendor/faiss/faiss/utils/Heap.cpp +2 -2
  163. data/vendor/faiss/faiss/utils/Heap.h +3 -3
  164. data/vendor/faiss/faiss/utils/NeuralNet.cpp +1 -1
  165. data/vendor/faiss/faiss/utils/NeuralNet.h +3 -3
  166. data/vendor/faiss/faiss/utils/approx_topk/approx_topk.h +2 -2
  167. data/vendor/faiss/faiss/utils/approx_topk/avx2-inl.h +2 -2
  168. data/vendor/faiss/faiss/utils/approx_topk/mode.h +1 -1
  169. data/vendor/faiss/faiss/utils/distances.cpp +0 -3
  170. data/vendor/faiss/faiss/utils/distances.h +2 -2
  171. data/vendor/faiss/faiss/utils/extra_distances-inl.h +3 -1
  172. data/vendor/faiss/faiss/utils/hamming-inl.h +2 -0
  173. data/vendor/faiss/faiss/utils/hamming.cpp +7 -6
  174. data/vendor/faiss/faiss/utils/hamming.h +1 -1
  175. data/vendor/faiss/faiss/utils/hamming_distance/common.h +1 -2
  176. data/vendor/faiss/faiss/utils/partitioning.cpp +5 -5
  177. data/vendor/faiss/faiss/utils/partitioning.h +2 -2
  178. data/vendor/faiss/faiss/utils/rabitq_simd.h +222 -336
  179. data/vendor/faiss/faiss/utils/random.cpp +1 -1
  180. data/vendor/faiss/faiss/utils/simdlib_avx2.h +1 -1
  181. data/vendor/faiss/faiss/utils/simdlib_avx512.h +1 -1
  182. data/vendor/faiss/faiss/utils/simdlib_neon.h +2 -2
  183. data/vendor/faiss/faiss/utils/transpose/transpose-avx512-inl.h +1 -1
  184. data/vendor/faiss/faiss/utils/utils.cpp +9 -2
  185. data/vendor/faiss/faiss/utils/utils.h +2 -2
  186. metadata +29 -1
@@ -116,7 +116,7 @@ struct IDSelectorBitmap : IDSelector {
116
116
  /** reverts the membership test of another selector */
117
117
  struct IDSelectorNot : IDSelector {
118
118
  const IDSelector* sel;
119
- IDSelectorNot(const IDSelector* sel) : sel(sel) {}
119
+ explicit IDSelectorNot(const IDSelector* sel) : sel(sel) {}
120
120
  bool is_member(idx_t id) const final {
121
121
  return !sel->is_member(id);
122
122
  }
@@ -131,7 +131,7 @@ struct IDSelectorAll : IDSelector {
131
131
  virtual ~IDSelectorAll() {}
132
132
  };
133
133
 
134
- /// does an AND operation on the the two given IDSelector's is_membership
134
+ /// does an AND operation on the two given IDSelector's is_membership
135
135
  /// results.
136
136
  struct IDSelectorAnd : IDSelector {
137
137
  const IDSelector* lhs;
@@ -144,7 +144,7 @@ struct IDSelectorAnd : IDSelector {
144
144
  virtual ~IDSelectorAnd() {}
145
145
  };
146
146
 
147
- /// does an OR operation on the the two given IDSelector's is_membership
147
+ /// does an OR operation on the two given IDSelector's is_membership
148
148
  /// results.
149
149
  struct IDSelectorOr : IDSelector {
150
150
  const IDSelector* lhs;
@@ -157,7 +157,7 @@ struct IDSelectorOr : IDSelector {
157
157
  virtual ~IDSelectorOr() {}
158
158
  };
159
159
 
160
- /// does an XOR operation on the the two given IDSelector's is_membership
160
+ /// does an XOR operation on the two given IDSelector's is_membership
161
161
  /// results.
162
162
  struct IDSelectorXOr : IDSelector {
163
163
  const IDSelector* lhs;
@@ -30,7 +30,7 @@
30
30
  #endif
31
31
 
32
32
  extern "C" {
33
- // LU decomoposition of a general matrix
33
+ // LU decomposition of a general matrix
34
34
  void sgetrf_(
35
35
  FINTEGER* m,
36
36
  FINTEGER* n,
@@ -65,7 +65,7 @@ int sgemm_(
65
65
  float* c,
66
66
  FINTEGER* ldc);
67
67
 
68
- // LU decomoposition of a general matrix
68
+ // LU decomposition of a general matrix
69
69
  void dgetrf_(
70
70
  FINTEGER* m,
71
71
  FINTEGER* n,
@@ -189,7 +189,7 @@ void LocalSearchQuantizer::train(size_t n, const float* x) {
189
189
  std::vector<int32_t> codes(n * M); // [n, M]
190
190
  random_int32(codes, 0, K - 1, gen);
191
191
 
192
- // compute standard derivations of each dimension
192
+ // compute standard deviations of each dimension
193
193
  std::vector<float> stddev(d, 0);
194
194
 
195
195
  #pragma omp parallel for
@@ -487,7 +487,7 @@ void LocalSearchQuantizer::update_codebooks(
487
487
  * L = (X - \sum cj)^2, j = 1, ..., M
488
488
  * L = X^2 - 2X * \sum cj + (\sum cj)^2
489
489
  *
490
- * X^2 is negligable since it is the same for all possible value
490
+ * X^2 is negligible since it is the same for all possible value
491
491
  * k of the m-th subcode.
492
492
  *
493
493
  * 2X * \sum cj is the unary term
@@ -138,7 +138,7 @@ struct LocalSearchQuantizer : AdditiveQuantizer {
138
138
  /** Add some perturbation to codebooks
139
139
  *
140
140
  * @param T temperature of simulated annealing
141
- * @param stddev standard derivations of each dimension in training data
141
+ * @param stddev standard deviations of each dimension in training data
142
142
  */
143
143
  void perturb_codebooks(
144
144
  float T,
@@ -63,7 +63,7 @@ struct DummyScaler {
63
63
  };
64
64
 
65
65
  /// consumes 2x4 bits to encode a norm as a scalar additive quantizer
66
- /// the norm is scaled because its range if larger than other components
66
+ /// the norm is scaled because its range is larger than other components
67
67
  struct NormTableScaler {
68
68
  static constexpr int nscale = 2;
69
69
  int scale_int;
@@ -177,7 +177,7 @@ void NNDescent::join(DistanceComputer& qdis) {
177
177
  }
178
178
  }
179
179
 
180
- /// Sample neighbors for each node to peform local join later
180
+ /// Sample neighbors for each node to perform local join later
181
181
  /// Store them in nn_new and nn_old
182
182
  void NNDescent::update() {
183
183
  // Step 1.
@@ -34,7 +34,7 @@ namespace faiss {
34
34
  *
35
35
  * Dong, Wei, Charikar Moses, and Kai Li, WWW 2011
36
36
  *
37
- * This implmentation is heavily influenced by the efanna
37
+ * This implementation is heavily influenced by the efanna
38
38
  * implementation by Cong Fu and the KGraph library by Wei Dong
39
39
  * (https://github.com/ZJULearning/efanna_graph)
40
40
  * (https://github.com/aaalgo/kgraph)
@@ -117,7 +117,7 @@ struct NNDescent {
117
117
  /// Perform local join on each node
118
118
  void join(DistanceComputer& qdis);
119
119
 
120
- /// Sample new neighbors for each node to peform local join later
120
+ /// Sample new neighbors for each node to perform local join later
121
121
  void update();
122
122
 
123
123
  /// Sample a small number of points to evaluate the quality of KNNG built
@@ -621,7 +621,7 @@ int NSG::attach_unlinked(
621
621
  }
622
622
  }
623
623
 
624
- // randomly choice annother node
624
+ // randomly choice another node
625
625
  if (!found) {
626
626
  do {
627
627
  node = rng.rand_int(ntotal);
@@ -0,0 +1,193 @@
1
+ /*
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
+ *
4
+ * This source code is licensed under the MIT license found in the
5
+ * LICENSE file in the root directory of this source tree.
6
+ */
7
+
8
+ #include <faiss/impl/Panorama.h>
9
+
10
+ #include <algorithm>
11
+ #include <cmath>
12
+ #include <cstring>
13
+ #include <vector>
14
+
15
+ namespace faiss {
16
+
17
+ /**************************************************************
18
+ * Panorama structure implementation
19
+ **************************************************************/
20
+
21
+ Panorama::Panorama(size_t code_size, size_t n_levels, size_t batch_size)
22
+ : code_size(code_size), n_levels(n_levels), batch_size(batch_size) {
23
+ set_derived_values();
24
+ }
25
+
26
+ void Panorama::set_derived_values() {
27
+ this->d = code_size / sizeof(float);
28
+ this->level_width_floats = ((d + n_levels - 1) / n_levels);
29
+ this->level_width = this->level_width_floats * sizeof(float);
30
+ }
31
+
32
+ /**
33
+ * @brief Copy codes to level-oriented layout
34
+ * @param codes The base pointer to codes
35
+ * @param offset Where to start writing new data (in number of vectors)
36
+ * @param n_entry The number of new vectors to write
37
+ * @param code The new vector data
38
+ */
39
+ void Panorama::copy_codes_to_level_layout(
40
+ uint8_t* codes,
41
+ size_t offset,
42
+ size_t n_entry,
43
+ const uint8_t* code) {
44
+ for (size_t entry_idx = 0; entry_idx < n_entry; entry_idx++) {
45
+ size_t current_pos = offset + entry_idx;
46
+
47
+ // Determine which batch we're in and position within that batch.
48
+ size_t batch_no = current_pos / batch_size;
49
+ size_t pos_in_batch = current_pos % batch_size;
50
+
51
+ // Copy entry into level-oriented layout for this batch.
52
+ size_t batch_offset = batch_no * batch_size * code_size;
53
+ for (size_t level = 0; level < n_levels; level++) {
54
+ size_t level_offset = level * level_width * batch_size;
55
+ size_t start_byte = level * level_width;
56
+ size_t actual_level_width =
57
+ std::min(level_width, code_size - level * level_width);
58
+
59
+ const uint8_t* src = code + entry_idx * code_size + start_byte;
60
+ uint8_t* dest = codes + batch_offset + level_offset +
61
+ pos_in_batch * actual_level_width;
62
+
63
+ memcpy(dest, src, actual_level_width);
64
+ }
65
+ }
66
+ }
67
+
68
+ void Panorama::compute_cumulative_sums(
69
+ float* cumsum_base,
70
+ size_t offset,
71
+ size_t n_entry,
72
+ const float* vectors) {
73
+ std::vector<float> suffix_sums(d + 1);
74
+
75
+ for (size_t entry_idx = 0; entry_idx < n_entry; entry_idx++) {
76
+ size_t current_pos = offset + entry_idx;
77
+ size_t batch_no = current_pos / batch_size;
78
+ size_t pos_in_batch = current_pos % batch_size;
79
+
80
+ const float* vector = vectors + entry_idx * d;
81
+
82
+ // Compute suffix sums of squared values.
83
+ suffix_sums[d] = 0.0f;
84
+ for (int j = d - 1; j >= 0; j--) {
85
+ float squared_val = vector[j] * vector[j];
86
+ suffix_sums[j] = suffix_sums[j + 1] + squared_val;
87
+ }
88
+
89
+ // Store cumulative sums in batch-oriented layout.
90
+ size_t cumsum_batch_offset = batch_no * batch_size * (n_levels + 1);
91
+
92
+ for (size_t level = 0; level < n_levels; level++) {
93
+ size_t start_idx = level * level_width_floats;
94
+ size_t cumsum_offset =
95
+ cumsum_batch_offset + level * batch_size + pos_in_batch;
96
+ if (start_idx < d) {
97
+ cumsum_base[cumsum_offset] = std::sqrt(suffix_sums[start_idx]);
98
+ } else {
99
+ cumsum_base[cumsum_offset] = 0.0f;
100
+ }
101
+ }
102
+
103
+ // Last level sum is always 0.
104
+ size_t cumsum_offset =
105
+ cumsum_batch_offset + n_levels * batch_size + pos_in_batch;
106
+ cumsum_base[cumsum_offset] = 0.0f;
107
+ }
108
+ }
109
+
110
+ void Panorama::compute_query_cum_sums(const float* query, float* query_cum_sums)
111
+ const {
112
+ std::vector<float> suffix_sums(d + 1);
113
+ suffix_sums[d] = 0.0f;
114
+
115
+ for (int j = d - 1; j >= 0; j--) {
116
+ float squared_val = query[j] * query[j];
117
+ suffix_sums[j] = suffix_sums[j + 1] + squared_val;
118
+ }
119
+
120
+ for (size_t level = 0; level < n_levels; level++) {
121
+ size_t start_idx = level * level_width_floats;
122
+ if (start_idx < d) {
123
+ query_cum_sums[level] = std::sqrt(suffix_sums[start_idx]);
124
+ } else {
125
+ query_cum_sums[level] = 0.0f;
126
+ }
127
+ }
128
+
129
+ query_cum_sums[n_levels] = 0.0f;
130
+ }
131
+
132
+ void Panorama::reconstruct(idx_t key, float* recons, const uint8_t* codes_base)
133
+ const {
134
+ uint8_t* recons_buffer = reinterpret_cast<uint8_t*>(recons);
135
+
136
+ size_t batch_no = key / batch_size;
137
+ size_t pos_in_batch = key % batch_size;
138
+ size_t batch_offset = batch_no * batch_size * code_size;
139
+
140
+ for (size_t level = 0; level < n_levels; level++) {
141
+ size_t level_offset = level * level_width * batch_size;
142
+ const uint8_t* src = codes_base + batch_offset + level_offset +
143
+ pos_in_batch * level_width;
144
+ uint8_t* dest = recons_buffer + level * level_width;
145
+ size_t copy_size =
146
+ std::min(level_width, code_size - level * level_width);
147
+ memcpy(dest, src, copy_size);
148
+ }
149
+ }
150
+
151
+ void Panorama::copy_entry(
152
+ uint8_t* dest_codes,
153
+ uint8_t* src_codes,
154
+ float* dest_cum_sums,
155
+ float* src_cum_sums,
156
+ size_t dest_idx,
157
+ size_t src_idx) const {
158
+ // Calculate positions
159
+ size_t src_batch_no = src_idx / batch_size;
160
+ size_t src_pos_in_batch = src_idx % batch_size;
161
+ size_t dest_batch_no = dest_idx / batch_size;
162
+ size_t dest_pos_in_batch = dest_idx % batch_size;
163
+
164
+ // Calculate offsets
165
+ size_t src_batch_offset = src_batch_no * batch_size * code_size;
166
+ size_t dest_batch_offset = dest_batch_no * batch_size * code_size;
167
+ size_t src_cumsum_batch_offset = src_batch_no * batch_size * (n_levels + 1);
168
+ size_t dest_cumsum_batch_offset =
169
+ dest_batch_no * batch_size * (n_levels + 1);
170
+
171
+ for (size_t level = 0; level < n_levels; level++) {
172
+ // Copy code
173
+ size_t level_offset = level * level_width * batch_size;
174
+ size_t actual_level_width =
175
+ std::min(level_width, code_size - level * level_width);
176
+
177
+ const uint8_t* src = src_codes + src_batch_offset + level_offset +
178
+ src_pos_in_batch * actual_level_width;
179
+ uint8_t* dest = dest_codes + dest_batch_offset + level_offset +
180
+ dest_pos_in_batch * actual_level_width;
181
+ memcpy(dest, src, actual_level_width);
182
+
183
+ // Copy cum_sums
184
+ size_t cumsum_level_offset = level * batch_size;
185
+
186
+ const size_t src_offset = src_cumsum_batch_offset +
187
+ cumsum_level_offset + src_pos_in_batch;
188
+ size_t dest_offset = dest_cumsum_batch_offset + cumsum_level_offset +
189
+ dest_pos_in_batch;
190
+ dest_cum_sums[dest_offset] = src_cum_sums[src_offset];
191
+ }
192
+ }
193
+ } // namespace faiss
@@ -0,0 +1,204 @@
1
+ /*
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
+ *
4
+ * This source code is licensed under the MIT license found in the
5
+ * LICENSE file in the root directory of this source tree.
6
+ */
7
+
8
+ // -*- c++ -*-
9
+
10
+ #ifndef FAISS_PANORAMA_H
11
+ #define FAISS_PANORAMA_H
12
+
13
+ #include <faiss/impl/IDSelector.h>
14
+ #include <faiss/impl/PanoramaStats.h>
15
+ #include <faiss/utils/distances.h>
16
+
17
+ #include <algorithm>
18
+ #include <cstddef>
19
+ #include <cstdint>
20
+ #include <vector>
21
+
22
+ namespace faiss {
23
+
24
+ /**
25
+ * Implements the core logic of Panorama-based refinement.
26
+ * arXiv: https://arxiv.org/abs/2510.00566
27
+ *
28
+ * Panorama partitions the dimensions of all vectors into L contiguous levels.
29
+ * During the refinement stage of ANNS, it computes distances between the query
30
+ * and its candidates level-by-level. After processing each level, it prunes the
31
+ * candidates whose lower bound exceeds the k-th best distance.
32
+ *
33
+ * In order to enable speedups, the dimensions (or codes) of each vector are
34
+ * stored in a batched, level-major manner. Within each batch of b vectors, the
35
+ * dimensions corresponding to level 1 will be stored first (for all elements in
36
+ * that batch), followed by level 2, and so on. This allows for efficient memory
37
+ * access patterns.
38
+ *
39
+ * Coupled with the appropriate orthogonal PreTransform (e.g. PCA, Cayley,
40
+ * etc.), Panorama can prune the vast majority of dimensions, greatly
41
+ * accelerating the refinement stage.
42
+ */
43
+ struct Panorama {
44
+ size_t d = 0;
45
+ size_t code_size = 0;
46
+ size_t n_levels = 0;
47
+ size_t level_width = 0;
48
+ size_t level_width_floats = 0;
49
+ size_t batch_size = 0;
50
+
51
+ explicit Panorama(size_t code_size, size_t n_levels, size_t batch_size);
52
+
53
+ void set_derived_values();
54
+
55
+ /// Helper method to copy codes into level-oriented batch layout at a given
56
+ /// offset in the list.
57
+ void copy_codes_to_level_layout(
58
+ uint8_t* codes,
59
+ size_t offset,
60
+ size_t n_entry,
61
+ const uint8_t* code);
62
+
63
+ /// Helper method to compute the cumulative sums of the codes.
64
+ /// The cumsums also follow the level-oriented batch layout to minimize the
65
+ /// number of random memory accesses.
66
+ void compute_cumulative_sums(
67
+ float* cumsum_base,
68
+ size_t offset,
69
+ size_t n_entry,
70
+ const float* vectors);
71
+
72
+ /// Compute the cumulative sums of the query vector.
73
+ void compute_query_cum_sums(const float* query, float* query_cum_sums)
74
+ const;
75
+
76
+ /// Copy single entry (code and cum_sum) from one location to another.
77
+ void copy_entry(
78
+ uint8_t* dest_codes,
79
+ uint8_t* src_codes,
80
+ float* dest_cum_sums,
81
+ float* src_cum_sums,
82
+ size_t dest_idx,
83
+ size_t src_idx) const;
84
+
85
+ /// Panorama's core progressive filtering algorithm:
86
+ /// Process vectors in batches for cache efficiency. For each batch:
87
+ /// 1. Apply ID selection filter and initialize distances
88
+ /// (||y||^2 + ||x||^2).
89
+ /// 2. Maintain an "active set" of candidate indices that haven't been
90
+ /// pruned yet.
91
+ /// 3. For each level, refine distances incrementally and compact the active
92
+ /// set:
93
+ /// - Compute dot product for current level: exact_dist -= 2*<x,y>.
94
+ /// - Use Cauchy-Schwarz bound on remaining levels to get lower bound
95
+ /// - Prune candidates whose lower bound exceeds k-th best distance.
96
+ /// - Compact active_indices to remove pruned candidates (branchless)
97
+ /// 4. After all levels, survivors are exact distances; update heap.
98
+ /// This achieves early termination while maintaining SIMD-friendly
99
+ /// sequential access patterns in the level-oriented storage layout.
100
+ template <typename C>
101
+ size_t progressive_filter_batch(
102
+ const uint8_t* codes_base,
103
+ const float* cum_sums,
104
+ const float* query,
105
+ const float* query_cum_sums,
106
+ size_t batch_no,
107
+ size_t list_size,
108
+ const IDSelector* sel,
109
+ const idx_t* ids,
110
+ bool use_sel,
111
+ std::vector<uint32_t>& active_indices,
112
+ std::vector<float>& exact_distances,
113
+ float threshold,
114
+ PanoramaStats& local_stats) const;
115
+
116
+ void reconstruct(idx_t key, float* recons, const uint8_t* codes_base) const;
117
+ };
118
+
119
+ template <typename C>
120
+ size_t Panorama::progressive_filter_batch(
121
+ const uint8_t* codes_base,
122
+ const float* cum_sums,
123
+ const float* query,
124
+ const float* query_cum_sums,
125
+ size_t batch_no,
126
+ size_t list_size,
127
+ const IDSelector* sel,
128
+ const idx_t* ids,
129
+ bool use_sel,
130
+ std::vector<uint32_t>& active_indices,
131
+ std::vector<float>& exact_distances,
132
+ float threshold,
133
+ PanoramaStats& local_stats) const {
134
+ size_t batch_start = batch_no * batch_size;
135
+ size_t curr_batch_size = std::min(list_size - batch_start, batch_size);
136
+
137
+ size_t cumsum_batch_offset = batch_no * batch_size * (n_levels + 1);
138
+ const float* batch_cum_sums = cum_sums + cumsum_batch_offset;
139
+ const float* level_cum_sums = batch_cum_sums + batch_size;
140
+ float q_norm = query_cum_sums[0] * query_cum_sums[0];
141
+
142
+ size_t batch_offset = batch_no * batch_size * code_size;
143
+ const uint8_t* storage_base = codes_base + batch_offset;
144
+
145
+ // Initialize active set with ID-filtered vectors.
146
+ size_t num_active = 0;
147
+ for (size_t i = 0; i < curr_batch_size; i++) {
148
+ size_t global_idx = batch_start + i;
149
+ idx_t id = (ids == nullptr) ? global_idx : ids[global_idx];
150
+ bool include = !use_sel || sel->is_member(id);
151
+
152
+ active_indices[num_active] = i;
153
+ float cum_sum = batch_cum_sums[i];
154
+ exact_distances[i] = cum_sum * cum_sum + q_norm;
155
+
156
+ num_active += include;
157
+ }
158
+
159
+ if (num_active == 0) {
160
+ return 0;
161
+ }
162
+
163
+ size_t total_active = num_active;
164
+ for (size_t level = 0; level < n_levels; level++) {
165
+ local_stats.total_dims_scanned += num_active;
166
+ local_stats.total_dims += total_active;
167
+
168
+ float query_cum_norm = query_cum_sums[level + 1];
169
+
170
+ size_t level_offset = level * level_width * batch_size;
171
+ const float* level_storage =
172
+ (const float*)(storage_base + level_offset);
173
+
174
+ size_t next_active = 0;
175
+ for (size_t i = 0; i < num_active; i++) {
176
+ uint32_t idx = active_indices[i];
177
+ size_t actual_level_width = std::min(
178
+ level_width_floats, d - level * level_width_floats);
179
+
180
+ const float* yj = level_storage + idx * actual_level_width;
181
+ const float* query_level = query + level * level_width_floats;
182
+
183
+ float dot_product =
184
+ fvec_inner_product(query_level, yj, actual_level_width);
185
+
186
+ exact_distances[idx] -= 2.0f * dot_product;
187
+
188
+ float cum_sum = level_cum_sums[idx];
189
+ float cauchy_schwarz_bound = 2.0f * cum_sum * query_cum_norm;
190
+ float lower_bound = exact_distances[idx] - cauchy_schwarz_bound;
191
+
192
+ active_indices[next_active] = idx;
193
+ next_active += C::cmp(threshold, lower_bound) ? 1 : 0;
194
+ }
195
+
196
+ num_active = next_active;
197
+ level_cum_sums += batch_size;
198
+ }
199
+
200
+ return num_active;
201
+ }
202
+ } // namespace faiss
203
+
204
+ #endif
@@ -0,0 +1,33 @@
1
+ /*
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
+ *
4
+ * This source code is licensed under the MIT license found in the
5
+ * LICENSE file in the root directory of this source tree.
6
+ */
7
+
8
+ // -*- c++ -*-
9
+
10
+ #include <faiss/impl/PanoramaStats.h>
11
+
12
+ namespace faiss {
13
+
14
+ void PanoramaStats::reset() {
15
+ total_dims_scanned = 0;
16
+ total_dims = 0;
17
+ ratio_dims_scanned = 1.0f;
18
+ }
19
+
20
+ void PanoramaStats::add(const PanoramaStats& other) {
21
+ total_dims_scanned += other.total_dims_scanned;
22
+ total_dims += other.total_dims;
23
+ if (total_dims > 0) {
24
+ ratio_dims_scanned =
25
+ static_cast<float>(total_dims_scanned) / total_dims;
26
+ } else {
27
+ ratio_dims_scanned = 1.0f;
28
+ }
29
+ }
30
+
31
+ PanoramaStats indexPanorama_stats;
32
+
33
+ } // namespace faiss
@@ -0,0 +1,38 @@
1
+ /*
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
+ *
4
+ * This source code is licensed under the MIT license found in the
5
+ * LICENSE file in the root directory of this source tree.
6
+ */
7
+
8
+ // -*- c++ -*-
9
+
10
+ #ifndef FAISS_PANORAMA_STATS_H
11
+ #define FAISS_PANORAMA_STATS_H
12
+
13
+ #include <faiss/impl/platform_macros.h>
14
+
15
+ namespace faiss {
16
+
17
+ /// Statistics are not robust to internal threading nor to
18
+ /// concurrent Panorama searches. Use these values in a
19
+ /// single-threaded context to accurately gauge Panorama's
20
+ /// pruning effectiveness.
21
+ struct PanoramaStats {
22
+ uint64_t total_dims_scanned = 0; // total dimensions scanned
23
+ uint64_t total_dims = 0; // total dimensions
24
+ float ratio_dims_scanned = 1.0f; // fraction of dimensions actually scanned
25
+
26
+ PanoramaStats() {
27
+ reset();
28
+ }
29
+ void reset();
30
+ void add(const PanoramaStats& other);
31
+ };
32
+
33
+ // Single global var for all Panorama indexes
34
+ FAISS_API extern PanoramaStats indexPanorama_stats;
35
+
36
+ } // namespace faiss
37
+
38
+ #endif
@@ -178,7 +178,7 @@ struct ReproduceWithHammingObjective : PermutationObjective {
178
178
  return x * x;
179
179
  }
180
180
 
181
- // weihgting of distances: it is more important to reproduce small
181
+ // weighting of distances: it is more important to reproduce small
182
182
  // distances well
183
183
  double dis_weight(double x) const {
184
184
  return exp(-dis_weight_factor * x);
@@ -295,7 +295,7 @@ struct ReproduceWithHammingObjective : PermutationObjective {
295
295
 
296
296
  } // anonymous namespace
297
297
 
298
- // weihgting of distances: it is more important to reproduce small
298
+ // weighting of distances: it is more important to reproduce small
299
299
  // distances well
300
300
  double ReproduceDistancesObjective::dis_weight(double x) const {
301
301
  return exp(-dis_weight_factor * x);
@@ -636,7 +636,7 @@ struct Score3Computer : PermutationObjective {
636
636
  return accu;
637
637
  }
638
638
 
639
- /// PermutationObjective implementeation (just negates the scores
639
+ /// PermutationObjective implementation (just negates the scores
640
640
  /// for minimization)
641
641
 
642
642
  double compute_cost(const int* perm) const override {
@@ -689,7 +689,7 @@ struct RankingScore2 : Score3Computer<float, double> {
689
689
  /// count nb of i, j in a x b st. i < j
690
690
  /// a and b should be sorted on input
691
691
  /// they are the ranks of j and k respectively.
692
- /// specific version for diff-of-rank weighting, cannot optimized
692
+ /// specific version for diff-of-rank weighting, cannot optimize
693
693
  /// with a cumulative table
694
694
  double accum_gt_weight_diff(
695
695
  const std::vector<int>& a,
@@ -985,7 +985,7 @@ size_t PolysemousTraining::memory_usage_per_thread(
985
985
  return n * n * n * sizeof(float);
986
986
  }
987
987
 
988
- FAISS_THROW_MSG("Invalid optmization type");
988
+ FAISS_THROW_MSG("Invalid optimization type");
989
989
  return 0;
990
990
  }
991
991
 
@@ -154,7 +154,7 @@ void ProductAdditiveQuantizer::compute_unpacked_codes(
154
154
  int32_t* unpacked_codes,
155
155
  size_t n,
156
156
  const float* centroids) const {
157
- /// TODO: actuallly we do not need to unpack and pack
157
+ /// TODO: actually we do not need to unpack and pack
158
158
  size_t offset_d = 0, offset_m = 0;
159
159
  std::vector<float> xsub;
160
160
  std::vector<uint8_t> codes;
@@ -46,7 +46,7 @@ struct ProductAdditiveQuantizer : AdditiveQuantizer {
46
46
 
47
47
  ProductAdditiveQuantizer();
48
48
 
49
- virtual ~ProductAdditiveQuantizer();
49
+ virtual ~ProductAdditiveQuantizer() override;
50
50
 
51
51
  void init(
52
52
  size_t d,
@@ -5,6 +5,8 @@
5
5
  * LICENSE file in the root directory of this source tree.
6
6
  */
7
7
 
8
+ #pragma once
9
+
8
10
  namespace faiss {
9
11
 
10
12
  inline PQEncoderGeneric::PQEncoderGeneric(
@@ -166,7 +166,7 @@ struct ProductQuantizer : Quantizer {
166
166
  /// Symmetric Distance Table
167
167
  std::vector<float> sdc_table;
168
168
 
169
- // intitialize the SDC table from the centroids
169
+ // initialize the SDC table from the centroids
170
170
  void compute_sdc_table();
171
171
 
172
172
  void search_sdc(