faiss 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +4 -0
  3. data/LICENSE.txt +18 -18
  4. data/README.md +1 -1
  5. data/lib/faiss/version.rb +1 -1
  6. data/vendor/faiss/Clustering.cpp +318 -53
  7. data/vendor/faiss/Clustering.h +39 -11
  8. data/vendor/faiss/DirectMap.cpp +267 -0
  9. data/vendor/faiss/DirectMap.h +120 -0
  10. data/vendor/faiss/IVFlib.cpp +24 -4
  11. data/vendor/faiss/IVFlib.h +4 -0
  12. data/vendor/faiss/Index.h +5 -24
  13. data/vendor/faiss/Index2Layer.cpp +0 -1
  14. data/vendor/faiss/IndexBinary.h +7 -3
  15. data/vendor/faiss/IndexBinaryFlat.cpp +5 -0
  16. data/vendor/faiss/IndexBinaryFlat.h +3 -0
  17. data/vendor/faiss/IndexBinaryHash.cpp +492 -0
  18. data/vendor/faiss/IndexBinaryHash.h +116 -0
  19. data/vendor/faiss/IndexBinaryIVF.cpp +160 -107
  20. data/vendor/faiss/IndexBinaryIVF.h +14 -4
  21. data/vendor/faiss/IndexFlat.h +2 -1
  22. data/vendor/faiss/IndexHNSW.cpp +68 -16
  23. data/vendor/faiss/IndexHNSW.h +3 -3
  24. data/vendor/faiss/IndexIVF.cpp +72 -76
  25. data/vendor/faiss/IndexIVF.h +24 -5
  26. data/vendor/faiss/IndexIVFFlat.cpp +19 -54
  27. data/vendor/faiss/IndexIVFFlat.h +1 -11
  28. data/vendor/faiss/IndexIVFPQ.cpp +49 -26
  29. data/vendor/faiss/IndexIVFPQ.h +9 -10
  30. data/vendor/faiss/IndexIVFPQR.cpp +2 -2
  31. data/vendor/faiss/IndexIVFSpectralHash.cpp +2 -2
  32. data/vendor/faiss/IndexLSH.h +4 -1
  33. data/vendor/faiss/IndexPreTransform.cpp +0 -1
  34. data/vendor/faiss/IndexScalarQuantizer.cpp +8 -1
  35. data/vendor/faiss/InvertedLists.cpp +0 -2
  36. data/vendor/faiss/MetaIndexes.cpp +0 -1
  37. data/vendor/faiss/MetricType.h +36 -0
  38. data/vendor/faiss/c_api/Clustering_c.cpp +13 -7
  39. data/vendor/faiss/c_api/Clustering_c.h +11 -5
  40. data/vendor/faiss/c_api/IndexIVF_c.cpp +7 -0
  41. data/vendor/faiss/c_api/IndexIVF_c.h +7 -0
  42. data/vendor/faiss/c_api/IndexPreTransform_c.cpp +21 -0
  43. data/vendor/faiss/c_api/IndexPreTransform_c.h +32 -0
  44. data/vendor/faiss/demos/demo_weighted_kmeans.cpp +185 -0
  45. data/vendor/faiss/gpu/GpuCloner.cpp +4 -0
  46. data/vendor/faiss/gpu/GpuClonerOptions.cpp +1 -1
  47. data/vendor/faiss/gpu/GpuDistance.h +93 -0
  48. data/vendor/faiss/gpu/GpuIndex.h +7 -0
  49. data/vendor/faiss/gpu/GpuIndexFlat.h +0 -10
  50. data/vendor/faiss/gpu/GpuIndexIVF.h +1 -0
  51. data/vendor/faiss/gpu/StandardGpuResources.cpp +8 -0
  52. data/vendor/faiss/gpu/test/TestGpuIndexFlat.cpp +49 -27
  53. data/vendor/faiss/gpu/test/TestGpuIndexIVFPQ.cpp +110 -2
  54. data/vendor/faiss/gpu/utils/DeviceUtils.h +6 -0
  55. data/vendor/faiss/impl/AuxIndexStructures.cpp +17 -0
  56. data/vendor/faiss/impl/AuxIndexStructures.h +14 -3
  57. data/vendor/faiss/impl/HNSW.cpp +0 -1
  58. data/vendor/faiss/impl/PolysemousTraining.h +5 -5
  59. data/vendor/faiss/impl/ProductQuantizer-inl.h +138 -0
  60. data/vendor/faiss/impl/ProductQuantizer.cpp +1 -113
  61. data/vendor/faiss/impl/ProductQuantizer.h +42 -47
  62. data/vendor/faiss/impl/index_read.cpp +103 -7
  63. data/vendor/faiss/impl/index_write.cpp +101 -5
  64. data/vendor/faiss/impl/io.cpp +111 -1
  65. data/vendor/faiss/impl/io.h +38 -0
  66. data/vendor/faiss/index_factory.cpp +0 -1
  67. data/vendor/faiss/tests/test_merge.cpp +0 -1
  68. data/vendor/faiss/tests/test_pq_encoding.cpp +6 -6
  69. data/vendor/faiss/tutorial/cpp/5-Multiple-GPUs.cpp +1 -0
  70. data/vendor/faiss/utils/distances.cpp +4 -5
  71. data/vendor/faiss/utils/distances_simd.cpp +0 -1
  72. data/vendor/faiss/utils/hamming.cpp +85 -3
  73. data/vendor/faiss/utils/hamming.h +20 -0
  74. data/vendor/faiss/utils/utils.cpp +0 -96
  75. data/vendor/faiss/utils/utils.h +0 -15
  76. metadata +11 -3
  77. data/lib/faiss/ext.bundle +0 -0
@@ -64,6 +64,12 @@ bool getFullUnifiedMemSupport(int device);
64
64
  /// Equivalent to getFullUnifiedMemSupport(getCurrentDevice())
65
65
  bool getFullUnifiedMemSupportCurrentDevice();
66
66
 
67
+ /// Does the given device support tensor core operations?
68
+ bool getTensorCoreSupport(int device);
69
+
70
+ /// Equivalent to getTensorCoreSupport(getCurrentDevice())
71
+ bool getTensorCoreSupportCurrentDevice();
72
+
67
73
  /// Returns the maximum k-selection value supported based on the CUDA SDK that
68
74
  /// we were compiled with. .cu files can use DeviceDefs.cuh, but this is for
69
75
  /// non-CUDA files
@@ -228,6 +228,23 @@ bool IDSelectorRange::is_member (idx_t id) const
228
228
  return id >= imin && id < imax;
229
229
  }
230
230
 
231
+ /***********************************************************************
232
+ * IDSelectorArray
233
+ ***********************************************************************/
234
+
235
+ IDSelectorArray::IDSelectorArray (size_t n, const idx_t *ids):
236
+ n (n), ids(ids)
237
+ {
238
+ }
239
+
240
+ bool IDSelectorArray::is_member (idx_t id) const
241
+ {
242
+ for (idx_t i = 0; i < n; i++) {
243
+ if (ids[i] == id) return true;
244
+ }
245
+ return false;
246
+ }
247
+
231
248
 
232
249
  /***********************************************************************
233
250
  * IDSelectorBatch
@@ -51,9 +51,7 @@ struct RangeSearchResult {
51
51
  };
52
52
 
53
53
 
54
- /**
55
-
56
- Encapsulates a set of ids to remove. */
54
+ /** Encapsulates a set of ids to remove. */
57
55
  struct IDSelector {
58
56
  typedef Index::idx_t idx_t;
59
57
  virtual bool is_member (idx_t id) const = 0;
@@ -71,6 +69,19 @@ struct IDSelectorRange: IDSelector {
71
69
  ~IDSelectorRange() override {}
72
70
  };
73
71
 
72
+ /** simple list of elements to remove
73
+ *
74
+ * this is inefficient in most cases, except for IndexIVF with
75
+ * maintain_direct_map
76
+ */
77
+ struct IDSelectorArray: IDSelector {
78
+ size_t n;
79
+ const idx_t *ids;
80
+
81
+ IDSelectorArray (size_t n, const idx_t *ids);
82
+ bool is_member(idx_t id) const override;
83
+ ~IDSelectorArray() override {}
84
+ };
74
85
 
75
86
  /** Remove ids from a set. Repetitions of ids in the indices set
76
87
  * passed to the constructor does not hurt performance. The hash
@@ -15,7 +15,6 @@
15
15
 
16
16
  namespace faiss {
17
17
 
18
- using idx_t = Index::idx_t;
19
18
 
20
19
  /**************************************************************
21
20
  * HNSW structure implementation
@@ -123,15 +123,15 @@ struct PolysemousTraining: SimulatedAnnealingParameters {
123
123
  enum Optimization_type_t {
124
124
  OT_None,
125
125
  OT_ReproduceDistances_affine, ///< default
126
- OT_Ranking_weighted_diff /// same as _2, but use rank of y+ - rank of y-
126
+ OT_Ranking_weighted_diff ///< same as _2, but use rank of y+ - rank of y-
127
127
  };
128
128
  Optimization_type_t optimization_type;
129
129
 
130
- // use 1/4 of the training points for the optimization, with
131
- // max. ntrain_permutation. If ntrain_permutation == 0: train on
132
- // centroids
130
+ /** use 1/4 of the training points for the optimization, with
131
+ * max. ntrain_permutation. If ntrain_permutation == 0: train on
132
+ * centroids */
133
133
  int ntrain_permutation;
134
- double dis_weight_factor; // decay of exp that weights distance loss
134
+ double dis_weight_factor; ///< decay of exp that weights distance loss
135
135
 
136
136
  // filename pattern for the logging of iterations
137
137
  std::string log_pattern;
@@ -0,0 +1,138 @@
1
+ /**
2
+ * Copyright (c) Facebook, Inc. and its affiliates.
3
+ *
4
+ * This source code is licensed under the MIT license found in the
5
+ * LICENSE file in the root directory of this source tree.
6
+ */
7
+
8
+ namespace faiss {
9
+
10
+ inline
11
+ PQEncoderGeneric::PQEncoderGeneric(uint8_t *code, int nbits,
12
+ uint8_t offset)
13
+ : code(code), offset(offset), nbits(nbits), reg(0)
14
+ {
15
+ assert(nbits <= 64);
16
+ if (offset > 0) {
17
+ reg = (*code & ((1 << offset) - 1));
18
+ }
19
+ }
20
+
21
+ inline
22
+ void PQEncoderGeneric::encode(uint64_t x)
23
+ {
24
+ reg |= (uint8_t)(x << offset);
25
+ x >>= (8 - offset);
26
+ if (offset + nbits >= 8) {
27
+ *code++ = reg;
28
+
29
+ for (int i = 0; i < (nbits - (8 - offset)) / 8; ++i) {
30
+ *code++ = (uint8_t)x;
31
+ x >>= 8;
32
+ }
33
+
34
+ offset += nbits;
35
+ offset &= 7;
36
+ reg = (uint8_t)x;
37
+ } else {
38
+ offset += nbits;
39
+ }
40
+ }
41
+
42
+ inline
43
+ PQEncoderGeneric::~PQEncoderGeneric()
44
+ {
45
+ if (offset > 0) {
46
+ *code = reg;
47
+ }
48
+ }
49
+
50
+
51
+ inline
52
+ PQEncoder8::PQEncoder8(uint8_t *code, int nbits)
53
+ : code(code) {
54
+ assert(8 == nbits);
55
+ }
56
+
57
+ inline
58
+ void PQEncoder8::encode(uint64_t x) {
59
+ *code++ = (uint8_t)x;
60
+ }
61
+
62
+ inline
63
+ PQEncoder16::PQEncoder16(uint8_t *code, int nbits)
64
+ : code((uint16_t *)code) {
65
+ assert(16 == nbits);
66
+ }
67
+
68
+ inline
69
+ void PQEncoder16::encode(uint64_t x) {
70
+ *code++ = (uint16_t)x;
71
+ }
72
+
73
+
74
+ inline
75
+ PQDecoderGeneric::PQDecoderGeneric(const uint8_t *code,
76
+ int nbits)
77
+ : code(code),
78
+ offset(0),
79
+ nbits(nbits),
80
+ mask((1ull << nbits) - 1),
81
+ reg(0) {
82
+ assert(nbits <= 64);
83
+ }
84
+
85
+ inline
86
+ uint64_t PQDecoderGeneric::decode() {
87
+ if (offset == 0) {
88
+ reg = *code;
89
+ }
90
+ uint64_t c = (reg >> offset);
91
+
92
+ if (offset + nbits >= 8) {
93
+ uint64_t e = 8 - offset;
94
+ ++code;
95
+ for (int i = 0; i < (nbits - (8 - offset)) / 8; ++i) {
96
+ c |= ((uint64_t)(*code++) << e);
97
+ e += 8;
98
+ }
99
+
100
+ offset += nbits;
101
+ offset &= 7;
102
+ if (offset > 0) {
103
+ reg = *code;
104
+ c |= ((uint64_t)reg << e);
105
+ }
106
+ } else {
107
+ offset += nbits;
108
+ }
109
+
110
+ return c & mask;
111
+ }
112
+
113
+
114
+ inline
115
+ PQDecoder8::PQDecoder8(const uint8_t *code, int nbits)
116
+ : code(code) {
117
+ assert(8 == nbits);
118
+ }
119
+
120
+ inline
121
+ uint64_t PQDecoder8::decode() {
122
+ return (uint64_t)(*code++);
123
+ }
124
+
125
+
126
+ inline
127
+ PQDecoder16::PQDecoder16(const uint8_t *code, int nbits)
128
+ : code((uint16_t *)code) {
129
+ assert(16 == nbits);
130
+ }
131
+
132
+ inline
133
+ uint64_t PQDecoder16::decode() {
134
+ return (uint64_t)(*code++);
135
+ }
136
+
137
+
138
+ } // namespace faiss
@@ -151,7 +151,7 @@ static inline void pq_estimators_from_tables_generic(const ProductQuantizer& pq,
151
151
  const size_t M = pq.M;
152
152
  const size_t ksub = pq.ksub;
153
153
  for (size_t j = 0; j < ncodes; ++j) {
154
- faiss::ProductQuantizer::PQDecoderGeneric decoder(
154
+ PQDecoderGeneric decoder(
155
155
  codes + j * pq.code_size, nbits
156
156
  );
157
157
  float dis = 0;
@@ -760,117 +760,5 @@ void ProductQuantizer::search_sdc (const uint8_t * qcodes,
760
760
  }
761
761
 
762
762
 
763
- ProductQuantizer::PQEncoderGeneric::PQEncoderGeneric(uint8_t *code, int nbits,
764
- uint8_t offset)
765
- : code(code), offset(offset), nbits(nbits), reg(0) {
766
- assert(nbits <= 64);
767
- if (offset > 0) {
768
- reg = (*code & ((1 << offset) - 1));
769
- }
770
- }
771
-
772
- void ProductQuantizer::PQEncoderGeneric::encode(uint64_t x) {
773
- reg |= (uint8_t)(x << offset);
774
- x >>= (8 - offset);
775
- if (offset + nbits >= 8) {
776
- *code++ = reg;
777
-
778
- for (int i = 0; i < (nbits - (8 - offset)) / 8; ++i) {
779
- *code++ = (uint8_t)x;
780
- x >>= 8;
781
- }
782
-
783
- offset += nbits;
784
- offset &= 7;
785
- reg = (uint8_t)x;
786
- } else {
787
- offset += nbits;
788
- }
789
- }
790
-
791
- ProductQuantizer::PQEncoderGeneric::~PQEncoderGeneric() {
792
- if (offset > 0) {
793
- *code = reg;
794
- }
795
- }
796
-
797
-
798
- ProductQuantizer::PQEncoder8::PQEncoder8(uint8_t *code, int nbits)
799
- : code(code) {
800
- assert(8 == nbits);
801
- }
802
-
803
- void ProductQuantizer::PQEncoder8::encode(uint64_t x) {
804
- *code++ = (uint8_t)x;
805
- }
806
-
807
-
808
- ProductQuantizer::PQEncoder16::PQEncoder16(uint8_t *code, int nbits)
809
- : code((uint16_t *)code) {
810
- assert(16 == nbits);
811
- }
812
-
813
- void ProductQuantizer::PQEncoder16::encode(uint64_t x) {
814
- *code++ = (uint16_t)x;
815
- }
816
-
817
-
818
- ProductQuantizer::PQDecoderGeneric::PQDecoderGeneric(const uint8_t *code,
819
- int nbits)
820
- : code(code),
821
- offset(0),
822
- nbits(nbits),
823
- mask((1ull << nbits) - 1),
824
- reg(0) {
825
- assert(nbits <= 64);
826
- }
827
-
828
- uint64_t ProductQuantizer::PQDecoderGeneric::decode() {
829
- if (offset == 0) {
830
- reg = *code;
831
- }
832
- uint64_t c = (reg >> offset);
833
-
834
- if (offset + nbits >= 8) {
835
- uint64_t e = 8 - offset;
836
- ++code;
837
- for (int i = 0; i < (nbits - (8 - offset)) / 8; ++i) {
838
- c |= ((uint64_t)(*code++) << e);
839
- e += 8;
840
- }
841
-
842
- offset += nbits;
843
- offset &= 7;
844
- if (offset > 0) {
845
- reg = *code;
846
- c |= ((uint64_t)reg << e);
847
- }
848
- } else {
849
- offset += nbits;
850
- }
851
-
852
- return c & mask;
853
- }
854
-
855
-
856
- ProductQuantizer::PQDecoder8::PQDecoder8(const uint8_t *code, int nbits)
857
- : code(code) {
858
- assert(8 == nbits);
859
- }
860
-
861
- uint64_t ProductQuantizer::PQDecoder8::decode() {
862
- return (uint64_t)(*code++);
863
- }
864
-
865
-
866
- ProductQuantizer::PQDecoder16::PQDecoder16(const uint8_t *code, int nbits)
867
- : code((uint16_t *)code) {
868
- assert(16 == nbits);
869
- }
870
-
871
- uint64_t ProductQuantizer::PQDecoder16::decode() {
872
- return (uint64_t)(*code++);
873
- }
874
-
875
763
 
876
764
  } // namespace faiss
@@ -173,70 +173,65 @@ struct ProductQuantizer {
173
173
  float_maxheap_array_t * res,
174
174
  bool init_finalize_heap = true) const;
175
175
 
176
- struct PQEncoderGeneric {
177
- uint8_t *code; ///< code for this vector
178
- uint8_t offset;
179
- const int nbits; ///< number of bits per subquantizer index
180
-
181
- uint8_t reg;
182
-
183
- PQEncoderGeneric(uint8_t *code, int nbits, uint8_t offset = 0);
184
-
185
- void encode(uint64_t x);
186
-
187
- ~PQEncoderGeneric();
188
- };
189
-
190
-
191
- struct PQEncoder8 {
192
- uint8_t *code;
193
-
194
- PQEncoder8(uint8_t *code, int nbits);
195
-
196
- void encode(uint64_t x);
197
- };
198
-
199
- struct PQEncoder16 {
200
- uint16_t *code;
176
+ };
201
177
 
202
- PQEncoder16(uint8_t *code, int nbits);
203
178
 
204
- void encode(uint64_t x);
205
- };
179
+ /*************************************************
180
+ * Objects to encode / decode strings of bits
181
+ *************************************************/
206
182
 
183
+ struct PQEncoderGeneric {
184
+ uint8_t *code; ///< code for this vector
185
+ uint8_t offset;
186
+ const int nbits; ///< number of bits per subquantizer index
207
187
 
208
- struct PQDecoderGeneric {
209
- const uint8_t *code;
210
- uint8_t offset;
211
- const int nbits;
212
- const uint64_t mask;
213
- uint8_t reg;
188
+ uint8_t reg;
214
189
 
215
- PQDecoderGeneric(const uint8_t *code, int nbits);
190
+ PQEncoderGeneric(uint8_t *code, int nbits, uint8_t offset = 0);
216
191
 
217
- uint64_t decode();
218
- };
192
+ void encode(uint64_t x);
219
193
 
220
- struct PQDecoder8 {
221
- const uint8_t *code;
194
+ ~PQEncoderGeneric();
195
+ };
222
196
 
223
- PQDecoder8(const uint8_t *code, int nbits);
224
197
 
225
- uint64_t decode();
226
- };
198
+ struct PQEncoder8 {
199
+ uint8_t *code;
200
+ PQEncoder8(uint8_t *code, int nbits);
201
+ void encode(uint64_t x);
202
+ };
227
203
 
228
- struct PQDecoder16 {
229
- const uint16_t *code;
204
+ struct PQEncoder16 {
205
+ uint16_t *code;
206
+ PQEncoder16(uint8_t *code, int nbits);
207
+ void encode(uint64_t x);
208
+ };
230
209
 
231
- PQDecoder16(const uint8_t *code, int nbits);
232
210
 
233
- uint64_t decode();
234
- };
211
+ struct PQDecoderGeneric {
212
+ const uint8_t *code;
213
+ uint8_t offset;
214
+ const int nbits;
215
+ const uint64_t mask;
216
+ uint8_t reg;
217
+ PQDecoderGeneric(const uint8_t *code, int nbits);
218
+ uint64_t decode();
219
+ };
235
220
 
221
+ struct PQDecoder8 {
222
+ const uint8_t *code;
223
+ PQDecoder8(const uint8_t *code, int nbits);
224
+ uint64_t decode();
236
225
  };
237
226
 
227
+ struct PQDecoder16 {
228
+ const uint16_t *code;
229
+ PQDecoder16(const uint8_t *code, int nbits);
230
+ uint64_t decode();
231
+ };
238
232
 
239
233
  } // namespace faiss
240
234
 
235
+ #include <faiss/impl/ProductQuantizer-inl.h>
241
236
 
242
237
  #endif