faiss 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 4541b0c40468723a8bed3db80d5778fb156afa999cbbd14b83653285b3ae6267
4
- data.tar.gz: 2beeacbad82a578e2a410938bc4447d73699ef3998c146b4309e4b44816f5e33
3
+ metadata.gz: 9d260b6585de456d0df620c322f5366a9ae29a1d275c0200c726233f5c5156aa
4
+ data.tar.gz: 1f463792926335213ae59a56c7271593fa9eaf93d4705fa936da02d74127a571
5
5
  SHA512:
6
- metadata.gz: 92d87492ff627e094ef29a48c8e4579dd976846ec9a95d94c45b816391dcea332d370a5fbde9c3f2bdc11964e061557d0e150b223ab6ac0209c092f1e1cd4a6e
7
- data.tar.gz: 86fc2aaf3151545f24128429cf1369074809f8449f277a2676e9dacea76c156191b3d6df6df83f0d7ac0f061bfdf537b81ef57f6cbbb861e4ba4b592d06e8ca2
6
+ metadata.gz: 70960a3c2244b8d8dceaa7a2cc98a04b8f765c783ff2d1879d61e2303f7a7337490b67a9a6a56d3f0a4e19939812418122394633f2a68c198a6e0776a3a2b29a
7
+ data.tar.gz: 0bc2edfdc86a7d3894923dfd98fe6db7dfa8a15cf640340198f92f7ef967b9548ed7a18e80d886018d8f621da01f73c1b061a85cf892318327144209d1a4ed87
data/CHANGELOG.md CHANGED
@@ -1,4 +1,9 @@
1
- ## 0.1.4 (2020-02-04)
1
+ ## 0.1.5 (2021-03-14)
2
+
3
+ - Added `--with-optflags` option
4
+ - Added support for inner product to `IndexHNSWFlat`
5
+
6
+ ## 0.1.4 (2021-02-04)
2
7
 
3
8
  - Updated Faiss to 1.7.0
4
9
 
data/README.md CHANGED
@@ -24,7 +24,7 @@ Add this line to your application’s Gemfile:
24
24
  gem 'faiss'
25
25
  ```
26
26
 
27
- Faiss is not available for Windows
27
+ It can take a few minutes to compile the gem. Faiss is not available for Windows.
28
28
 
29
29
  ## Getting Started
30
30
 
@@ -200,10 +200,10 @@ Data can be an array of arrays
200
200
  [[1, 2, 3], [4, 5, 6]]
201
201
  ```
202
202
 
203
- Or a Numo NArray
203
+ Or a Numo array
204
204
 
205
205
  ```ruby
206
- Numo::SFloat.new(3, 2).seq
206
+ Numo::NArray.cast([[1, 2, 3], [4, 5, 6]])
207
207
  ```
208
208
 
209
209
  ## History
data/ext/faiss/ext.cpp CHANGED
@@ -1,315 +1,19 @@
1
- #include <faiss/IndexFlat.h>
2
- #include <faiss/IndexHNSW.h>
3
- #include <faiss/IndexIVFFlat.h>
4
- #include <faiss/IndexLSH.h>
5
- #include <faiss/IndexScalarQuantizer.h>
6
- #include <faiss/IndexPQ.h>
7
- #include <faiss/IndexIVFPQ.h>
8
- #include <faiss/IndexIVFPQR.h>
9
-
10
- #include <faiss/IndexBinaryFlat.h>
11
- #include <faiss/IndexBinaryIVF.h>
12
- #include <faiss/index_factory.h>
13
- #include <faiss/index_io.h>
14
-
15
- #include <faiss/Clustering.h>
16
- #include <faiss/VectorTransform.h>
17
-
18
- #include <rice/Array.hpp>
19
- #include <rice/Class.hpp>
20
- #include <rice/Constructor.hpp>
21
1
  #include <rice/Module.hpp>
22
2
 
23
- float* float_array(Rice::Object o)
24
- {
25
- Rice::String s = o.call("to_binary");
26
- return (float*) s.c_str();
27
- }
28
-
29
- uint8_t* uint8_array(Rice::Object o)
30
- {
31
- Rice::String s = o.call("to_binary");
32
- return (uint8_t*) s.c_str();
33
- }
34
-
35
- // TODO return Numo::SFloat
36
- Rice::String result(float* ptr, int64_t length)
37
- {
38
- return Rice::String(std::string((char*) ptr, length * sizeof(float)));
39
- }
40
-
41
- // TODO return Numo::UInt8
42
- Rice::String result(uint8_t* ptr, int64_t length)
43
- {
44
- return Rice::String(std::string((char*) ptr, length * sizeof(uint8_t)));
45
- }
46
-
47
- // TODO return Numo::Int32
48
- Rice::String result(int32_t* ptr, int64_t length)
49
- {
50
- return Rice::String(std::string((char*) ptr, length * sizeof(int32_t)));
51
- }
52
-
53
- // TODO return Numo::Int64
54
- Rice::String result(int64_t* ptr, int64_t length)
55
- {
56
- return Rice::String(std::string((char*) ptr, length * sizeof(int64_t)));
57
- }
3
+ void init_index(Rice::Module& m);
4
+ void init_index_binary(Rice::Module& m);
5
+ void init_kmeans(Rice::Module& m);
6
+ void init_pca_matrix(Rice::Module& m);
7
+ void init_product_quantizer(Rice::Module& m);
58
8
 
59
9
  extern "C"
60
10
  void Init_ext()
61
11
  {
62
- Rice::Module rb_mFaiss = Rice::define_module("Faiss")
63
- .define_singleton_method(
64
- "index_binary_factory",
65
- *[](int d, const char *description) {
66
- return faiss::index_binary_factory(d, description);
67
- });
68
-
69
- Rice::define_class_under<faiss::Index>(rb_mFaiss, "Index")
70
- .define_method(
71
- "d",
72
- *[](faiss::Index &self) {
73
- return self.d;
74
- })
75
- .define_method(
76
- "trained?",
77
- *[](faiss::Index &self) {
78
- return self.is_trained;
79
- })
80
- .define_method(
81
- "ntotal",
82
- *[](faiss::Index &self) {
83
- return self.ntotal;
84
- })
85
- .define_method(
86
- "_train",
87
- *[](faiss::Index &self, int64_t n, Rice::Object o) {
88
- const float *x = float_array(o);
89
- self.train(n, x);
90
- })
91
- .define_method(
92
- "_add",
93
- *[](faiss::Index &self, int64_t n, Rice::Object o) {
94
- const float *x = float_array(o);
95
- self.add(n, x);
96
- })
97
- .define_method(
98
- "_search",
99
- *[](faiss::Index &self, int64_t n, Rice::Object o, int64_t k) {
100
- const float *x = float_array(o);
101
- float *distances = new float[k * n];
102
- int64_t *labels = new int64_t[k * n];
103
-
104
- self.search(n, x, k, distances, labels);
105
-
106
- auto dstr = result(distances, k * n);
107
- auto lstr = result(labels, k * n);
108
-
109
- Rice::Array ret;
110
- ret.push(dstr);
111
- ret.push(lstr);
112
- return ret;
113
- })
114
- .define_method(
115
- "save",
116
- *[](faiss::Index &self, const char *fname) {
117
- faiss::write_index(&self, fname);
118
- })
119
- .define_singleton_method(
120
- "load",
121
- *[](const char *fname) {
122
- return faiss::read_index(fname);
123
- });
124
-
125
- Rice::define_class_under<faiss::IndexBinary>(rb_mFaiss, "IndexBinary")
126
- .define_method(
127
- "d",
128
- *[](faiss::Index &self) {
129
- return self.d;
130
- })
131
- .define_method(
132
- "trained?",
133
- *[](faiss::IndexBinary &self) {
134
- return self.is_trained;
135
- })
136
- .define_method(
137
- "ntotal",
138
- *[](faiss::IndexBinary &self) {
139
- return self.ntotal;
140
- })
141
- .define_method(
142
- "_train",
143
- *[](faiss::IndexBinary &self, int64_t n, Rice::Object o) {
144
- const uint8_t *x = uint8_array(o);
145
- self.train(n, x);
146
- })
147
- .define_method(
148
- "_add",
149
- *[](faiss::IndexBinary &self, int64_t n, Rice::Object o) {
150
- const uint8_t *x = uint8_array(o);
151
- self.add(n, x);
152
- })
153
- .define_method(
154
- "_search",
155
- *[](faiss::IndexBinary &self, int64_t n, Rice::Object o, int64_t k) {
156
- const uint8_t *x = uint8_array(o);
157
- int32_t *distances = new int32_t[k * n];
158
- int64_t *labels = new int64_t[k * n];
159
-
160
- self.search(n, x, k, distances, labels);
161
-
162
- auto dstr = result(distances, k * n);
163
- auto lstr = result(labels, k * n);
164
-
165
- Rice::Array ret;
166
- ret.push(dstr);
167
- ret.push(lstr);
168
- return ret;
169
- })
170
- .define_method(
171
- "save",
172
- *[](faiss::IndexBinary &self, const char *fname) {
173
- faiss::write_index_binary(&self, fname);
174
- })
175
- .define_singleton_method(
176
- "load",
177
- *[](const char *fname) {
178
- return faiss::read_index_binary(fname);
179
- });
180
-
181
- Rice::define_class_under<faiss::IndexFlatL2, faiss::Index>(rb_mFaiss, "IndexFlatL2")
182
- .define_constructor(Rice::Constructor<faiss::IndexFlatL2, int64_t>());
183
-
184
- Rice::define_class_under<faiss::IndexFlatIP, faiss::Index>(rb_mFaiss, "IndexFlatIP")
185
- .define_constructor(Rice::Constructor<faiss::IndexFlatIP, int64_t>());
186
-
187
- Rice::define_class_under<faiss::IndexHNSWFlat, faiss::Index>(rb_mFaiss, "IndexHNSWFlat")
188
- .define_constructor(Rice::Constructor<faiss::IndexHNSWFlat, int, int>());
189
-
190
- Rice::define_class_under<faiss::IndexIVFFlat, faiss::Index>(rb_mFaiss, "IndexIVFFlat")
191
- .define_constructor(Rice::Constructor<faiss::IndexIVFFlat, faiss::Index*, size_t, size_t>());
192
-
193
- Rice::define_class_under<faiss::IndexLSH, faiss::Index>(rb_mFaiss, "IndexLSH")
194
- .define_constructor(Rice::Constructor<faiss::IndexLSH, int64_t, int>());
195
-
196
- Rice::define_class_under<faiss::IndexScalarQuantizer, faiss::Index>(rb_mFaiss, "IndexScalarQuantizer")
197
- .define_constructor(Rice::Constructor<faiss::IndexScalarQuantizer, int, faiss::ScalarQuantizer::QuantizerType>());
198
-
199
- Rice::define_class_under<faiss::IndexPQ, faiss::Index>(rb_mFaiss, "IndexPQ")
200
- .define_constructor(Rice::Constructor<faiss::IndexPQ, int, size_t, size_t>());
201
-
202
- Rice::define_class_under<faiss::IndexIVFScalarQuantizer, faiss::Index>(rb_mFaiss, "IndexIVFScalarQuantizer")
203
- .define_constructor(Rice::Constructor<faiss::IndexIVFScalarQuantizer, faiss::Index*, size_t, size_t, faiss::ScalarQuantizer::QuantizerType>());
204
-
205
- Rice::define_class_under<faiss::IndexIVFPQ, faiss::Index>(rb_mFaiss, "IndexIVFPQ")
206
- .define_constructor(Rice::Constructor<faiss::IndexIVFPQ, faiss::Index*, size_t, size_t, size_t, size_t>());
207
-
208
- Rice::define_class_under<faiss::IndexIVFPQR, faiss::Index>(rb_mFaiss, "IndexIVFPQR")
209
- .define_constructor(Rice::Constructor<faiss::IndexIVFPQR, faiss::Index*, size_t, size_t, size_t, size_t, size_t, size_t>());
210
-
211
- Rice::define_class_under<faiss::IndexBinaryFlat, faiss::IndexBinary>(rb_mFaiss, "IndexBinaryFlat")
212
- .define_constructor(Rice::Constructor<faiss::IndexBinaryFlat, int64_t>());
213
-
214
- Rice::define_class_under<faiss::IndexBinaryIVF, faiss::IndexBinary>(rb_mFaiss, "IndexBinaryIVF")
215
- .define_constructor(Rice::Constructor<faiss::IndexBinaryIVF, faiss::IndexBinary*, size_t, size_t>());
216
-
217
- Rice::define_class_under<faiss::Clustering>(rb_mFaiss, "Kmeans")
218
- .define_constructor(Rice::Constructor<faiss::Clustering, int, int>())
219
- .define_method(
220
- "d",
221
- *[](faiss::Clustering &self) {
222
- return self.d;
223
- })
224
- .define_method(
225
- "k",
226
- *[](faiss::Clustering &self) {
227
- return self.k;
228
- })
229
- .define_method(
230
- "_centroids",
231
- *[](faiss::Clustering &self) {
232
- float *centroids = new float[self.k * self.d];
233
- for (size_t i = 0; i < self.centroids.size(); i++) {
234
- centroids[i] = self.centroids[i];
235
- }
236
- return result(centroids, self.k * self.d);
237
- })
238
- .define_method(
239
- "_train",
240
- *[](faiss::Clustering &self, int64_t n, Rice::Object o, faiss::Index & index) {
241
- const float *x = float_array(o);
242
- self.train(n, x, index);
243
- });
244
-
245
- Rice::define_class_under<faiss::PCAMatrix>(rb_mFaiss, "PCAMatrix")
246
- .define_constructor(Rice::Constructor<faiss::PCAMatrix, int, int>())
247
- .define_method(
248
- "d_in",
249
- *[](faiss::PCAMatrix &self) {
250
- return self.d_in;
251
- })
252
- .define_method(
253
- "d_out",
254
- *[](faiss::PCAMatrix &self) {
255
- return self.d_out;
256
- })
257
- .define_method(
258
- "_train",
259
- *[](faiss::PCAMatrix &self, int64_t n, Rice::Object o) {
260
- const float *x = float_array(o);
261
- self.train(n, x);
262
- })
263
- .define_method(
264
- "_apply",
265
- *[](faiss::PCAMatrix &self, int64_t n, Rice::Object o) {
266
- const float *x = float_array(o);
267
- float* res = self.apply(n, x);
268
- return result(res, n * self.d_out);
269
- });
12
+ auto m = Rice::define_module("Faiss");
270
13
 
271
- Rice::define_class_under<faiss::ProductQuantizer>(rb_mFaiss, "ProductQuantizer")
272
- .define_constructor(Rice::Constructor<faiss::ProductQuantizer, size_t, size_t, size_t>())
273
- .define_method(
274
- "d",
275
- *[](faiss::ProductQuantizer &self) {
276
- return self.d;
277
- })
278
- .define_method(
279
- "m",
280
- *[](faiss::ProductQuantizer &self) {
281
- return self.M;
282
- })
283
- .define_method(
284
- "_train",
285
- *[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
286
- const float *x = float_array(o);
287
- self.train(n, x);
288
- })
289
- .define_method(
290
- "_compute_codes",
291
- *[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
292
- const float *x = float_array(o);
293
- uint8_t *codes = new uint8_t[n * self.M];
294
- self.compute_codes(x, codes, n);
295
- return result(codes, n * self.M);
296
- })
297
- .define_method(
298
- "_decode",
299
- *[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
300
- const uint8_t *codes = uint8_array(o);
301
- float *x = new float[n * self.d];
302
- self.decode(codes, x, n);
303
- return result(x, n * self.d);
304
- })
305
- .define_method(
306
- "save",
307
- *[](faiss::ProductQuantizer &self, const char *fname) {
308
- faiss::write_ProductQuantizer(&self, fname);
309
- })
310
- .define_singleton_method(
311
- "load",
312
- *[](const char *fname) {
313
- return faiss::read_ProductQuantizer(fname);
314
- });
14
+ init_index(m);
15
+ init_index_binary(m);
16
+ init_kmeans(m);
17
+ init_pca_matrix(m);
18
+ init_product_quantizer(m);
315
19
  }
data/ext/faiss/extconf.rb CHANGED
@@ -4,7 +4,7 @@ abort "BLAS not found" unless have_library("blas")
4
4
  abort "LAPACK not found" unless have_library("lapack")
5
5
  abort "OpenMP not found" unless have_library("omp") || have_library("gomp")
6
6
 
7
- $CXXFLAGS << " -std=c++11 -march=native -DFINTEGER=int"
7
+ $CXXFLAGS << " -std=c++11 -DFINTEGER=int " << with_config("optflags", "-march=native")
8
8
 
9
9
  ext = File.expand_path(".", __dir__)
10
10
  vendor = File.expand_path("../../vendor/faiss", __dir__)
@@ -0,0 +1,118 @@
1
+ #include <faiss/Index.h>
2
+ #include <faiss/IndexFlat.h>
3
+ #include <faiss/IndexHNSW.h>
4
+ #include <faiss/IndexIVFFlat.h>
5
+ #include <faiss/IndexLSH.h>
6
+ #include <faiss/IndexScalarQuantizer.h>
7
+ #include <faiss/IndexPQ.h>
8
+ #include <faiss/IndexIVFPQ.h>
9
+ #include <faiss/IndexIVFPQR.h>
10
+ #include <faiss/index_io.h>
11
+
12
+ #include <rice/Array.hpp>
13
+ #include <rice/Constructor.hpp>
14
+ #include <rice/Module.hpp>
15
+
16
+ #include "utils.h"
17
+
18
+ template<>
19
+ faiss::MetricType from_ruby<faiss::MetricType>(Rice::Object x)
20
+ {
21
+ auto s = x.to_s().str();
22
+ if (s == "inner_product") {
23
+ return faiss::METRIC_INNER_PRODUCT;
24
+ } else if (s == "l2") {
25
+ return faiss::METRIC_L2;
26
+ } else {
27
+ // TODO throw argument error
28
+ throw std::runtime_error("Invalid metric: " + s);
29
+ }
30
+ }
31
+
32
+ void init_index(Rice::Module& m) {
33
+ Rice::define_class_under<faiss::Index>(m, "Index")
34
+ .define_method(
35
+ "d",
36
+ *[](faiss::Index &self) {
37
+ return self.d;
38
+ })
39
+ .define_method(
40
+ "trained?",
41
+ *[](faiss::Index &self) {
42
+ return self.is_trained;
43
+ })
44
+ .define_method(
45
+ "ntotal",
46
+ *[](faiss::Index &self) {
47
+ return self.ntotal;
48
+ })
49
+ .define_method(
50
+ "_train",
51
+ *[](faiss::Index &self, int64_t n, Rice::Object o) {
52
+ const float *x = float_array(o);
53
+ self.train(n, x);
54
+ })
55
+ .define_method(
56
+ "_add",
57
+ *[](faiss::Index &self, int64_t n, Rice::Object o) {
58
+ const float *x = float_array(o);
59
+ self.add(n, x);
60
+ })
61
+ .define_method(
62
+ "_search",
63
+ *[](faiss::Index &self, int64_t n, Rice::Object o, int64_t k) {
64
+ const float *x = float_array(o);
65
+ float *distances = new float[k * n];
66
+ int64_t *labels = new int64_t[k * n];
67
+
68
+ self.search(n, x, k, distances, labels);
69
+
70
+ auto dstr = result(distances, k * n);
71
+ auto lstr = result(labels, k * n);
72
+
73
+ Rice::Array ret;
74
+ ret.push(dstr);
75
+ ret.push(lstr);
76
+ return ret;
77
+ })
78
+ .define_method(
79
+ "save",
80
+ *[](faiss::Index &self, const char *fname) {
81
+ faiss::write_index(&self, fname);
82
+ })
83
+ .define_singleton_method(
84
+ "load",
85
+ *[](const char *fname) {
86
+ return faiss::read_index(fname);
87
+ });
88
+
89
+ Rice::define_class_under<faiss::IndexFlatL2, faiss::Index>(m, "IndexFlatL2")
90
+ .define_constructor(Rice::Constructor<faiss::IndexFlatL2, int64_t>());
91
+
92
+ Rice::define_class_under<faiss::IndexFlatIP, faiss::Index>(m, "IndexFlatIP")
93
+ .define_constructor(Rice::Constructor<faiss::IndexFlatIP, int64_t>());
94
+
95
+ Rice::define_class_under<faiss::IndexHNSWFlat, faiss::Index>(m, "IndexHNSWFlat")
96
+ .define_constructor(Rice::Constructor<faiss::IndexHNSWFlat, int, int, faiss::MetricType>(), (Rice::Arg("d"), Rice::Arg("M"), Rice::Arg("metric") = faiss::METRIC_L2));
97
+
98
+ Rice::define_class_under<faiss::IndexIVFFlat, faiss::Index>(m, "IndexIVFFlat")
99
+ .define_constructor(Rice::Constructor<faiss::IndexIVFFlat, faiss::Index*, size_t, size_t>());
100
+
101
+ Rice::define_class_under<faiss::IndexLSH, faiss::Index>(m, "IndexLSH")
102
+ .define_constructor(Rice::Constructor<faiss::IndexLSH, int64_t, int>());
103
+
104
+ Rice::define_class_under<faiss::IndexScalarQuantizer, faiss::Index>(m, "IndexScalarQuantizer")
105
+ .define_constructor(Rice::Constructor<faiss::IndexScalarQuantizer, int, faiss::ScalarQuantizer::QuantizerType>());
106
+
107
+ Rice::define_class_under<faiss::IndexPQ, faiss::Index>(m, "IndexPQ")
108
+ .define_constructor(Rice::Constructor<faiss::IndexPQ, int, size_t, size_t>());
109
+
110
+ Rice::define_class_under<faiss::IndexIVFScalarQuantizer, faiss::Index>(m, "IndexIVFScalarQuantizer")
111
+ .define_constructor(Rice::Constructor<faiss::IndexIVFScalarQuantizer, faiss::Index*, size_t, size_t, faiss::ScalarQuantizer::QuantizerType>());
112
+
113
+ Rice::define_class_under<faiss::IndexIVFPQ, faiss::Index>(m, "IndexIVFPQ")
114
+ .define_constructor(Rice::Constructor<faiss::IndexIVFPQ, faiss::Index*, size_t, size_t, size_t, size_t>());
115
+
116
+ Rice::define_class_under<faiss::IndexIVFPQR, faiss::Index>(m, "IndexIVFPQR")
117
+ .define_constructor(Rice::Constructor<faiss::IndexIVFPQR, faiss::Index*, size_t, size_t, size_t, size_t, size_t, size_t>());
118
+ }
@@ -0,0 +1,81 @@
1
+ #include <faiss/IndexBinary.h>
2
+ #include <faiss/IndexBinaryFlat.h>
3
+ #include <faiss/IndexBinaryIVF.h>
4
+ #include <faiss/index_factory.h>
5
+ #include <faiss/index_io.h>
6
+
7
+ #include <rice/Array.hpp>
8
+ #include <rice/Constructor.hpp>
9
+ #include <rice/Module.hpp>
10
+
11
+ #include "utils.h"
12
+
13
+ void init_index_binary(Rice::Module& m) {
14
+ Rice::define_class_under<faiss::IndexBinary>(m, "IndexBinary")
15
+ .define_method(
16
+ "d",
17
+ *[](faiss::IndexBinary &self) {
18
+ return self.d;
19
+ })
20
+ .define_method(
21
+ "trained?",
22
+ *[](faiss::IndexBinary &self) {
23
+ return self.is_trained;
24
+ })
25
+ .define_method(
26
+ "ntotal",
27
+ *[](faiss::IndexBinary &self) {
28
+ return self.ntotal;
29
+ })
30
+ .define_method(
31
+ "_train",
32
+ *[](faiss::IndexBinary &self, int64_t n, Rice::Object o) {
33
+ const uint8_t *x = uint8_array(o);
34
+ self.train(n, x);
35
+ })
36
+ .define_method(
37
+ "_add",
38
+ *[](faiss::IndexBinary &self, int64_t n, Rice::Object o) {
39
+ const uint8_t *x = uint8_array(o);
40
+ self.add(n, x);
41
+ })
42
+ .define_method(
43
+ "_search",
44
+ *[](faiss::IndexBinary &self, int64_t n, Rice::Object o, int64_t k) {
45
+ const uint8_t *x = uint8_array(o);
46
+ int32_t *distances = new int32_t[k * n];
47
+ int64_t *labels = new int64_t[k * n];
48
+
49
+ self.search(n, x, k, distances, labels);
50
+
51
+ auto dstr = result(distances, k * n);
52
+ auto lstr = result(labels, k * n);
53
+
54
+ Rice::Array ret;
55
+ ret.push(dstr);
56
+ ret.push(lstr);
57
+ return ret;
58
+ })
59
+ .define_method(
60
+ "save",
61
+ *[](faiss::IndexBinary &self, const char *fname) {
62
+ faiss::write_index_binary(&self, fname);
63
+ })
64
+ .define_singleton_method(
65
+ "load",
66
+ *[](const char *fname) {
67
+ return faiss::read_index_binary(fname);
68
+ });
69
+
70
+ Rice::define_class_under<faiss::IndexBinaryFlat, faiss::IndexBinary>(m, "IndexBinaryFlat")
71
+ .define_constructor(Rice::Constructor<faiss::IndexBinaryFlat, int64_t>());
72
+
73
+ Rice::define_class_under<faiss::IndexBinaryIVF, faiss::IndexBinary>(m, "IndexBinaryIVF")
74
+ .define_constructor(Rice::Constructor<faiss::IndexBinaryIVF, faiss::IndexBinary*, size_t, size_t>());
75
+
76
+ m.define_singleton_method(
77
+ "index_binary_factory",
78
+ *[](int d, const char *description) {
79
+ return faiss::index_binary_factory(d, description);
80
+ });
81
+ }
@@ -0,0 +1,36 @@
1
+ #include <faiss/Clustering.h>
2
+
3
+ #include <rice/Constructor.hpp>
4
+ #include <rice/Module.hpp>
5
+
6
+ #include "utils.h"
7
+
8
+ void init_kmeans(Rice::Module& m) {
9
+ Rice::define_class_under<faiss::Clustering>(m, "Kmeans")
10
+ .define_constructor(Rice::Constructor<faiss::Clustering, int, int>())
11
+ .define_method(
12
+ "d",
13
+ *[](faiss::Clustering &self) {
14
+ return self.d;
15
+ })
16
+ .define_method(
17
+ "k",
18
+ *[](faiss::Clustering &self) {
19
+ return self.k;
20
+ })
21
+ .define_method(
22
+ "_centroids",
23
+ *[](faiss::Clustering &self) {
24
+ float *centroids = new float[self.k * self.d];
25
+ for (size_t i = 0; i < self.centroids.size(); i++) {
26
+ centroids[i] = self.centroids[i];
27
+ }
28
+ return result(centroids, self.k * self.d);
29
+ })
30
+ .define_method(
31
+ "_train",
32
+ *[](faiss::Clustering &self, int64_t n, Rice::Object o, faiss::Index & index) {
33
+ const float *x = float_array(o);
34
+ self.train(n, x, index);
35
+ });
36
+ }
@@ -0,0 +1,34 @@
1
+ #include <faiss/VectorTransform.h>
2
+
3
+ #include <rice/Constructor.hpp>
4
+ #include <rice/Module.hpp>
5
+
6
+ #include "utils.h"
7
+
8
+ void init_pca_matrix(Rice::Module& m) {
9
+ Rice::define_class_under<faiss::PCAMatrix>(m, "PCAMatrix")
10
+ .define_constructor(Rice::Constructor<faiss::PCAMatrix, int, int>())
11
+ .define_method(
12
+ "d_in",
13
+ *[](faiss::PCAMatrix &self) {
14
+ return self.d_in;
15
+ })
16
+ .define_method(
17
+ "d_out",
18
+ *[](faiss::PCAMatrix &self) {
19
+ return self.d_out;
20
+ })
21
+ .define_method(
22
+ "_train",
23
+ *[](faiss::PCAMatrix &self, int64_t n, Rice::Object o) {
24
+ const float *x = float_array(o);
25
+ self.train(n, x);
26
+ })
27
+ .define_method(
28
+ "_apply",
29
+ *[](faiss::PCAMatrix &self, int64_t n, Rice::Object o) {
30
+ const float *x = float_array(o);
31
+ float* res = self.apply(n, x);
32
+ return result(res, n * self.d_out);
33
+ });
34
+ }
@@ -0,0 +1,54 @@
1
+ #include <faiss/impl/ProductQuantizer.h>
2
+ #include <faiss/index_io.h>
3
+
4
+ #include <rice/Constructor.hpp>
5
+ #include <rice/Module.hpp>
6
+
7
+ #include "utils.h"
8
+
9
+ void init_product_quantizer(Rice::Module& m) {
10
+ Rice::define_class_under<faiss::ProductQuantizer>(m, "ProductQuantizer")
11
+ .define_constructor(Rice::Constructor<faiss::ProductQuantizer, size_t, size_t, size_t>())
12
+ .define_method(
13
+ "d",
14
+ *[](faiss::ProductQuantizer &self) {
15
+ return self.d;
16
+ })
17
+ .define_method(
18
+ "m",
19
+ *[](faiss::ProductQuantizer &self) {
20
+ return self.M;
21
+ })
22
+ .define_method(
23
+ "_train",
24
+ *[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
25
+ const float *x = float_array(o);
26
+ self.train(n, x);
27
+ })
28
+ .define_method(
29
+ "_compute_codes",
30
+ *[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
31
+ const float *x = float_array(o);
32
+ uint8_t *codes = new uint8_t[n * self.M];
33
+ self.compute_codes(x, codes, n);
34
+ return result(codes, n * self.M);
35
+ })
36
+ .define_method(
37
+ "_decode",
38
+ *[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
39
+ const uint8_t *codes = uint8_array(o);
40
+ float *x = new float[n * self.d];
41
+ self.decode(codes, x, n);
42
+ return result(x, n * self.d);
43
+ })
44
+ .define_method(
45
+ "save",
46
+ *[](faiss::ProductQuantizer &self, const char *fname) {
47
+ faiss::write_ProductQuantizer(&self, fname);
48
+ })
49
+ .define_singleton_method(
50
+ "load",
51
+ *[](const char *fname) {
52
+ return faiss::read_ProductQuantizer(fname);
53
+ });
54
+ }
@@ -0,0 +1,40 @@
1
+ #include "utils.h"
2
+
3
+ #include <rice/Object.hpp>
4
+ #include <rice/String.hpp>
5
+
6
+ float* float_array(Rice::Object o)
7
+ {
8
+ Rice::String s = o.call("to_binary");
9
+ return (float*) s.c_str();
10
+ }
11
+
12
+ uint8_t* uint8_array(Rice::Object o)
13
+ {
14
+ Rice::String s = o.call("to_binary");
15
+ return (uint8_t*) s.c_str();
16
+ }
17
+
18
+ // TODO return Numo::SFloat
19
+ Rice::String result(float* ptr, int64_t length)
20
+ {
21
+ return Rice::String(std::string((char*) ptr, length * sizeof(float)));
22
+ }
23
+
24
+ // TODO return Numo::UInt8
25
+ Rice::String result(uint8_t* ptr, int64_t length)
26
+ {
27
+ return Rice::String(std::string((char*) ptr, length * sizeof(uint8_t)));
28
+ }
29
+
30
+ // TODO return Numo::Int32
31
+ Rice::String result(int32_t* ptr, int64_t length)
32
+ {
33
+ return Rice::String(std::string((char*) ptr, length * sizeof(int32_t)));
34
+ }
35
+
36
+ // TODO return Numo::Int64
37
+ Rice::String result(int64_t* ptr, int64_t length)
38
+ {
39
+ return Rice::String(std::string((char*) ptr, length * sizeof(int64_t)));
40
+ }
data/ext/faiss/utils.h ADDED
@@ -0,0 +1,16 @@
1
+ #pragma once
2
+
3
+ #include <rice/Object.hpp>
4
+ #include <rice/String.hpp>
5
+
6
+ float* float_array(Rice::Object o);
7
+ uint8_t* uint8_array(Rice::Object o);
8
+
9
+ // TODO return Numo::SFloat
10
+ Rice::String result(float* ptr, int64_t length);
11
+ // TODO return Numo::UInt8
12
+ Rice::String result(uint8_t* ptr, int64_t length);
13
+ // TODO return Numo::Int32
14
+ Rice::String result(int32_t* ptr, int64_t length);
15
+ // TODO return Numo::Int64
16
+ Rice::String result(int64_t* ptr, int64_t length);
data/lib/faiss/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Faiss
2
- VERSION = "0.1.4"
2
+ VERSION = "0.1.5"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: faiss
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.4
4
+ version: 0.1.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Kane
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-02-05 00:00:00.000000000 Z
11
+ date: 2021-03-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rice
@@ -50,6 +50,13 @@ files:
50
50
  - README.md
51
51
  - ext/faiss/ext.cpp
52
52
  - ext/faiss/extconf.rb
53
+ - ext/faiss/index.cpp
54
+ - ext/faiss/index_binary.cpp
55
+ - ext/faiss/kmeans.cpp
56
+ - ext/faiss/pca_matrix.cpp
57
+ - ext/faiss/product_quantizer.cpp
58
+ - ext/faiss/utils.cpp
59
+ - ext/faiss/utils.h
53
60
  - lib/faiss.rb
54
61
  - lib/faiss/index.rb
55
62
  - lib/faiss/index_binary.rb