faiss 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -1
- data/README.md +3 -3
- data/ext/faiss/ext.cpp +11 -307
- data/ext/faiss/extconf.rb +1 -1
- data/ext/faiss/index.cpp +118 -0
- data/ext/faiss/index_binary.cpp +81 -0
- data/ext/faiss/kmeans.cpp +36 -0
- data/ext/faiss/pca_matrix.cpp +34 -0
- data/ext/faiss/product_quantizer.cpp +54 -0
- data/ext/faiss/utils.cpp +40 -0
- data/ext/faiss/utils.h +16 -0
- data/lib/faiss/version.rb +1 -1
- metadata +9 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9d260b6585de456d0df620c322f5366a9ae29a1d275c0200c726233f5c5156aa
|
4
|
+
data.tar.gz: 1f463792926335213ae59a56c7271593fa9eaf93d4705fa936da02d74127a571
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 70960a3c2244b8d8dceaa7a2cc98a04b8f765c783ff2d1879d61e2303f7a7337490b67a9a6a56d3f0a4e19939812418122394633f2a68c198a6e0776a3a2b29a
|
7
|
+
data.tar.gz: 0bc2edfdc86a7d3894923dfd98fe6db7dfa8a15cf640340198f92f7ef967b9548ed7a18e80d886018d8f621da01f73c1b061a85cf892318327144209d1a4ed87
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -24,7 +24,7 @@ Add this line to your application’s Gemfile:
|
|
24
24
|
gem 'faiss'
|
25
25
|
```
|
26
26
|
|
27
|
-
Faiss is not available for Windows
|
27
|
+
It can take a few minutes to compile the gem. Faiss is not available for Windows.
|
28
28
|
|
29
29
|
## Getting Started
|
30
30
|
|
@@ -200,10 +200,10 @@ Data can be an array of arrays
|
|
200
200
|
[[1, 2, 3], [4, 5, 6]]
|
201
201
|
```
|
202
202
|
|
203
|
-
Or a Numo
|
203
|
+
Or a Numo array
|
204
204
|
|
205
205
|
```ruby
|
206
|
-
Numo::
|
206
|
+
Numo::NArray.cast([[1, 2, 3], [4, 5, 6]])
|
207
207
|
```
|
208
208
|
|
209
209
|
## History
|
data/ext/faiss/ext.cpp
CHANGED
@@ -1,315 +1,19 @@
|
|
1
|
-
#include <faiss/IndexFlat.h>
|
2
|
-
#include <faiss/IndexHNSW.h>
|
3
|
-
#include <faiss/IndexIVFFlat.h>
|
4
|
-
#include <faiss/IndexLSH.h>
|
5
|
-
#include <faiss/IndexScalarQuantizer.h>
|
6
|
-
#include <faiss/IndexPQ.h>
|
7
|
-
#include <faiss/IndexIVFPQ.h>
|
8
|
-
#include <faiss/IndexIVFPQR.h>
|
9
|
-
|
10
|
-
#include <faiss/IndexBinaryFlat.h>
|
11
|
-
#include <faiss/IndexBinaryIVF.h>
|
12
|
-
#include <faiss/index_factory.h>
|
13
|
-
#include <faiss/index_io.h>
|
14
|
-
|
15
|
-
#include <faiss/Clustering.h>
|
16
|
-
#include <faiss/VectorTransform.h>
|
17
|
-
|
18
|
-
#include <rice/Array.hpp>
|
19
|
-
#include <rice/Class.hpp>
|
20
|
-
#include <rice/Constructor.hpp>
|
21
1
|
#include <rice/Module.hpp>
|
22
2
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
uint8_t* uint8_array(Rice::Object o)
|
30
|
-
{
|
31
|
-
Rice::String s = o.call("to_binary");
|
32
|
-
return (uint8_t*) s.c_str();
|
33
|
-
}
|
34
|
-
|
35
|
-
// TODO return Numo::SFloat
|
36
|
-
Rice::String result(float* ptr, int64_t length)
|
37
|
-
{
|
38
|
-
return Rice::String(std::string((char*) ptr, length * sizeof(float)));
|
39
|
-
}
|
40
|
-
|
41
|
-
// TODO return Numo::UInt8
|
42
|
-
Rice::String result(uint8_t* ptr, int64_t length)
|
43
|
-
{
|
44
|
-
return Rice::String(std::string((char*) ptr, length * sizeof(uint8_t)));
|
45
|
-
}
|
46
|
-
|
47
|
-
// TODO return Numo::Int32
|
48
|
-
Rice::String result(int32_t* ptr, int64_t length)
|
49
|
-
{
|
50
|
-
return Rice::String(std::string((char*) ptr, length * sizeof(int32_t)));
|
51
|
-
}
|
52
|
-
|
53
|
-
// TODO return Numo::Int64
|
54
|
-
Rice::String result(int64_t* ptr, int64_t length)
|
55
|
-
{
|
56
|
-
return Rice::String(std::string((char*) ptr, length * sizeof(int64_t)));
|
57
|
-
}
|
3
|
+
void init_index(Rice::Module& m);
|
4
|
+
void init_index_binary(Rice::Module& m);
|
5
|
+
void init_kmeans(Rice::Module& m);
|
6
|
+
void init_pca_matrix(Rice::Module& m);
|
7
|
+
void init_product_quantizer(Rice::Module& m);
|
58
8
|
|
59
9
|
extern "C"
|
60
10
|
void Init_ext()
|
61
11
|
{
|
62
|
-
|
63
|
-
.define_singleton_method(
|
64
|
-
"index_binary_factory",
|
65
|
-
*[](int d, const char *description) {
|
66
|
-
return faiss::index_binary_factory(d, description);
|
67
|
-
});
|
68
|
-
|
69
|
-
Rice::define_class_under<faiss::Index>(rb_mFaiss, "Index")
|
70
|
-
.define_method(
|
71
|
-
"d",
|
72
|
-
*[](faiss::Index &self) {
|
73
|
-
return self.d;
|
74
|
-
})
|
75
|
-
.define_method(
|
76
|
-
"trained?",
|
77
|
-
*[](faiss::Index &self) {
|
78
|
-
return self.is_trained;
|
79
|
-
})
|
80
|
-
.define_method(
|
81
|
-
"ntotal",
|
82
|
-
*[](faiss::Index &self) {
|
83
|
-
return self.ntotal;
|
84
|
-
})
|
85
|
-
.define_method(
|
86
|
-
"_train",
|
87
|
-
*[](faiss::Index &self, int64_t n, Rice::Object o) {
|
88
|
-
const float *x = float_array(o);
|
89
|
-
self.train(n, x);
|
90
|
-
})
|
91
|
-
.define_method(
|
92
|
-
"_add",
|
93
|
-
*[](faiss::Index &self, int64_t n, Rice::Object o) {
|
94
|
-
const float *x = float_array(o);
|
95
|
-
self.add(n, x);
|
96
|
-
})
|
97
|
-
.define_method(
|
98
|
-
"_search",
|
99
|
-
*[](faiss::Index &self, int64_t n, Rice::Object o, int64_t k) {
|
100
|
-
const float *x = float_array(o);
|
101
|
-
float *distances = new float[k * n];
|
102
|
-
int64_t *labels = new int64_t[k * n];
|
103
|
-
|
104
|
-
self.search(n, x, k, distances, labels);
|
105
|
-
|
106
|
-
auto dstr = result(distances, k * n);
|
107
|
-
auto lstr = result(labels, k * n);
|
108
|
-
|
109
|
-
Rice::Array ret;
|
110
|
-
ret.push(dstr);
|
111
|
-
ret.push(lstr);
|
112
|
-
return ret;
|
113
|
-
})
|
114
|
-
.define_method(
|
115
|
-
"save",
|
116
|
-
*[](faiss::Index &self, const char *fname) {
|
117
|
-
faiss::write_index(&self, fname);
|
118
|
-
})
|
119
|
-
.define_singleton_method(
|
120
|
-
"load",
|
121
|
-
*[](const char *fname) {
|
122
|
-
return faiss::read_index(fname);
|
123
|
-
});
|
124
|
-
|
125
|
-
Rice::define_class_under<faiss::IndexBinary>(rb_mFaiss, "IndexBinary")
|
126
|
-
.define_method(
|
127
|
-
"d",
|
128
|
-
*[](faiss::Index &self) {
|
129
|
-
return self.d;
|
130
|
-
})
|
131
|
-
.define_method(
|
132
|
-
"trained?",
|
133
|
-
*[](faiss::IndexBinary &self) {
|
134
|
-
return self.is_trained;
|
135
|
-
})
|
136
|
-
.define_method(
|
137
|
-
"ntotal",
|
138
|
-
*[](faiss::IndexBinary &self) {
|
139
|
-
return self.ntotal;
|
140
|
-
})
|
141
|
-
.define_method(
|
142
|
-
"_train",
|
143
|
-
*[](faiss::IndexBinary &self, int64_t n, Rice::Object o) {
|
144
|
-
const uint8_t *x = uint8_array(o);
|
145
|
-
self.train(n, x);
|
146
|
-
})
|
147
|
-
.define_method(
|
148
|
-
"_add",
|
149
|
-
*[](faiss::IndexBinary &self, int64_t n, Rice::Object o) {
|
150
|
-
const uint8_t *x = uint8_array(o);
|
151
|
-
self.add(n, x);
|
152
|
-
})
|
153
|
-
.define_method(
|
154
|
-
"_search",
|
155
|
-
*[](faiss::IndexBinary &self, int64_t n, Rice::Object o, int64_t k) {
|
156
|
-
const uint8_t *x = uint8_array(o);
|
157
|
-
int32_t *distances = new int32_t[k * n];
|
158
|
-
int64_t *labels = new int64_t[k * n];
|
159
|
-
|
160
|
-
self.search(n, x, k, distances, labels);
|
161
|
-
|
162
|
-
auto dstr = result(distances, k * n);
|
163
|
-
auto lstr = result(labels, k * n);
|
164
|
-
|
165
|
-
Rice::Array ret;
|
166
|
-
ret.push(dstr);
|
167
|
-
ret.push(lstr);
|
168
|
-
return ret;
|
169
|
-
})
|
170
|
-
.define_method(
|
171
|
-
"save",
|
172
|
-
*[](faiss::IndexBinary &self, const char *fname) {
|
173
|
-
faiss::write_index_binary(&self, fname);
|
174
|
-
})
|
175
|
-
.define_singleton_method(
|
176
|
-
"load",
|
177
|
-
*[](const char *fname) {
|
178
|
-
return faiss::read_index_binary(fname);
|
179
|
-
});
|
180
|
-
|
181
|
-
Rice::define_class_under<faiss::IndexFlatL2, faiss::Index>(rb_mFaiss, "IndexFlatL2")
|
182
|
-
.define_constructor(Rice::Constructor<faiss::IndexFlatL2, int64_t>());
|
183
|
-
|
184
|
-
Rice::define_class_under<faiss::IndexFlatIP, faiss::Index>(rb_mFaiss, "IndexFlatIP")
|
185
|
-
.define_constructor(Rice::Constructor<faiss::IndexFlatIP, int64_t>());
|
186
|
-
|
187
|
-
Rice::define_class_under<faiss::IndexHNSWFlat, faiss::Index>(rb_mFaiss, "IndexHNSWFlat")
|
188
|
-
.define_constructor(Rice::Constructor<faiss::IndexHNSWFlat, int, int>());
|
189
|
-
|
190
|
-
Rice::define_class_under<faiss::IndexIVFFlat, faiss::Index>(rb_mFaiss, "IndexIVFFlat")
|
191
|
-
.define_constructor(Rice::Constructor<faiss::IndexIVFFlat, faiss::Index*, size_t, size_t>());
|
192
|
-
|
193
|
-
Rice::define_class_under<faiss::IndexLSH, faiss::Index>(rb_mFaiss, "IndexLSH")
|
194
|
-
.define_constructor(Rice::Constructor<faiss::IndexLSH, int64_t, int>());
|
195
|
-
|
196
|
-
Rice::define_class_under<faiss::IndexScalarQuantizer, faiss::Index>(rb_mFaiss, "IndexScalarQuantizer")
|
197
|
-
.define_constructor(Rice::Constructor<faiss::IndexScalarQuantizer, int, faiss::ScalarQuantizer::QuantizerType>());
|
198
|
-
|
199
|
-
Rice::define_class_under<faiss::IndexPQ, faiss::Index>(rb_mFaiss, "IndexPQ")
|
200
|
-
.define_constructor(Rice::Constructor<faiss::IndexPQ, int, size_t, size_t>());
|
201
|
-
|
202
|
-
Rice::define_class_under<faiss::IndexIVFScalarQuantizer, faiss::Index>(rb_mFaiss, "IndexIVFScalarQuantizer")
|
203
|
-
.define_constructor(Rice::Constructor<faiss::IndexIVFScalarQuantizer, faiss::Index*, size_t, size_t, faiss::ScalarQuantizer::QuantizerType>());
|
204
|
-
|
205
|
-
Rice::define_class_under<faiss::IndexIVFPQ, faiss::Index>(rb_mFaiss, "IndexIVFPQ")
|
206
|
-
.define_constructor(Rice::Constructor<faiss::IndexIVFPQ, faiss::Index*, size_t, size_t, size_t, size_t>());
|
207
|
-
|
208
|
-
Rice::define_class_under<faiss::IndexIVFPQR, faiss::Index>(rb_mFaiss, "IndexIVFPQR")
|
209
|
-
.define_constructor(Rice::Constructor<faiss::IndexIVFPQR, faiss::Index*, size_t, size_t, size_t, size_t, size_t, size_t>());
|
210
|
-
|
211
|
-
Rice::define_class_under<faiss::IndexBinaryFlat, faiss::IndexBinary>(rb_mFaiss, "IndexBinaryFlat")
|
212
|
-
.define_constructor(Rice::Constructor<faiss::IndexBinaryFlat, int64_t>());
|
213
|
-
|
214
|
-
Rice::define_class_under<faiss::IndexBinaryIVF, faiss::IndexBinary>(rb_mFaiss, "IndexBinaryIVF")
|
215
|
-
.define_constructor(Rice::Constructor<faiss::IndexBinaryIVF, faiss::IndexBinary*, size_t, size_t>());
|
216
|
-
|
217
|
-
Rice::define_class_under<faiss::Clustering>(rb_mFaiss, "Kmeans")
|
218
|
-
.define_constructor(Rice::Constructor<faiss::Clustering, int, int>())
|
219
|
-
.define_method(
|
220
|
-
"d",
|
221
|
-
*[](faiss::Clustering &self) {
|
222
|
-
return self.d;
|
223
|
-
})
|
224
|
-
.define_method(
|
225
|
-
"k",
|
226
|
-
*[](faiss::Clustering &self) {
|
227
|
-
return self.k;
|
228
|
-
})
|
229
|
-
.define_method(
|
230
|
-
"_centroids",
|
231
|
-
*[](faiss::Clustering &self) {
|
232
|
-
float *centroids = new float[self.k * self.d];
|
233
|
-
for (size_t i = 0; i < self.centroids.size(); i++) {
|
234
|
-
centroids[i] = self.centroids[i];
|
235
|
-
}
|
236
|
-
return result(centroids, self.k * self.d);
|
237
|
-
})
|
238
|
-
.define_method(
|
239
|
-
"_train",
|
240
|
-
*[](faiss::Clustering &self, int64_t n, Rice::Object o, faiss::Index & index) {
|
241
|
-
const float *x = float_array(o);
|
242
|
-
self.train(n, x, index);
|
243
|
-
});
|
244
|
-
|
245
|
-
Rice::define_class_under<faiss::PCAMatrix>(rb_mFaiss, "PCAMatrix")
|
246
|
-
.define_constructor(Rice::Constructor<faiss::PCAMatrix, int, int>())
|
247
|
-
.define_method(
|
248
|
-
"d_in",
|
249
|
-
*[](faiss::PCAMatrix &self) {
|
250
|
-
return self.d_in;
|
251
|
-
})
|
252
|
-
.define_method(
|
253
|
-
"d_out",
|
254
|
-
*[](faiss::PCAMatrix &self) {
|
255
|
-
return self.d_out;
|
256
|
-
})
|
257
|
-
.define_method(
|
258
|
-
"_train",
|
259
|
-
*[](faiss::PCAMatrix &self, int64_t n, Rice::Object o) {
|
260
|
-
const float *x = float_array(o);
|
261
|
-
self.train(n, x);
|
262
|
-
})
|
263
|
-
.define_method(
|
264
|
-
"_apply",
|
265
|
-
*[](faiss::PCAMatrix &self, int64_t n, Rice::Object o) {
|
266
|
-
const float *x = float_array(o);
|
267
|
-
float* res = self.apply(n, x);
|
268
|
-
return result(res, n * self.d_out);
|
269
|
-
});
|
12
|
+
auto m = Rice::define_module("Faiss");
|
270
13
|
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
return self.d;
|
277
|
-
})
|
278
|
-
.define_method(
|
279
|
-
"m",
|
280
|
-
*[](faiss::ProductQuantizer &self) {
|
281
|
-
return self.M;
|
282
|
-
})
|
283
|
-
.define_method(
|
284
|
-
"_train",
|
285
|
-
*[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
|
286
|
-
const float *x = float_array(o);
|
287
|
-
self.train(n, x);
|
288
|
-
})
|
289
|
-
.define_method(
|
290
|
-
"_compute_codes",
|
291
|
-
*[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
|
292
|
-
const float *x = float_array(o);
|
293
|
-
uint8_t *codes = new uint8_t[n * self.M];
|
294
|
-
self.compute_codes(x, codes, n);
|
295
|
-
return result(codes, n * self.M);
|
296
|
-
})
|
297
|
-
.define_method(
|
298
|
-
"_decode",
|
299
|
-
*[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
|
300
|
-
const uint8_t *codes = uint8_array(o);
|
301
|
-
float *x = new float[n * self.d];
|
302
|
-
self.decode(codes, x, n);
|
303
|
-
return result(x, n * self.d);
|
304
|
-
})
|
305
|
-
.define_method(
|
306
|
-
"save",
|
307
|
-
*[](faiss::ProductQuantizer &self, const char *fname) {
|
308
|
-
faiss::write_ProductQuantizer(&self, fname);
|
309
|
-
})
|
310
|
-
.define_singleton_method(
|
311
|
-
"load",
|
312
|
-
*[](const char *fname) {
|
313
|
-
return faiss::read_ProductQuantizer(fname);
|
314
|
-
});
|
14
|
+
init_index(m);
|
15
|
+
init_index_binary(m);
|
16
|
+
init_kmeans(m);
|
17
|
+
init_pca_matrix(m);
|
18
|
+
init_product_quantizer(m);
|
315
19
|
}
|
data/ext/faiss/extconf.rb
CHANGED
@@ -4,7 +4,7 @@ abort "BLAS not found" unless have_library("blas")
|
|
4
4
|
abort "LAPACK not found" unless have_library("lapack")
|
5
5
|
abort "OpenMP not found" unless have_library("omp") || have_library("gomp")
|
6
6
|
|
7
|
-
$CXXFLAGS << " -std=c++11 -
|
7
|
+
$CXXFLAGS << " -std=c++11 -DFINTEGER=int " << with_config("optflags", "-march=native")
|
8
8
|
|
9
9
|
ext = File.expand_path(".", __dir__)
|
10
10
|
vendor = File.expand_path("../../vendor/faiss", __dir__)
|
data/ext/faiss/index.cpp
ADDED
@@ -0,0 +1,118 @@
|
|
1
|
+
#include <faiss/Index.h>
|
2
|
+
#include <faiss/IndexFlat.h>
|
3
|
+
#include <faiss/IndexHNSW.h>
|
4
|
+
#include <faiss/IndexIVFFlat.h>
|
5
|
+
#include <faiss/IndexLSH.h>
|
6
|
+
#include <faiss/IndexScalarQuantizer.h>
|
7
|
+
#include <faiss/IndexPQ.h>
|
8
|
+
#include <faiss/IndexIVFPQ.h>
|
9
|
+
#include <faiss/IndexIVFPQR.h>
|
10
|
+
#include <faiss/index_io.h>
|
11
|
+
|
12
|
+
#include <rice/Array.hpp>
|
13
|
+
#include <rice/Constructor.hpp>
|
14
|
+
#include <rice/Module.hpp>
|
15
|
+
|
16
|
+
#include "utils.h"
|
17
|
+
|
18
|
+
template<>
|
19
|
+
faiss::MetricType from_ruby<faiss::MetricType>(Rice::Object x)
|
20
|
+
{
|
21
|
+
auto s = x.to_s().str();
|
22
|
+
if (s == "inner_product") {
|
23
|
+
return faiss::METRIC_INNER_PRODUCT;
|
24
|
+
} else if (s == "l2") {
|
25
|
+
return faiss::METRIC_L2;
|
26
|
+
} else {
|
27
|
+
// TODO throw argument error
|
28
|
+
throw std::runtime_error("Invalid metric: " + s);
|
29
|
+
}
|
30
|
+
}
|
31
|
+
|
32
|
+
void init_index(Rice::Module& m) {
|
33
|
+
Rice::define_class_under<faiss::Index>(m, "Index")
|
34
|
+
.define_method(
|
35
|
+
"d",
|
36
|
+
*[](faiss::Index &self) {
|
37
|
+
return self.d;
|
38
|
+
})
|
39
|
+
.define_method(
|
40
|
+
"trained?",
|
41
|
+
*[](faiss::Index &self) {
|
42
|
+
return self.is_trained;
|
43
|
+
})
|
44
|
+
.define_method(
|
45
|
+
"ntotal",
|
46
|
+
*[](faiss::Index &self) {
|
47
|
+
return self.ntotal;
|
48
|
+
})
|
49
|
+
.define_method(
|
50
|
+
"_train",
|
51
|
+
*[](faiss::Index &self, int64_t n, Rice::Object o) {
|
52
|
+
const float *x = float_array(o);
|
53
|
+
self.train(n, x);
|
54
|
+
})
|
55
|
+
.define_method(
|
56
|
+
"_add",
|
57
|
+
*[](faiss::Index &self, int64_t n, Rice::Object o) {
|
58
|
+
const float *x = float_array(o);
|
59
|
+
self.add(n, x);
|
60
|
+
})
|
61
|
+
.define_method(
|
62
|
+
"_search",
|
63
|
+
*[](faiss::Index &self, int64_t n, Rice::Object o, int64_t k) {
|
64
|
+
const float *x = float_array(o);
|
65
|
+
float *distances = new float[k * n];
|
66
|
+
int64_t *labels = new int64_t[k * n];
|
67
|
+
|
68
|
+
self.search(n, x, k, distances, labels);
|
69
|
+
|
70
|
+
auto dstr = result(distances, k * n);
|
71
|
+
auto lstr = result(labels, k * n);
|
72
|
+
|
73
|
+
Rice::Array ret;
|
74
|
+
ret.push(dstr);
|
75
|
+
ret.push(lstr);
|
76
|
+
return ret;
|
77
|
+
})
|
78
|
+
.define_method(
|
79
|
+
"save",
|
80
|
+
*[](faiss::Index &self, const char *fname) {
|
81
|
+
faiss::write_index(&self, fname);
|
82
|
+
})
|
83
|
+
.define_singleton_method(
|
84
|
+
"load",
|
85
|
+
*[](const char *fname) {
|
86
|
+
return faiss::read_index(fname);
|
87
|
+
});
|
88
|
+
|
89
|
+
Rice::define_class_under<faiss::IndexFlatL2, faiss::Index>(m, "IndexFlatL2")
|
90
|
+
.define_constructor(Rice::Constructor<faiss::IndexFlatL2, int64_t>());
|
91
|
+
|
92
|
+
Rice::define_class_under<faiss::IndexFlatIP, faiss::Index>(m, "IndexFlatIP")
|
93
|
+
.define_constructor(Rice::Constructor<faiss::IndexFlatIP, int64_t>());
|
94
|
+
|
95
|
+
Rice::define_class_under<faiss::IndexHNSWFlat, faiss::Index>(m, "IndexHNSWFlat")
|
96
|
+
.define_constructor(Rice::Constructor<faiss::IndexHNSWFlat, int, int, faiss::MetricType>(), (Rice::Arg("d"), Rice::Arg("M"), Rice::Arg("metric") = faiss::METRIC_L2));
|
97
|
+
|
98
|
+
Rice::define_class_under<faiss::IndexIVFFlat, faiss::Index>(m, "IndexIVFFlat")
|
99
|
+
.define_constructor(Rice::Constructor<faiss::IndexIVFFlat, faiss::Index*, size_t, size_t>());
|
100
|
+
|
101
|
+
Rice::define_class_under<faiss::IndexLSH, faiss::Index>(m, "IndexLSH")
|
102
|
+
.define_constructor(Rice::Constructor<faiss::IndexLSH, int64_t, int>());
|
103
|
+
|
104
|
+
Rice::define_class_under<faiss::IndexScalarQuantizer, faiss::Index>(m, "IndexScalarQuantizer")
|
105
|
+
.define_constructor(Rice::Constructor<faiss::IndexScalarQuantizer, int, faiss::ScalarQuantizer::QuantizerType>());
|
106
|
+
|
107
|
+
Rice::define_class_under<faiss::IndexPQ, faiss::Index>(m, "IndexPQ")
|
108
|
+
.define_constructor(Rice::Constructor<faiss::IndexPQ, int, size_t, size_t>());
|
109
|
+
|
110
|
+
Rice::define_class_under<faiss::IndexIVFScalarQuantizer, faiss::Index>(m, "IndexIVFScalarQuantizer")
|
111
|
+
.define_constructor(Rice::Constructor<faiss::IndexIVFScalarQuantizer, faiss::Index*, size_t, size_t, faiss::ScalarQuantizer::QuantizerType>());
|
112
|
+
|
113
|
+
Rice::define_class_under<faiss::IndexIVFPQ, faiss::Index>(m, "IndexIVFPQ")
|
114
|
+
.define_constructor(Rice::Constructor<faiss::IndexIVFPQ, faiss::Index*, size_t, size_t, size_t, size_t>());
|
115
|
+
|
116
|
+
Rice::define_class_under<faiss::IndexIVFPQR, faiss::Index>(m, "IndexIVFPQR")
|
117
|
+
.define_constructor(Rice::Constructor<faiss::IndexIVFPQR, faiss::Index*, size_t, size_t, size_t, size_t, size_t, size_t>());
|
118
|
+
}
|
@@ -0,0 +1,81 @@
|
|
1
|
+
#include <faiss/IndexBinary.h>
|
2
|
+
#include <faiss/IndexBinaryFlat.h>
|
3
|
+
#include <faiss/IndexBinaryIVF.h>
|
4
|
+
#include <faiss/index_factory.h>
|
5
|
+
#include <faiss/index_io.h>
|
6
|
+
|
7
|
+
#include <rice/Array.hpp>
|
8
|
+
#include <rice/Constructor.hpp>
|
9
|
+
#include <rice/Module.hpp>
|
10
|
+
|
11
|
+
#include "utils.h"
|
12
|
+
|
13
|
+
void init_index_binary(Rice::Module& m) {
|
14
|
+
Rice::define_class_under<faiss::IndexBinary>(m, "IndexBinary")
|
15
|
+
.define_method(
|
16
|
+
"d",
|
17
|
+
*[](faiss::IndexBinary &self) {
|
18
|
+
return self.d;
|
19
|
+
})
|
20
|
+
.define_method(
|
21
|
+
"trained?",
|
22
|
+
*[](faiss::IndexBinary &self) {
|
23
|
+
return self.is_trained;
|
24
|
+
})
|
25
|
+
.define_method(
|
26
|
+
"ntotal",
|
27
|
+
*[](faiss::IndexBinary &self) {
|
28
|
+
return self.ntotal;
|
29
|
+
})
|
30
|
+
.define_method(
|
31
|
+
"_train",
|
32
|
+
*[](faiss::IndexBinary &self, int64_t n, Rice::Object o) {
|
33
|
+
const uint8_t *x = uint8_array(o);
|
34
|
+
self.train(n, x);
|
35
|
+
})
|
36
|
+
.define_method(
|
37
|
+
"_add",
|
38
|
+
*[](faiss::IndexBinary &self, int64_t n, Rice::Object o) {
|
39
|
+
const uint8_t *x = uint8_array(o);
|
40
|
+
self.add(n, x);
|
41
|
+
})
|
42
|
+
.define_method(
|
43
|
+
"_search",
|
44
|
+
*[](faiss::IndexBinary &self, int64_t n, Rice::Object o, int64_t k) {
|
45
|
+
const uint8_t *x = uint8_array(o);
|
46
|
+
int32_t *distances = new int32_t[k * n];
|
47
|
+
int64_t *labels = new int64_t[k * n];
|
48
|
+
|
49
|
+
self.search(n, x, k, distances, labels);
|
50
|
+
|
51
|
+
auto dstr = result(distances, k * n);
|
52
|
+
auto lstr = result(labels, k * n);
|
53
|
+
|
54
|
+
Rice::Array ret;
|
55
|
+
ret.push(dstr);
|
56
|
+
ret.push(lstr);
|
57
|
+
return ret;
|
58
|
+
})
|
59
|
+
.define_method(
|
60
|
+
"save",
|
61
|
+
*[](faiss::IndexBinary &self, const char *fname) {
|
62
|
+
faiss::write_index_binary(&self, fname);
|
63
|
+
})
|
64
|
+
.define_singleton_method(
|
65
|
+
"load",
|
66
|
+
*[](const char *fname) {
|
67
|
+
return faiss::read_index_binary(fname);
|
68
|
+
});
|
69
|
+
|
70
|
+
Rice::define_class_under<faiss::IndexBinaryFlat, faiss::IndexBinary>(m, "IndexBinaryFlat")
|
71
|
+
.define_constructor(Rice::Constructor<faiss::IndexBinaryFlat, int64_t>());
|
72
|
+
|
73
|
+
Rice::define_class_under<faiss::IndexBinaryIVF, faiss::IndexBinary>(m, "IndexBinaryIVF")
|
74
|
+
.define_constructor(Rice::Constructor<faiss::IndexBinaryIVF, faiss::IndexBinary*, size_t, size_t>());
|
75
|
+
|
76
|
+
m.define_singleton_method(
|
77
|
+
"index_binary_factory",
|
78
|
+
*[](int d, const char *description) {
|
79
|
+
return faiss::index_binary_factory(d, description);
|
80
|
+
});
|
81
|
+
}
|
@@ -0,0 +1,36 @@
|
|
1
|
+
#include <faiss/Clustering.h>
|
2
|
+
|
3
|
+
#include <rice/Constructor.hpp>
|
4
|
+
#include <rice/Module.hpp>
|
5
|
+
|
6
|
+
#include "utils.h"
|
7
|
+
|
8
|
+
void init_kmeans(Rice::Module& m) {
|
9
|
+
Rice::define_class_under<faiss::Clustering>(m, "Kmeans")
|
10
|
+
.define_constructor(Rice::Constructor<faiss::Clustering, int, int>())
|
11
|
+
.define_method(
|
12
|
+
"d",
|
13
|
+
*[](faiss::Clustering &self) {
|
14
|
+
return self.d;
|
15
|
+
})
|
16
|
+
.define_method(
|
17
|
+
"k",
|
18
|
+
*[](faiss::Clustering &self) {
|
19
|
+
return self.k;
|
20
|
+
})
|
21
|
+
.define_method(
|
22
|
+
"_centroids",
|
23
|
+
*[](faiss::Clustering &self) {
|
24
|
+
float *centroids = new float[self.k * self.d];
|
25
|
+
for (size_t i = 0; i < self.centroids.size(); i++) {
|
26
|
+
centroids[i] = self.centroids[i];
|
27
|
+
}
|
28
|
+
return result(centroids, self.k * self.d);
|
29
|
+
})
|
30
|
+
.define_method(
|
31
|
+
"_train",
|
32
|
+
*[](faiss::Clustering &self, int64_t n, Rice::Object o, faiss::Index & index) {
|
33
|
+
const float *x = float_array(o);
|
34
|
+
self.train(n, x, index);
|
35
|
+
});
|
36
|
+
}
|
@@ -0,0 +1,34 @@
|
|
1
|
+
#include <faiss/VectorTransform.h>
|
2
|
+
|
3
|
+
#include <rice/Constructor.hpp>
|
4
|
+
#include <rice/Module.hpp>
|
5
|
+
|
6
|
+
#include "utils.h"
|
7
|
+
|
8
|
+
void init_pca_matrix(Rice::Module& m) {
|
9
|
+
Rice::define_class_under<faiss::PCAMatrix>(m, "PCAMatrix")
|
10
|
+
.define_constructor(Rice::Constructor<faiss::PCAMatrix, int, int>())
|
11
|
+
.define_method(
|
12
|
+
"d_in",
|
13
|
+
*[](faiss::PCAMatrix &self) {
|
14
|
+
return self.d_in;
|
15
|
+
})
|
16
|
+
.define_method(
|
17
|
+
"d_out",
|
18
|
+
*[](faiss::PCAMatrix &self) {
|
19
|
+
return self.d_out;
|
20
|
+
})
|
21
|
+
.define_method(
|
22
|
+
"_train",
|
23
|
+
*[](faiss::PCAMatrix &self, int64_t n, Rice::Object o) {
|
24
|
+
const float *x = float_array(o);
|
25
|
+
self.train(n, x);
|
26
|
+
})
|
27
|
+
.define_method(
|
28
|
+
"_apply",
|
29
|
+
*[](faiss::PCAMatrix &self, int64_t n, Rice::Object o) {
|
30
|
+
const float *x = float_array(o);
|
31
|
+
float* res = self.apply(n, x);
|
32
|
+
return result(res, n * self.d_out);
|
33
|
+
});
|
34
|
+
}
|
@@ -0,0 +1,54 @@
|
|
1
|
+
#include <faiss/impl/ProductQuantizer.h>
|
2
|
+
#include <faiss/index_io.h>
|
3
|
+
|
4
|
+
#include <rice/Constructor.hpp>
|
5
|
+
#include <rice/Module.hpp>
|
6
|
+
|
7
|
+
#include "utils.h"
|
8
|
+
|
9
|
+
void init_product_quantizer(Rice::Module& m) {
|
10
|
+
Rice::define_class_under<faiss::ProductQuantizer>(m, "ProductQuantizer")
|
11
|
+
.define_constructor(Rice::Constructor<faiss::ProductQuantizer, size_t, size_t, size_t>())
|
12
|
+
.define_method(
|
13
|
+
"d",
|
14
|
+
*[](faiss::ProductQuantizer &self) {
|
15
|
+
return self.d;
|
16
|
+
})
|
17
|
+
.define_method(
|
18
|
+
"m",
|
19
|
+
*[](faiss::ProductQuantizer &self) {
|
20
|
+
return self.M;
|
21
|
+
})
|
22
|
+
.define_method(
|
23
|
+
"_train",
|
24
|
+
*[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
|
25
|
+
const float *x = float_array(o);
|
26
|
+
self.train(n, x);
|
27
|
+
})
|
28
|
+
.define_method(
|
29
|
+
"_compute_codes",
|
30
|
+
*[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
|
31
|
+
const float *x = float_array(o);
|
32
|
+
uint8_t *codes = new uint8_t[n * self.M];
|
33
|
+
self.compute_codes(x, codes, n);
|
34
|
+
return result(codes, n * self.M);
|
35
|
+
})
|
36
|
+
.define_method(
|
37
|
+
"_decode",
|
38
|
+
*[](faiss::ProductQuantizer &self, int n, Rice::Object o) {
|
39
|
+
const uint8_t *codes = uint8_array(o);
|
40
|
+
float *x = new float[n * self.d];
|
41
|
+
self.decode(codes, x, n);
|
42
|
+
return result(x, n * self.d);
|
43
|
+
})
|
44
|
+
.define_method(
|
45
|
+
"save",
|
46
|
+
*[](faiss::ProductQuantizer &self, const char *fname) {
|
47
|
+
faiss::write_ProductQuantizer(&self, fname);
|
48
|
+
})
|
49
|
+
.define_singleton_method(
|
50
|
+
"load",
|
51
|
+
*[](const char *fname) {
|
52
|
+
return faiss::read_ProductQuantizer(fname);
|
53
|
+
});
|
54
|
+
}
|
data/ext/faiss/utils.cpp
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
#include "utils.h"
|
2
|
+
|
3
|
+
#include <rice/Object.hpp>
|
4
|
+
#include <rice/String.hpp>
|
5
|
+
|
6
|
+
float* float_array(Rice::Object o)
|
7
|
+
{
|
8
|
+
Rice::String s = o.call("to_binary");
|
9
|
+
return (float*) s.c_str();
|
10
|
+
}
|
11
|
+
|
12
|
+
uint8_t* uint8_array(Rice::Object o)
|
13
|
+
{
|
14
|
+
Rice::String s = o.call("to_binary");
|
15
|
+
return (uint8_t*) s.c_str();
|
16
|
+
}
|
17
|
+
|
18
|
+
// TODO return Numo::SFloat
|
19
|
+
Rice::String result(float* ptr, int64_t length)
|
20
|
+
{
|
21
|
+
return Rice::String(std::string((char*) ptr, length * sizeof(float)));
|
22
|
+
}
|
23
|
+
|
24
|
+
// TODO return Numo::UInt8
|
25
|
+
Rice::String result(uint8_t* ptr, int64_t length)
|
26
|
+
{
|
27
|
+
return Rice::String(std::string((char*) ptr, length * sizeof(uint8_t)));
|
28
|
+
}
|
29
|
+
|
30
|
+
// TODO return Numo::Int32
|
31
|
+
Rice::String result(int32_t* ptr, int64_t length)
|
32
|
+
{
|
33
|
+
return Rice::String(std::string((char*) ptr, length * sizeof(int32_t)));
|
34
|
+
}
|
35
|
+
|
36
|
+
// TODO return Numo::Int64
|
37
|
+
Rice::String result(int64_t* ptr, int64_t length)
|
38
|
+
{
|
39
|
+
return Rice::String(std::string((char*) ptr, length * sizeof(int64_t)));
|
40
|
+
}
|
data/ext/faiss/utils.h
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
#pragma once
|
2
|
+
|
3
|
+
#include <rice/Object.hpp>
|
4
|
+
#include <rice/String.hpp>
|
5
|
+
|
6
|
+
float* float_array(Rice::Object o);
|
7
|
+
uint8_t* uint8_array(Rice::Object o);
|
8
|
+
|
9
|
+
// TODO return Numo::SFloat
|
10
|
+
Rice::String result(float* ptr, int64_t length);
|
11
|
+
// TODO return Numo::UInt8
|
12
|
+
Rice::String result(uint8_t* ptr, int64_t length);
|
13
|
+
// TODO return Numo::Int32
|
14
|
+
Rice::String result(int32_t* ptr, int64_t length);
|
15
|
+
// TODO return Numo::Int64
|
16
|
+
Rice::String result(int64_t* ptr, int64_t length);
|
data/lib/faiss/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: faiss
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Kane
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-03-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rice
|
@@ -50,6 +50,13 @@ files:
|
|
50
50
|
- README.md
|
51
51
|
- ext/faiss/ext.cpp
|
52
52
|
- ext/faiss/extconf.rb
|
53
|
+
- ext/faiss/index.cpp
|
54
|
+
- ext/faiss/index_binary.cpp
|
55
|
+
- ext/faiss/kmeans.cpp
|
56
|
+
- ext/faiss/pca_matrix.cpp
|
57
|
+
- ext/faiss/product_quantizer.cpp
|
58
|
+
- ext/faiss/utils.cpp
|
59
|
+
- ext/faiss/utils.h
|
53
60
|
- lib/faiss.rb
|
54
61
|
- lib/faiss/index.rb
|
55
62
|
- lib/faiss/index_binary.rb
|