ffi-fasttext 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +44 -0
- data/.travis.yml +5 -0
- data/Gemfile +6 -0
- data/LICENSE.txt +21 -0
- data/README.md +59 -0
- data/Rakefile +19 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/ext/ffi/fasttext/Rakefile +71 -0
- data/ffi-fasttext.gemspec +40 -0
- data/lib/ffi/fasttext.rb +108 -0
- data/lib/ffi/fasttext/version.rb +5 -0
- data/vendor/fasttext/LICENSE +30 -0
- data/vendor/fasttext/PATENTS +33 -0
- data/vendor/fasttext/args.cc +250 -0
- data/vendor/fasttext/args.h +71 -0
- data/vendor/fasttext/dictionary.cc +475 -0
- data/vendor/fasttext/dictionary.h +112 -0
- data/vendor/fasttext/fasttext.cc +693 -0
- data/vendor/fasttext/fasttext.h +97 -0
- data/vendor/fasttext/ffi_fasttext.cc +66 -0
- data/vendor/fasttext/main.cc +270 -0
- data/vendor/fasttext/matrix.cc +144 -0
- data/vendor/fasttext/matrix.h +57 -0
- data/vendor/fasttext/model.cc +341 -0
- data/vendor/fasttext/model.h +110 -0
- data/vendor/fasttext/productquantizer.cc +211 -0
- data/vendor/fasttext/productquantizer.h +67 -0
- data/vendor/fasttext/qmatrix.cc +121 -0
- data/vendor/fasttext/qmatrix.h +65 -0
- data/vendor/fasttext/real.h +19 -0
- data/vendor/fasttext/utils.cc +29 -0
- data/vendor/fasttext/utils.h +25 -0
- data/vendor/fasttext/vector.cc +137 -0
- data/vendor/fasttext/vector.h +53 -0
- metadata +151 -0
@@ -0,0 +1,19 @@
|
|
1
|
+
/**
|
2
|
+
* Copyright (c) 2016-present, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under the BSD-style license found in the
|
6
|
+
* LICENSE file in the root directory of this source tree. An additional grant
|
7
|
+
* of patent rights can be found in the PATENTS file in the same directory.
|
8
|
+
*/
|
9
|
+
|
10
|
+
#ifndef FASTTEXT_REAL_H
|
11
|
+
#define FASTTEXT_REAL_H
|
12
|
+
|
13
|
+
namespace fasttext {
|
14
|
+
|
15
|
+
typedef float real;
|
16
|
+
|
17
|
+
}
|
18
|
+
|
19
|
+
#endif
|
@@ -0,0 +1,29 @@
|
|
1
|
+
/**
|
2
|
+
* Copyright (c) 2016-present, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under the BSD-style license found in the
|
6
|
+
* LICENSE file in the root directory of this source tree. An additional grant
|
7
|
+
* of patent rights can be found in the PATENTS file in the same directory.
|
8
|
+
*/
|
9
|
+
|
10
|
+
#include "utils.h"
|
11
|
+
|
12
|
+
#include <ios>
|
13
|
+
|
14
|
+
namespace fasttext {
|
15
|
+
|
16
|
+
namespace utils {
|
17
|
+
|
18
|
+
int64_t size(std::ifstream& ifs) {
|
19
|
+
ifs.seekg(std::streamoff(0), std::ios::end);
|
20
|
+
return ifs.tellg();
|
21
|
+
}
|
22
|
+
|
23
|
+
void seek(std::ifstream& ifs, int64_t pos) {
|
24
|
+
ifs.clear();
|
25
|
+
ifs.seekg(std::streampos(pos));
|
26
|
+
}
|
27
|
+
}
|
28
|
+
|
29
|
+
}
|
@@ -0,0 +1,25 @@
|
|
1
|
+
/**
|
2
|
+
* Copyright (c) 2016-present, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under the BSD-style license found in the
|
6
|
+
* LICENSE file in the root directory of this source tree. An additional grant
|
7
|
+
* of patent rights can be found in the PATENTS file in the same directory.
|
8
|
+
*/
|
9
|
+
|
10
|
+
#ifndef FASTTEXT_UTILS_H
|
11
|
+
#define FASTTEXT_UTILS_H
|
12
|
+
|
13
|
+
#include <fstream>
|
14
|
+
|
15
|
+
namespace fasttext {
|
16
|
+
|
17
|
+
namespace utils {
|
18
|
+
|
19
|
+
int64_t size(std::ifstream&);
|
20
|
+
void seek(std::ifstream&, int64_t);
|
21
|
+
}
|
22
|
+
|
23
|
+
}
|
24
|
+
|
25
|
+
#endif
|
@@ -0,0 +1,137 @@
|
|
1
|
+
/**
|
2
|
+
* Copyright (c) 2016-present, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under the BSD-style license found in the
|
6
|
+
* LICENSE file in the root directory of this source tree. An additional grant
|
7
|
+
* of patent rights can be found in the PATENTS file in the same directory.
|
8
|
+
*/
|
9
|
+
|
10
|
+
#include "vector.h"
|
11
|
+
|
12
|
+
#include <assert.h>
|
13
|
+
|
14
|
+
#include <iomanip>
|
15
|
+
#include <cmath>
|
16
|
+
|
17
|
+
#include "matrix.h"
|
18
|
+
#include "qmatrix.h"
|
19
|
+
|
20
|
+
namespace fasttext {
|
21
|
+
|
22
|
+
Vector::Vector(int64_t m) {
|
23
|
+
m_ = m;
|
24
|
+
data_ = new real[m];
|
25
|
+
}
|
26
|
+
|
27
|
+
Vector::~Vector() {
|
28
|
+
delete[] data_;
|
29
|
+
}
|
30
|
+
|
31
|
+
int64_t Vector::size() const {
|
32
|
+
return m_;
|
33
|
+
}
|
34
|
+
|
35
|
+
void Vector::zero() {
|
36
|
+
for (int64_t i = 0; i < m_; i++) {
|
37
|
+
data_[i] = 0.0;
|
38
|
+
}
|
39
|
+
}
|
40
|
+
|
41
|
+
real Vector::norm() const {
|
42
|
+
real sum = 0;
|
43
|
+
for (int64_t i = 0; i < m_; i++) {
|
44
|
+
sum += data_[i] * data_[i];
|
45
|
+
}
|
46
|
+
return std::sqrt(sum);
|
47
|
+
}
|
48
|
+
|
49
|
+
void Vector::mul(real a) {
|
50
|
+
for (int64_t i = 0; i < m_; i++) {
|
51
|
+
data_[i] *= a;
|
52
|
+
}
|
53
|
+
}
|
54
|
+
|
55
|
+
void Vector::addVector(const Vector& source) {
|
56
|
+
assert(m_ == source.m_);
|
57
|
+
for (int64_t i = 0; i < m_; i++) {
|
58
|
+
data_[i] += source.data_[i];
|
59
|
+
}
|
60
|
+
}
|
61
|
+
|
62
|
+
void Vector::addVector(const Vector& source, real s) {
|
63
|
+
assert(m_ == source.m_);
|
64
|
+
for (int64_t i = 0; i < m_; i++) {
|
65
|
+
data_[i] += s * source.data_[i];
|
66
|
+
}
|
67
|
+
}
|
68
|
+
|
69
|
+
void Vector::addRow(const Matrix& A, int64_t i) {
|
70
|
+
assert(i >= 0);
|
71
|
+
assert(i < A.m_);
|
72
|
+
assert(m_ == A.n_);
|
73
|
+
for (int64_t j = 0; j < A.n_; j++) {
|
74
|
+
data_[j] += A.at(i, j);
|
75
|
+
}
|
76
|
+
}
|
77
|
+
|
78
|
+
void Vector::addRow(const Matrix& A, int64_t i, real a) {
|
79
|
+
assert(i >= 0);
|
80
|
+
assert(i < A.m_);
|
81
|
+
assert(m_ == A.n_);
|
82
|
+
for (int64_t j = 0; j < A.n_; j++) {
|
83
|
+
data_[j] += a * A.at(i, j);
|
84
|
+
}
|
85
|
+
}
|
86
|
+
|
87
|
+
void Vector::addRow(const QMatrix& A, int64_t i) {
|
88
|
+
assert(i >= 0);
|
89
|
+
A.addToVector(*this, i);
|
90
|
+
}
|
91
|
+
|
92
|
+
void Vector::mul(const Matrix& A, const Vector& vec) {
|
93
|
+
assert(A.m_ == m_);
|
94
|
+
assert(A.n_ == vec.m_);
|
95
|
+
for (int64_t i = 0; i < m_; i++) {
|
96
|
+
data_[i] = A.dotRow(vec, i);
|
97
|
+
}
|
98
|
+
}
|
99
|
+
|
100
|
+
void Vector::mul(const QMatrix& A, const Vector& vec) {
|
101
|
+
assert(A.getM() == m_);
|
102
|
+
assert(A.getN() == vec.m_);
|
103
|
+
for (int64_t i = 0; i < m_; i++) {
|
104
|
+
data_[i] = A.dotRow(vec, i);
|
105
|
+
}
|
106
|
+
}
|
107
|
+
|
108
|
+
int64_t Vector::argmax() {
|
109
|
+
real max = data_[0];
|
110
|
+
int64_t argmax = 0;
|
111
|
+
for (int64_t i = 1; i < m_; i++) {
|
112
|
+
if (data_[i] > max) {
|
113
|
+
max = data_[i];
|
114
|
+
argmax = i;
|
115
|
+
}
|
116
|
+
}
|
117
|
+
return argmax;
|
118
|
+
}
|
119
|
+
|
120
|
+
real& Vector::operator[](int64_t i) {
|
121
|
+
return data_[i];
|
122
|
+
}
|
123
|
+
|
124
|
+
const real& Vector::operator[](int64_t i) const {
|
125
|
+
return data_[i];
|
126
|
+
}
|
127
|
+
|
128
|
+
std::ostream& operator<<(std::ostream& os, const Vector& v)
|
129
|
+
{
|
130
|
+
os << std::setprecision(5);
|
131
|
+
for (int64_t j = 0; j < v.m_; j++) {
|
132
|
+
os << v.data_[j] << ' ';
|
133
|
+
}
|
134
|
+
return os;
|
135
|
+
}
|
136
|
+
|
137
|
+
}
|
@@ -0,0 +1,53 @@
|
|
1
|
+
/**
|
2
|
+
* Copyright (c) 2016-present, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under the BSD-style license found in the
|
6
|
+
* LICENSE file in the root directory of this source tree. An additional grant
|
7
|
+
* of patent rights can be found in the PATENTS file in the same directory.
|
8
|
+
*/
|
9
|
+
|
10
|
+
#ifndef FASTTEXT_VECTOR_H
|
11
|
+
#define FASTTEXT_VECTOR_H
|
12
|
+
|
13
|
+
#include <cstdint>
|
14
|
+
#include <ostream>
|
15
|
+
|
16
|
+
#include "real.h"
|
17
|
+
|
18
|
+
namespace fasttext {
|
19
|
+
|
20
|
+
class Matrix;
|
21
|
+
class QMatrix;
|
22
|
+
|
23
|
+
class Vector {
|
24
|
+
|
25
|
+
public:
|
26
|
+
int64_t m_;
|
27
|
+
real* data_;
|
28
|
+
|
29
|
+
explicit Vector(int64_t);
|
30
|
+
~Vector();
|
31
|
+
|
32
|
+
real& operator[](int64_t);
|
33
|
+
const real& operator[](int64_t) const;
|
34
|
+
|
35
|
+
int64_t size() const;
|
36
|
+
void zero();
|
37
|
+
void mul(real);
|
38
|
+
real norm() const;
|
39
|
+
void addVector(const Vector& source);
|
40
|
+
void addVector(const Vector&, real);
|
41
|
+
void addRow(const Matrix&, int64_t);
|
42
|
+
void addRow(const QMatrix&, int64_t);
|
43
|
+
void addRow(const Matrix&, int64_t, real);
|
44
|
+
void mul(const QMatrix&, const Vector&);
|
45
|
+
void mul(const Matrix&, const Vector&);
|
46
|
+
int64_t argmax();
|
47
|
+
};
|
48
|
+
|
49
|
+
std::ostream& operator<<(std::ostream&, const Vector&);
|
50
|
+
|
51
|
+
}
|
52
|
+
|
53
|
+
#endif
|
metadata
ADDED
@@ -0,0 +1,151 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ffi-fasttext
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Brandon Dewitt
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2018-05-27 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: ffi
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: pry
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: bundler
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.16'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.16'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rake
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '10.0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '10.0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: minitest
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '5.0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '5.0'
|
83
|
+
description: " FFI bindings for Facebook's FastText text classification library "
|
84
|
+
email:
|
85
|
+
- brandonsdewitt+rubygems@gmail.com
|
86
|
+
executables: []
|
87
|
+
extensions:
|
88
|
+
- ext/ffi/fasttext/Rakefile
|
89
|
+
extra_rdoc_files: []
|
90
|
+
files:
|
91
|
+
- ".gitignore"
|
92
|
+
- ".travis.yml"
|
93
|
+
- Gemfile
|
94
|
+
- LICENSE.txt
|
95
|
+
- README.md
|
96
|
+
- Rakefile
|
97
|
+
- bin/console
|
98
|
+
- bin/setup
|
99
|
+
- ext/ffi/fasttext/Rakefile
|
100
|
+
- ffi-fasttext.gemspec
|
101
|
+
- lib/ffi/fasttext.rb
|
102
|
+
- lib/ffi/fasttext/version.rb
|
103
|
+
- vendor/fasttext/LICENSE
|
104
|
+
- vendor/fasttext/PATENTS
|
105
|
+
- vendor/fasttext/args.cc
|
106
|
+
- vendor/fasttext/args.h
|
107
|
+
- vendor/fasttext/dictionary.cc
|
108
|
+
- vendor/fasttext/dictionary.h
|
109
|
+
- vendor/fasttext/fasttext.cc
|
110
|
+
- vendor/fasttext/fasttext.h
|
111
|
+
- vendor/fasttext/ffi_fasttext.cc
|
112
|
+
- vendor/fasttext/main.cc
|
113
|
+
- vendor/fasttext/matrix.cc
|
114
|
+
- vendor/fasttext/matrix.h
|
115
|
+
- vendor/fasttext/model.cc
|
116
|
+
- vendor/fasttext/model.h
|
117
|
+
- vendor/fasttext/productquantizer.cc
|
118
|
+
- vendor/fasttext/productquantizer.h
|
119
|
+
- vendor/fasttext/qmatrix.cc
|
120
|
+
- vendor/fasttext/qmatrix.h
|
121
|
+
- vendor/fasttext/real.h
|
122
|
+
- vendor/fasttext/utils.cc
|
123
|
+
- vendor/fasttext/utils.h
|
124
|
+
- vendor/fasttext/vector.cc
|
125
|
+
- vendor/fasttext/vector.h
|
126
|
+
homepage: https://www.github.com/abrandoned/ffi-fastext
|
127
|
+
licenses:
|
128
|
+
- MIT
|
129
|
+
metadata:
|
130
|
+
allowed_push_host: https://rubygems.org
|
131
|
+
post_install_message:
|
132
|
+
rdoc_options: []
|
133
|
+
require_paths:
|
134
|
+
- lib
|
135
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
136
|
+
requirements:
|
137
|
+
- - ">="
|
138
|
+
- !ruby/object:Gem::Version
|
139
|
+
version: '0'
|
140
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
141
|
+
requirements:
|
142
|
+
- - ">="
|
143
|
+
- !ruby/object:Gem::Version
|
144
|
+
version: '0'
|
145
|
+
requirements: []
|
146
|
+
rubyforge_project:
|
147
|
+
rubygems_version: 2.6.13
|
148
|
+
signing_key:
|
149
|
+
specification_version: 4
|
150
|
+
summary: FFI bindings for Facebook's FastText text classification library
|
151
|
+
test_files: []
|