rumale 0.8.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (85) hide show
  1. checksums.yaml +7 -0
  2. data/.coveralls.yml +1 -0
  3. data/.gitignore +20 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +47 -0
  6. data/.rubocop_todo.yml +58 -0
  7. data/.travis.yml +13 -0
  8. data/CHANGELOG.md +2 -0
  9. data/CODE_OF_CONDUCT.md +74 -0
  10. data/Gemfile +4 -0
  11. data/LICENSE.txt +23 -0
  12. data/README.md +175 -0
  13. data/Rakefile +6 -0
  14. data/bin/console +14 -0
  15. data/bin/setup +8 -0
  16. data/lib/rumale.rb +70 -0
  17. data/lib/rumale/base/base_estimator.rb +13 -0
  18. data/lib/rumale/base/classifier.rb +36 -0
  19. data/lib/rumale/base/cluster_analyzer.rb +31 -0
  20. data/lib/rumale/base/evaluator.rb +17 -0
  21. data/lib/rumale/base/regressor.rb +36 -0
  22. data/lib/rumale/base/splitter.rb +21 -0
  23. data/lib/rumale/base/transformer.rb +22 -0
  24. data/lib/rumale/clustering/dbscan.rb +125 -0
  25. data/lib/rumale/clustering/k_means.rb +138 -0
  26. data/lib/rumale/dataset.rb +110 -0
  27. data/lib/rumale/decomposition/nmf.rb +141 -0
  28. data/lib/rumale/decomposition/pca.rb +148 -0
  29. data/lib/rumale/ensemble/ada_boost_classifier.rb +196 -0
  30. data/lib/rumale/ensemble/ada_boost_regressor.rb +178 -0
  31. data/lib/rumale/ensemble/random_forest_classifier.rb +180 -0
  32. data/lib/rumale/ensemble/random_forest_regressor.rb +141 -0
  33. data/lib/rumale/evaluation_measure/accuracy.rb +29 -0
  34. data/lib/rumale/evaluation_measure/f_score.rb +50 -0
  35. data/lib/rumale/evaluation_measure/log_loss.rb +45 -0
  36. data/lib/rumale/evaluation_measure/mean_absolute_error.rb +29 -0
  37. data/lib/rumale/evaluation_measure/mean_squared_error.rb +29 -0
  38. data/lib/rumale/evaluation_measure/normalized_mutual_information.rb +62 -0
  39. data/lib/rumale/evaluation_measure/precision.rb +50 -0
  40. data/lib/rumale/evaluation_measure/precision_recall.rb +91 -0
  41. data/lib/rumale/evaluation_measure/purity.rb +40 -0
  42. data/lib/rumale/evaluation_measure/r2_score.rb +43 -0
  43. data/lib/rumale/evaluation_measure/recall.rb +50 -0
  44. data/lib/rumale/kernel_approximation/rbf.rb +121 -0
  45. data/lib/rumale/kernel_machine/kernel_svc.rb +193 -0
  46. data/lib/rumale/linear_model/base_linear_model.rb +89 -0
  47. data/lib/rumale/linear_model/lasso.rb +136 -0
  48. data/lib/rumale/linear_model/linear_regression.rb +110 -0
  49. data/lib/rumale/linear_model/logistic_regression.rb +159 -0
  50. data/lib/rumale/linear_model/ridge.rb +110 -0
  51. data/lib/rumale/linear_model/svc.rb +183 -0
  52. data/lib/rumale/linear_model/svr.rb +122 -0
  53. data/lib/rumale/model_selection/cross_validation.rb +123 -0
  54. data/lib/rumale/model_selection/grid_search_cv.rb +247 -0
  55. data/lib/rumale/model_selection/k_fold.rb +76 -0
  56. data/lib/rumale/model_selection/stratified_k_fold.rb +94 -0
  57. data/lib/rumale/multiclass/one_vs_rest_classifier.rb +100 -0
  58. data/lib/rumale/naive_bayes/naive_bayes.rb +315 -0
  59. data/lib/rumale/nearest_neighbors/k_neighbors_classifier.rb +111 -0
  60. data/lib/rumale/nearest_neighbors/k_neighbors_regressor.rb +93 -0
  61. data/lib/rumale/optimizer/nadam.rb +90 -0
  62. data/lib/rumale/optimizer/rmsprop.rb +69 -0
  63. data/lib/rumale/optimizer/sgd.rb +65 -0
  64. data/lib/rumale/optimizer/yellow_fin.rb +144 -0
  65. data/lib/rumale/pairwise_metric.rb +91 -0
  66. data/lib/rumale/pipeline/pipeline.rb +197 -0
  67. data/lib/rumale/polynomial_model/base_factorization_machine.rb +99 -0
  68. data/lib/rumale/polynomial_model/factorization_machine_classifier.rb +197 -0
  69. data/lib/rumale/polynomial_model/factorization_machine_regressor.rb +131 -0
  70. data/lib/rumale/preprocessing/l2_normalizer.rb +62 -0
  71. data/lib/rumale/preprocessing/label_encoder.rb +94 -0
  72. data/lib/rumale/preprocessing/min_max_scaler.rb +92 -0
  73. data/lib/rumale/preprocessing/one_hot_encoder.rb +98 -0
  74. data/lib/rumale/preprocessing/standard_scaler.rb +86 -0
  75. data/lib/rumale/probabilistic_output.rb +112 -0
  76. data/lib/rumale/tree/base_decision_tree.rb +153 -0
  77. data/lib/rumale/tree/decision_tree_classifier.rb +163 -0
  78. data/lib/rumale/tree/decision_tree_regressor.rb +135 -0
  79. data/lib/rumale/tree/node.rb +70 -0
  80. data/lib/rumale/utils.rb +37 -0
  81. data/lib/rumale/validation.rb +79 -0
  82. data/lib/rumale/values.rb +13 -0
  83. data/lib/rumale/version.rb +6 -0
  84. data/rumale.gemspec +41 -0
  85. metadata +204 -0
@@ -0,0 +1,91 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rumale/validation'
4
+
5
+ module Rumale
6
+ # Module for calculating pairwise distances, similarities, and kernels.
7
+ module PairwiseMetric
8
+ class << self
9
+ # Calculate the pairwise euclidean distances between x and y.
10
+ #
11
+ # @param x [Numo::DFloat] (shape: [n_samples_x, n_features])
12
+ # @param y [Numo::DFloat] (shape: [n_samples_y, n_features])
13
+ # @return [Numo::DFloat] (shape: [n_samples_x, n_samples_x] or [n_samples_x, n_samples_y] if y is given)
14
+ def euclidean_distance(x, y = nil)
15
+ y = x if y.nil?
16
+ Rumale::Validation.check_sample_array(x)
17
+ Rumale::Validation.check_sample_array(y)
18
+ sum_x_vec = (x**2).sum(1)
19
+ sum_y_vec = (y**2).sum(1)
20
+ dot_xy_mat = x.dot(y.transpose)
21
+ distance_matrix = dot_xy_mat * -2.0 +
22
+ sum_x_vec.tile(y.shape[0], 1).transpose +
23
+ sum_y_vec.tile(x.shape[0], 1)
24
+ Numo::NMath.sqrt(distance_matrix.abs)
25
+ end
26
+
27
+ # Calculate the rbf kernel between x and y.
28
+ #
29
+ # @param x [Numo::DFloat] (shape: [n_samples_x, n_features])
30
+ # @param y [Numo::DFloat] (shape: [n_samples_y, n_features])
31
+ # @param gamma [Float] The parameter of rbf kernel, if nil it is 1 / n_features.
32
+ # @return [Numo::DFloat] (shape: [n_samples_x, n_samples_x] or [n_samples_x, n_samples_y] if y is given)
33
+ def rbf_kernel(x, y = nil, gamma = nil)
34
+ y = x if y.nil?
35
+ gamma ||= 1.0 / x.shape[1]
36
+ Rumale::Validation.check_sample_array(x)
37
+ Rumale::Validation.check_sample_array(y)
38
+ Rumale::Validation.check_params_float(gamma: gamma)
39
+ distance_matrix = euclidean_distance(x, y)
40
+ Numo::NMath.exp((distance_matrix**2) * -gamma)
41
+ end
42
+
43
+ # Calculate the linear kernel between x and y.
44
+ #
45
+ # @param x [Numo::DFloat] (shape: [n_samples_x, n_features])
46
+ # @param y [Numo::DFloat] (shape: [n_samples_y, n_features])
47
+ # @return [Numo::DFloat] (shape: [n_samples_x, n_samples_x] or [n_samples_x, n_samples_y] if y is given)
48
+ def linear_kernel(x, y = nil)
49
+ y = x if y.nil?
50
+ Rumale::Validation.check_sample_array(x)
51
+ Rumale::Validation.check_sample_array(y)
52
+ x.dot(y.transpose)
53
+ end
54
+
55
+ # Calculate the polynomial kernel between x and y.
56
+ #
57
+ # @param x [Numo::DFloat] (shape: [n_samples_x, n_features])
58
+ # @param y [Numo::DFloat] (shape: [n_samples_y, n_features])
59
+ # @param degree [Integer] The parameter of polynomial kernel.
60
+ # @param gamma [Float] The parameter of polynomial kernel, if nil it is 1 / n_features.
61
+ # @param coef [Integer] The parameter of polynomial kernel.
62
+ # @return [Numo::DFloat] (shape: [n_samples_x, n_samples_x] or [n_samples_x, n_samples_y] if y is given)
63
+ def polynomial_kernel(x, y = nil, degree = 3, gamma = nil, coef = 1)
64
+ y = x if y.nil?
65
+ gamma ||= 1.0 / x.shape[1]
66
+ Rumale::Validation.check_sample_array(x)
67
+ Rumale::Validation.check_sample_array(y)
68
+ Rumale::Validation.check_params_float(gamma: gamma)
69
+ Rumale::Validation.check_params_integer(degree: degree, coef: coef)
70
+ (x.dot(y.transpose) * gamma + coef)**degree
71
+ end
72
+
73
+ # Calculate the sigmoid kernel between x and y.
74
+ #
75
+ # @param x [Numo::DFloat] (shape: [n_samples_x, n_features])
76
+ # @param y [Numo::DFloat] (shape: [n_samples_y, n_features])
77
+ # @param gamma [Float] The parameter of polynomial kernel, if nil it is 1 / n_features.
78
+ # @param coef [Integer] The parameter of polynomial kernel.
79
+ # @return [Numo::DFloat] (shape: [n_samples_x, n_samples_x] or [n_samples_x, n_samples_y] if y is given)
80
+ def sigmoid_kernel(x, y = nil, gamma = nil, coef = 1)
81
+ y = x if y.nil?
82
+ gamma ||= 1.0 / x.shape[1]
83
+ Rumale::Validation.check_sample_array(x)
84
+ Rumale::Validation.check_sample_array(y)
85
+ Rumale::Validation.check_params_float(gamma: gamma)
86
+ Rumale::Validation.check_params_integer(coef: coef)
87
+ Numo::NMath.tanh(x.dot(y.transpose) * gamma + coef)
88
+ end
89
+ end
90
+ end
91
+ end
@@ -0,0 +1,197 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rumale/validation'
4
+ require 'rumale/base/base_estimator'
5
+
6
+ module Rumale
7
+ # Module implements utilities of pipeline that cosists of a chain of transfomers and estimators.
8
+ module Pipeline
9
+ # Pipeline is a class that implements the function to perform the transformers and estimators sequencially.
10
+ #
11
+ # @example
12
+ # rbf = Rumale::KernelApproximation::RBF.new(gamma: 1.0, n_coponents: 128, random_seed: 1)
13
+ # svc = Rumale::LinearModel::SVC.new(reg_param: 1.0, fit_bias: true, max_iter: 5000, random_seed: 1)
14
+ # pipeline = Rumale::Pipeline::Pipeline.new(steps: { trs: rbf, est: svc })
15
+ # pipeline.fit(training_samples, traininig_labels)
16
+ # results = pipeline.predict(testing_samples)
17
+ #
18
+ class Pipeline
19
+ include Base::BaseEstimator
20
+ include Validation
21
+
22
+ # Return the steps.
23
+ # @return [Hash]
24
+ attr_reader :steps
25
+
26
+ # Create a new pipeline.
27
+ #
28
+ # @param steps [Hash] List of transformers and estimators. The order of transforms follows the insertion order of hash keys.
29
+ # The last entry is considered an estimator.
30
+ def initialize(steps:)
31
+ check_params_type(Hash, steps: steps)
32
+ validate_steps(steps)
33
+ @params = {}
34
+ @steps = steps
35
+ end
36
+
37
+ # Fit the model with given training data.
38
+ #
39
+ # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be transformed and used for fitting the model.
40
+ # @param y [Numo::NArray] (shape: [n_samples, n_outputs]) The target values or labels to be used for fitting the model.
41
+ # @return [Pipeline] The learned pipeline itself.
42
+ def fit(x, y)
43
+ check_sample_array(x)
44
+ trans_x = apply_transforms(x, y, fit: true)
45
+ last_estimator&.fit(trans_x, y)
46
+ self
47
+ end
48
+
49
+ # Call the fit_predict method of last estimator after applying all transforms.
50
+ #
51
+ # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be transformed and used for fitting the model.
52
+ # @param y [Numo::NArray] (shape: [n_samples, n_outputs], default: nil) The target values or labels to be used for fitting the model.
53
+ # @return [Numo::NArray] The predicted results by last estimator.
54
+ def fit_predict(x, y = nil)
55
+ check_sample_array(x)
56
+ trans_x = apply_transforms(x, y, fit: true)
57
+ last_estimator.fit_predict(trans_x)
58
+ end
59
+
60
+ # Call the fit_transform method of last estimator after applying all transforms.
61
+ #
62
+ # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be transformed and used for fitting the model.
63
+ # @param y [Numo::NArray] (shape: [n_samples, n_outputs], default: nil) The target values or labels to be used for fitting the model.
64
+ # @return [Numo::NArray] The predicted results by last estimator.
65
+ def fit_transform(x, y = nil)
66
+ check_sample_array(x)
67
+ trans_x = apply_transforms(x, y, fit: true)
68
+ last_estimator.fit_transform(trans_x, y)
69
+ end
70
+
71
+ # Call the decision_function method of last estimator after applying all transforms.
72
+ #
73
+ # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
74
+ # @return [Numo::DFloat] (shape: [n_samples]) Confidence score per sample.
75
+ def decision_function(x)
76
+ check_sample_array(x)
77
+ trans_x = apply_transforms(x)
78
+ last_estimator.decision_function(trans_x)
79
+ end
80
+
81
+ # Call the predict method of last estimator after applying all transforms.
82
+ #
83
+ # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to obtain prediction result.
84
+ # @return [Numo::NArray] The predicted results by last estimator.
85
+ def predict(x)
86
+ check_sample_array(x)
87
+ trans_x = apply_transforms(x)
88
+ last_estimator.predict(trans_x)
89
+ end
90
+
91
+ # Call the predict_log_proba method of last estimator after applying all transforms.
92
+ #
93
+ # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the log-probailities.
94
+ # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted log-probability of each class per sample.
95
+ def predict_log_proba(x)
96
+ check_sample_array(x)
97
+ trans_x = apply_transforms(x)
98
+ last_estimator.predict_log_proba(trans_x)
99
+ end
100
+
101
+ # Call the predict_proba method of last estimator after applying all transforms.
102
+ #
103
+ # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probailities.
104
+ # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
105
+ def predict_proba(x)
106
+ check_sample_array(x)
107
+ trans_x = apply_transforms(x)
108
+ last_estimator.predict_proba(trans_x)
109
+ end
110
+
111
+ # Call the transform method of last estimator after applying all transforms.
112
+ #
113
+ # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to be transformed.
114
+ # @return [Numo::DFloat] (shape: [n_samples, n_components]) The transformed samples.
115
+ def transform(x)
116
+ check_sample_array(x)
117
+ trans_x = apply_transforms(x)
118
+ last_estimator.nil? ? trans_x : last_estimator.transform(trans_x)
119
+ end
120
+
121
+ # Call the inverse_transform method in reverse order.
122
+ #
123
+ # @param z [Numo::DFloat] (shape: [n_samples, n_components]) The transformed samples to be restored into original space.
124
+ # @return [Numo::DFloat] (shape: [n_samples, n_featuress]) The restored samples.
125
+ def inverse_transform(z)
126
+ check_sample_array(z)
127
+ itrans_z = z
128
+ @steps.keys.reverse_each do |name|
129
+ transformer = @steps[name]
130
+ next if transformer.nil?
131
+ itrans_z = transformer.inverse_transform(itrans_z)
132
+ end
133
+ itrans_z
134
+ end
135
+
136
+ # Call the score method of last estimator after applying all transforms.
137
+ #
138
+ # @param x [Numo::DFloat] (shape: [n_samples, n_features]) Testing data.
139
+ # @param y [Numo::NArray] (shape: [n_samples, n_outputs]) True target values or labels for testing data.
140
+ # @return [Float] The score of last estimator
141
+ def score(x, y)
142
+ check_sample_array(x)
143
+ trans_x = apply_transforms(x)
144
+ last_estimator.score(trans_x, y)
145
+ end
146
+
147
+ # Dump marshal data.
148
+ # @return [Hash] The marshal data about Pipeline.
149
+ def marshal_dump
150
+ { params: @params,
151
+ steps: @steps }
152
+ end
153
+
154
+ # Load marshal data.
155
+ # @return [nil]
156
+ def marshal_load(obj)
157
+ @params = obj[:params]
158
+ @steps = obj[:steps]
159
+ nil
160
+ end
161
+
162
+ private
163
+
164
+ def validate_steps(steps)
165
+ steps.keys[0...-1].each do |name|
166
+ transformer = steps[name]
167
+ next if transformer.nil? || %i[fit transform].all? { |m| transformer.class.method_defined?(m) }
168
+ raise TypeError,
169
+ 'Class of intermediate step in pipeline should be implemented fit and transform methods: ' \
170
+ "#{name} => #{transformer.class}"
171
+ end
172
+
173
+ estimator = steps[steps.keys.last]
174
+ unless estimator.nil? || estimator.class.method_defined?(:fit) # rubocop:disable Style/GuardClause
175
+ raise TypeError,
176
+ 'Class of last step in pipeline should be implemented fit method: ' \
177
+ "#{steps.keys.last} => #{estimator.class}"
178
+ end
179
+ end
180
+
181
+ def apply_transforms(x, y = nil, fit: false)
182
+ trans_x = x
183
+ @steps.keys[0...-1].each do |name|
184
+ transformer = @steps[name]
185
+ next if transformer.nil?
186
+ transformer.fit(trans_x, y) if fit
187
+ trans_x = transformer.transform(trans_x)
188
+ end
189
+ trans_x
190
+ end
191
+
192
+ def last_estimator
193
+ @steps[@steps.keys.last]
194
+ end
195
+ end
196
+ end
197
+ end
@@ -0,0 +1,99 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rumale/base/base_estimator'
4
+ require 'rumale/optimizer/nadam'
5
+
6
+ module Rumale
7
+ # This module consists of the classes that implement polynomial models.
8
+ module PolynomialModel
9
+ # BaseFactorizationMachine is an abstract class for implementation of Factorization Machine-based estimators.
10
+ # This class is used internally.
11
+ class BaseFactorizationMachine
12
+ include Base::BaseEstimator
13
+
14
+ # Initialize a Factorization Machine-based estimator.
15
+ #
16
+ # @param n_factors [Integer] The maximum number of iterations.
17
+ # @param loss [String] The loss function ('hinge' or 'logistic' or nil).
18
+ # @param reg_param_linear [Float] The regularization parameter for linear model.
19
+ # @param reg_param_factor [Float] The regularization parameter for factor matrix.
20
+ # @param max_iter [Integer] The maximum number of iterations.
21
+ # @param batch_size [Integer] The size of the mini batches.
22
+ # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
23
+ # If nil is given, Nadam is used.
24
+ # @param random_seed [Integer] The seed value using to initialize the random generator.
25
+ def initialize(n_factors: 2, loss: nil, reg_param_linear: 1.0, reg_param_factor: 1.0,
26
+ max_iter: 1000, batch_size: 10, optimizer: nil, random_seed: nil)
27
+ @params = {}
28
+ @params[:n_factors] = n_factors
29
+ @params[:loss] = loss unless loss.nil?
30
+ @params[:reg_param_linear] = reg_param_linear
31
+ @params[:reg_param_factor] = reg_param_factor
32
+ @params[:max_iter] = max_iter
33
+ @params[:batch_size] = batch_size
34
+ @params[:optimizer] = optimizer
35
+ @params[:optimizer] ||= Optimizer::Nadam.new
36
+ @params[:random_seed] = random_seed
37
+ @params[:random_seed] ||= srand
38
+ @factor_mat = nil
39
+ @weight_vec = nil
40
+ @bias_term = nil
41
+ @rng = Random.new(@params[:random_seed])
42
+ end
43
+
44
+ private
45
+
46
+ def partial_fit(x, y)
47
+ # Initialize some variables.
48
+ n_samples, n_features = x.shape
49
+ rand_ids = [*0...n_samples].shuffle(random: @rng)
50
+ weight_vec = Numo::DFloat.zeros(n_features + 1)
51
+ factor_mat = Numo::DFloat.zeros(@params[:n_factors], n_features)
52
+ weight_optimizer = @params[:optimizer].dup
53
+ factor_optimizers = Array.new(@params[:n_factors]) { @params[:optimizer].dup }
54
+ # Start optimization.
55
+ @params[:max_iter].times do |_t|
56
+ # Random sampling.
57
+ subset_ids = rand_ids.shift(@params[:batch_size])
58
+ rand_ids.concat(subset_ids)
59
+ data = x[subset_ids, true]
60
+ ex_data = expand_feature(data)
61
+ targets = y[subset_ids]
62
+ # Calculate gradients for loss function.
63
+ loss_grad = loss_gradient(data, ex_data, targets, factor_mat, weight_vec)
64
+ next if loss_grad.ne(0.0).count.zero?
65
+ # Update each parameter.
66
+ weight_vec = weight_optimizer.call(weight_vec, weight_gradient(loss_grad, ex_data, weight_vec))
67
+ @params[:n_factors].times do |n|
68
+ factor_mat[n, true] = factor_optimizers[n].call(factor_mat[n, true],
69
+ factor_gradient(loss_grad, data, factor_mat[n, true]))
70
+ end
71
+ end
72
+ [factor_mat, *split_weight_vec_bias(weight_vec)]
73
+ end
74
+
75
+ def loss_gradient(_x, _expanded_x, _y, _factor, _weight)
76
+ raise NotImplementedError, "#{__method__} has to be implemented in #{self.class}."
77
+ end
78
+
79
+ def weight_gradient(loss_grad, data, weight)
80
+ (loss_grad.expand_dims(1) * data).mean(0) + @params[:reg_param_linear] * weight
81
+ end
82
+
83
+ def factor_gradient(loss_grad, data, factor)
84
+ (loss_grad.expand_dims(1) * (data * data.dot(factor).expand_dims(1) - factor * (data**2))).mean(0) +
85
+ @params[:reg_param_factor] * factor
86
+ end
87
+
88
+ def expand_feature(x)
89
+ Numo::NArray.hstack([x, Numo::DFloat.ones([x.shape[0], 1])])
90
+ end
91
+
92
+ def split_weight_vec_bias(weight_vec)
93
+ weights = weight_vec[0...-1].dup
94
+ bias = weight_vec[-1]
95
+ [weights, bias]
96
+ end
97
+ end
98
+ end
99
+ end
@@ -0,0 +1,197 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rumale/base/classifier'
4
+ require 'rumale/polynomial_model/base_factorization_machine'
5
+
6
+ module Rumale
7
+ # This module consists of the classes that implement polynomial models.
8
+ module PolynomialModel
9
+ # FactorizationMachineClassifier is a class that implements Factorization Machine
10
+ # with stochastic gradient descent (SGD) optimization.
11
+ # For multiclass classification problem, it uses one-vs-the-rest strategy.
12
+ #
13
+ # @example
14
+ # estimator =
15
+ # Rumale::PolynomialModel::FactorizationMachineClassifier.new(
16
+ # n_factors: 10, loss: 'hinge', reg_param_linear: 0.001, reg_param_factor: 0.001,
17
+ # max_iter: 5000, batch_size: 50, random_seed: 1)
18
+ # estimator.fit(training_samples, traininig_labels)
19
+ # results = estimator.predict(testing_samples)
20
+ #
21
+ # *Reference*
22
+ # - S. Rendle, "Factorization Machines with libFM," ACM TIST, vol. 3 (3), pp. 57:1--57:22, 2012.
23
+ # - S. Rendle, "Factorization Machines," Proc. ICDM'10, pp. 995--1000, 2010.
24
+ class FactorizationMachineClassifier < BaseFactorizationMachine
25
+ include Base::Classifier
26
+
27
+ # Return the factor matrix for Factorization Machine.
28
+ # @return [Numo::DFloat] (shape: [n_classes, n_factors, n_features])
29
+ attr_reader :factor_mat
30
+
31
+ # Return the weight vector for Factorization Machine.
32
+ # @return [Numo::DFloat] (shape: [n_classes, n_features])
33
+ attr_reader :weight_vec
34
+
35
+ # Return the bias term for Factoriazation Machine.
36
+ # @return [Numo::DFloat] (shape: [n_classes])
37
+ attr_reader :bias_term
38
+
39
+ # Return the class labels.
40
+ # @return [Numo::Int32] (shape: [n_classes])
41
+ attr_reader :classes
42
+
43
+ # Return the random generator for random sampling.
44
+ # @return [Random]
45
+ attr_reader :rng
46
+
47
+ # Create a new classifier with Factorization Machine.
48
+ #
49
+ # @param n_factors [Integer] The maximum number of iterations.
50
+ # @param loss [String] The loss function ('hinge' or 'logistic').
51
+ # @param reg_param_linear [Float] The regularization parameter for linear model.
52
+ # @param reg_param_factor [Float] The regularization parameter for factor matrix.
53
+ # @param max_iter [Integer] The maximum number of iterations.
54
+ # @param batch_size [Integer] The size of the mini batches.
55
+ # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
56
+ # If nil is given, Nadam is used.
57
+ # @param random_seed [Integer] The seed value using to initialize the random generator.
58
+ def initialize(n_factors: 2, loss: 'hinge', reg_param_linear: 1.0, reg_param_factor: 1.0,
59
+ max_iter: 1000, batch_size: 10, optimizer: nil, random_seed: nil)
60
+ check_params_float(reg_param_linear: reg_param_linear, reg_param_factor: reg_param_factor)
61
+ check_params_integer(n_factors: n_factors, max_iter: max_iter, batch_size: batch_size)
62
+ check_params_string(loss: loss)
63
+ check_params_type_or_nil(Integer, random_seed: random_seed)
64
+ check_params_positive(n_factors: n_factors,
65
+ reg_param_linear: reg_param_linear, reg_param_factor: reg_param_factor,
66
+ max_iter: max_iter, batch_size: batch_size)
67
+ super
68
+ @classes = nil
69
+ end
70
+
71
+ # Fit the model with given training data.
72
+ #
73
+ # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
74
+ # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
75
+ # @return [FactorizationMachineClassifier] The learned classifier itself.
76
+ def fit(x, y)
77
+ check_sample_array(x)
78
+ check_label_array(y)
79
+ check_sample_label_size(x, y)
80
+
81
+ @classes = Numo::Int32[*y.to_a.uniq.sort]
82
+ n_classes = @classes.size
83
+ _n_samples, n_features = x.shape
84
+
85
+ if n_classes > 2
86
+ @factor_mat = Numo::DFloat.zeros(n_classes, @params[:n_factors], n_features)
87
+ @weight_vec = Numo::DFloat.zeros(n_classes, n_features)
88
+ @bias_term = Numo::DFloat.zeros(n_classes)
89
+ n_classes.times do |n|
90
+ bin_y = Numo::Int32.cast(y.eq(@classes[n])) * 2 - 1
91
+ @factor_mat[n, true, true], @weight_vec[n, true], @bias_term[n] = partial_fit(x, bin_y)
92
+ end
93
+ else
94
+ negative_label = y.to_a.uniq.min
95
+ bin_y = Numo::Int32.cast(y.ne(negative_label)) * 2 - 1
96
+ @factor_mat, @weight_vec, @bias_term = partial_fit(x, bin_y)
97
+ end
98
+
99
+ self
100
+ end
101
+
102
+ # Calculate confidence scores for samples.
103
+ #
104
+ # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
105
+ # @return [Numo::DFloat] (shape: [n_samples]) Confidence score per sample.
106
+ def decision_function(x)
107
+ check_sample_array(x)
108
+ linear_term = @bias_term + x.dot(@weight_vec.transpose)
109
+ factor_term = if @classes.size <= 2
110
+ 0.5 * (@factor_mat.dot(x.transpose)**2 - (@factor_mat**2).dot(x.transpose**2)).sum(0)
111
+ else
112
+ 0.5 * (@factor_mat.dot(x.transpose)**2 - (@factor_mat**2).dot(x.transpose**2)).sum(1).transpose
113
+ end
114
+ linear_term + factor_term
115
+ end
116
+
117
+ # Predict class labels for samples.
118
+ #
119
+ # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
120
+ # @return [Numo::Int32] (shape: [n_samples]) Predicted class label per sample.
121
+ def predict(x)
122
+ check_sample_array(x)
123
+ return Numo::Int32.cast(decision_function(x).ge(0.0)) * 2 - 1 if @classes.size <= 2
124
+
125
+ n_samples, = x.shape
126
+ decision_values = decision_function(x)
127
+ Numo::Int32.asarray(Array.new(n_samples) { |n| @classes[decision_values[n, true].max_index] })
128
+ end
129
+
130
+ # Predict probability for samples.
131
+ #
132
+ # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probailities.
133
+ # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
134
+ def predict_proba(x)
135
+ check_sample_array(x)
136
+ proba = 1.0 / (Numo::NMath.exp(-decision_function(x)) + 1.0)
137
+ return (proba.transpose / proba.sum(axis: 1)).transpose if @classes.size > 2
138
+
139
+ n_samples, = x.shape
140
+ probs = Numo::DFloat.zeros(n_samples, 2)
141
+ probs[true, 1] = proba
142
+ probs[true, 0] = 1.0 - proba
143
+ probs
144
+ end
145
+
146
+ # Dump marshal data.
147
+ # @return [Hash] The marshal data about FactorizationMachineClassifier.
148
+ def marshal_dump
149
+ { params: @params,
150
+ factor_mat: @factor_mat,
151
+ weight_vec: @weight_vec,
152
+ bias_term: @bias_term,
153
+ classes: @classes,
154
+ rng: @rng }
155
+ end
156
+
157
+ # Load marshal data.
158
+ # @return [nil]
159
+ def marshal_load(obj)
160
+ @params = obj[:params]
161
+ @factor_mat = obj[:factor_mat]
162
+ @weight_vec = obj[:weight_vec]
163
+ @bias_term = obj[:bias_term]
164
+ @classes = obj[:classes]
165
+ @rng = obj[:rng]
166
+ nil
167
+ end
168
+
169
+ private
170
+
171
+ def bin_decision_function(x, ex_x, factor, weight)
172
+ ex_x.dot(weight) + 0.5 * (factor.dot(x.transpose)**2 - (factor**2).dot(x.transpose**2)).sum(0)
173
+ end
174
+
175
+ def hinge_loss_gradient(x, ex_x, y, factor, weight)
176
+ evaluated = y * bin_decision_function(x, ex_x, factor, weight)
177
+ gradient = Numo::DFloat.zeros(evaluated.size)
178
+ gradient[evaluated < 1.0] = -y[evaluated < 1.0]
179
+ gradient
180
+ end
181
+
182
+ def logistic_loss_gradient(x, ex_x, y, factor, weight)
183
+ evaluated = y * bin_decision_function(x, ex_x, factor, weight)
184
+ sigmoid_func = 1.0 / (Numo::NMath.exp(-evaluated) + 1.0)
185
+ (sigmoid_func - 1.0) * y
186
+ end
187
+
188
+ def loss_gradient(x, ex_x, y, factor, weight)
189
+ if @params[:loss] == 'hinge'
190
+ hinge_loss_gradient(x, ex_x, y, factor, weight)
191
+ else
192
+ logistic_loss_gradient(x, ex_x, y, factor, weight)
193
+ end
194
+ end
195
+ end
196
+ end
197
+ end