rumale-linear_model 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +27 -0
- data/README.md +34 -0
- data/lib/rumale/linear_model/base_sgd.rb +275 -0
- data/lib/rumale/linear_model/elastic_net.rb +115 -0
- data/lib/rumale/linear_model/lasso.rb +111 -0
- data/lib/rumale/linear_model/linear_regression.rb +199 -0
- data/lib/rumale/linear_model/logistic_regression.rb +266 -0
- data/lib/rumale/linear_model/nnls.rb +141 -0
- data/lib/rumale/linear_model/ridge.rb +206 -0
- data/lib/rumale/linear_model/svc.rb +203 -0
- data/lib/rumale/linear_model/svr.rb +126 -0
- data/lib/rumale/linear_model/version.rb +10 -0
- data/lib/rumale/linear_model.rb +14 -0
- metadata +106 -0
@@ -0,0 +1,206 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'lbfgsb'
|
4
|
+
|
5
|
+
require 'rumale/base/regressor'
|
6
|
+
require 'rumale/validation'
|
7
|
+
require 'rumale/linear_model/base_sgd'
|
8
|
+
|
9
|
+
module Rumale
|
10
|
+
module LinearModel
|
11
|
+
# Ridge is a class that implements Ridge Regression
|
12
|
+
# with stochastic gradient descent (SGD) optimization,
|
13
|
+
# singular value decomposition (SVD), or L-BFGS optimization.
|
14
|
+
#
|
15
|
+
# @example
|
16
|
+
# require 'rumale/linear_model/ridge'
|
17
|
+
#
|
18
|
+
# estimator =
|
19
|
+
# Rumale::LinearModel::Ridge.new(reg_param: 0.1, max_iter: 1000, batch_size: 20, random_seed: 1)
|
20
|
+
# estimator.fit(training_samples, traininig_values)
|
21
|
+
# results = estimator.predict(testing_samples)
|
22
|
+
#
|
23
|
+
# # If Numo::Linalg is installed, you can specify 'svd' for the solver option.
|
24
|
+
# require 'numo/linalg/autoloader'
|
25
|
+
# require 'rumale/linear_model/ridge'
|
26
|
+
#
|
27
|
+
# estimator = Rumale::LinearModel::Ridge.new(reg_param: 0.1, solver: 'svd')
|
28
|
+
# estimator.fit(training_samples, traininig_values)
|
29
|
+
# results = estimator.predict(testing_samples)
|
30
|
+
#
|
31
|
+
# *Reference*
|
32
|
+
# - Bottou, L., "Large-Scale Machine Learning with Stochastic Gradient Descent," Proc. COMPSTAT'10, pp. 177--186, 2010.
|
33
|
+
class Ridge < BaseSGD
|
34
|
+
include ::Rumale::Base::Regressor
|
35
|
+
|
36
|
+
# Return the weight vector.
|
37
|
+
# @return [Numo::DFloat] (shape: [n_outputs, n_features])
|
38
|
+
attr_reader :weight_vec
|
39
|
+
|
40
|
+
# Return the bias term (a.k.a. intercept).
|
41
|
+
# @return [Numo::DFloat] (shape: [n_outputs])
|
42
|
+
attr_reader :bias_term
|
43
|
+
|
44
|
+
# Return the random generator for random sampling.
|
45
|
+
# @return [Random]
|
46
|
+
attr_reader :rng
|
47
|
+
|
48
|
+
# Create a new Ridge regressor.
|
49
|
+
#
|
50
|
+
# @param learning_rate [Float] The initial value of learning rate.
|
51
|
+
# The learning rate decreases as the iteration proceeds according to the equation: learning_rate / (1 + decay * t).
|
52
|
+
# If solver is not 'sgd', this parameter is ignored.
|
53
|
+
# @param decay [Float] The smoothing parameter for decreasing learning rate as the iteration proceeds.
|
54
|
+
# If nil is given, the decay sets to 'reg_param * learning_rate'.
|
55
|
+
# If solver is not 'sgd', this parameter is ignored.
|
56
|
+
# @param momentum [Float] The momentum factor.
|
57
|
+
# If solver is not 'sgd', this parameter is ignored.
|
58
|
+
# @param reg_param [Float] The regularization parameter.
|
59
|
+
# @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
|
60
|
+
# @param bias_scale [Float] The scale of the bias term.
|
61
|
+
# @param max_iter [Integer] The maximum number of epochs that indicates
|
62
|
+
# how many times the whole data is given to the training process.
|
63
|
+
# If solver is 'svd', this parameter is ignored.
|
64
|
+
# @param batch_size [Integer] The size of the mini batches.
|
65
|
+
# If solver is not 'sgd', this parameter is ignored.
|
66
|
+
# @param tol [Float] The tolerance of loss for terminating optimization.
|
67
|
+
# If solver is 'svd', this parameter is ignored.
|
68
|
+
# @param solver [String] The algorithm to calculate weights. ('auto', 'sgd', 'svd', or 'lbfgs').
|
69
|
+
# 'auto' chooses the 'svd' solver if Numo::Linalg is loaded. Otherwise, it chooses the 'lbfgs' solver.
|
70
|
+
# 'sgd' uses the stochastic gradient descent optimization.
|
71
|
+
# 'svd' performs singular value decomposition of samples.
|
72
|
+
# 'lbfgs' uses the L-BFGS method for optimization.
|
73
|
+
# @param n_jobs [Integer] The number of jobs for running the fit method in parallel.
|
74
|
+
# If nil is given, the method does not execute in parallel.
|
75
|
+
# If zero or less is given, it becomes equal to the number of processors.
|
76
|
+
# This parameter is ignored if the Parallel gem is not loaded or solver is not 'sgd'.
|
77
|
+
# @param verbose [Boolean] The flag indicating whether to output loss during iteration.
|
78
|
+
# If solver is 'svd', this parameter is ignored.
|
79
|
+
# @param random_seed [Integer] The seed value using to initialize the random generator.
|
80
|
+
def initialize(learning_rate: 0.01, decay: nil, momentum: 0.9,
|
81
|
+
reg_param: 1.0, fit_bias: true, bias_scale: 1.0,
|
82
|
+
max_iter: 1000, batch_size: 50, tol: 1e-4,
|
83
|
+
solver: 'auto',
|
84
|
+
n_jobs: nil, verbose: false, random_seed: nil)
|
85
|
+
super()
|
86
|
+
@params.merge!(method(:initialize).parameters.to_h { |_t, arg| [arg, binding.local_variable_get(arg)] })
|
87
|
+
@params[:solver] = if solver == 'auto'
|
88
|
+
enable_linalg?(warning: false) ? 'svd' : 'lbfgs'
|
89
|
+
else
|
90
|
+
solver.match?(/^svd$|^sgd$|^lbfgs$/) ? solver : 'lbfgs'
|
91
|
+
end
|
92
|
+
@params[:decay] ||= @params[:reg_param] * @params[:learning_rate]
|
93
|
+
@params[:random_seed] ||= srand
|
94
|
+
@rng = Random.new(@params[:random_seed])
|
95
|
+
@penalty_type = L2_PENALTY
|
96
|
+
@loss_func = ::Rumale::LinearModel::Loss::MeanSquaredError.new
|
97
|
+
end
|
98
|
+
|
99
|
+
# Fit the model with given training data.
|
100
|
+
#
|
101
|
+
# @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
|
102
|
+
# @param y [Numo::DFloat] (shape: [n_samples, n_outputs]) The target values to be used for fitting the model.
|
103
|
+
# @return [Ridge] The learned regressor itself.
|
104
|
+
def fit(x, y)
|
105
|
+
x = ::Rumale::Validation.check_convert_sample_array(x)
|
106
|
+
y = ::Rumale::Validation.check_convert_target_value_array(y)
|
107
|
+
::Rumale::Validation.check_sample_size(x, y)
|
108
|
+
|
109
|
+
if @params[:solver] == 'svd' && enable_linalg?(warning: false)
|
110
|
+
fit_svd(x, y)
|
111
|
+
elsif @params[:solver] == 'lbfgs'
|
112
|
+
fit_lbfgs(x, y)
|
113
|
+
else
|
114
|
+
fit_sgd(x, y)
|
115
|
+
end
|
116
|
+
|
117
|
+
self
|
118
|
+
end
|
119
|
+
|
120
|
+
# Predict values for samples.
|
121
|
+
#
|
122
|
+
# @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the values.
|
123
|
+
# @return [Numo::DFloat] (shape: [n_samples, n_outputs]) Predicted values per sample.
|
124
|
+
def predict(x)
|
125
|
+
x = ::Rumale::Validation.check_convert_sample_array(x)
|
126
|
+
|
127
|
+
x.dot(@weight_vec.transpose) + @bias_term
|
128
|
+
end
|
129
|
+
|
130
|
+
private
|
131
|
+
|
132
|
+
def fit_svd(x, y)
|
133
|
+
x = expand_feature(x) if fit_bias?
|
134
|
+
|
135
|
+
s, u, vt = Numo::Linalg.svd(x, driver: 'sdd', job: 'S')
|
136
|
+
d = (s / (s**2 + @params[:reg_param])).diag
|
137
|
+
w = vt.transpose.dot(d).dot(u.transpose).dot(y)
|
138
|
+
|
139
|
+
@weight_vec, @bias_term = single_target?(y) ? split_weight(w) : split_weight_mult(w)
|
140
|
+
end
|
141
|
+
|
142
|
+
def fit_lbfgs(x, y)
|
143
|
+
fnc = proc do |w, x, y, a| # rubocop:disable Lint/ShadowingOuterLocalVariable
|
144
|
+
n_samples, n_features = x.shape
|
145
|
+
w = w.reshape(y.shape[1], n_features) unless y.shape[1].nil?
|
146
|
+
z = x.dot(w.transpose)
|
147
|
+
d = z - y
|
148
|
+
loss = (d**2).sum.fdiv(n_samples) + a * (w * w).sum
|
149
|
+
gradient = 2.fdiv(n_samples) * d.transpose.dot(x) + 2.0 * a * w
|
150
|
+
[loss, gradient.flatten.dup]
|
151
|
+
end
|
152
|
+
|
153
|
+
x = expand_feature(x) if fit_bias?
|
154
|
+
|
155
|
+
n_features = x.shape[1]
|
156
|
+
n_outputs = single_target?(y) ? 1 : y.shape[1]
|
157
|
+
|
158
|
+
res = Lbfgsb.minimize(
|
159
|
+
fnc: fnc, jcb: true, x_init: init_weight(n_features, n_outputs), args: [x, y, @params[:reg_param]],
|
160
|
+
maxiter: @params[:max_iter], factr: @params[:tol] / Lbfgsb::DBL_EPSILON,
|
161
|
+
verbose: @params[:verbose] ? 1 : -1
|
162
|
+
)
|
163
|
+
|
164
|
+
@weight_vec, @bias_term =
|
165
|
+
if single_target?(y)
|
166
|
+
split_weight(res[:x])
|
167
|
+
else
|
168
|
+
split_weight_mult(res[:x].reshape(n_outputs, n_features).transpose)
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
def fit_sgd(x, y)
|
173
|
+
if single_target?(y)
|
174
|
+
@weight_vec, @bias_term = partial_fit(x, y)
|
175
|
+
else
|
176
|
+
n_outputs = y.shape[1]
|
177
|
+
n_features = x.shape[1]
|
178
|
+
@weight_vec = Numo::DFloat.zeros(n_outputs, n_features)
|
179
|
+
@bias_term = Numo::DFloat.zeros(n_outputs)
|
180
|
+
if enable_parallel?
|
181
|
+
models = parallel_map(n_outputs) { |n| partial_fit(x, y[true, n]) }
|
182
|
+
n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = models[n] }
|
183
|
+
else
|
184
|
+
n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = partial_fit(x, y[true, n]) }
|
185
|
+
end
|
186
|
+
end
|
187
|
+
end
|
188
|
+
|
189
|
+
def single_target?(y)
|
190
|
+
y.ndim == 1
|
191
|
+
end
|
192
|
+
|
193
|
+
def init_weight(n_features, n_outputs)
|
194
|
+
::Rumale::Utils.rand_normal([n_outputs, n_features], @rng.dup).flatten.dup
|
195
|
+
end
|
196
|
+
|
197
|
+
def split_weight_mult(w)
|
198
|
+
if fit_bias?
|
199
|
+
[w[0...-1, true].dup, w[-1, true].dup]
|
200
|
+
else
|
201
|
+
[w.dup, Numo::DFloat.zeros(w.shape[1])]
|
202
|
+
end
|
203
|
+
end
|
204
|
+
end
|
205
|
+
end
|
206
|
+
end
|
@@ -0,0 +1,203 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'rumale/base/classifier'
|
4
|
+
require 'rumale/linear_model/base_sgd'
|
5
|
+
require 'rumale/probabilistic_output'
|
6
|
+
require 'rumale/validation'
|
7
|
+
|
8
|
+
module Rumale
|
9
|
+
# This module consists of the classes that implement generalized linear models.
|
10
|
+
module LinearModel
|
11
|
+
# SVC is a class that implements Support Vector Classifier
|
12
|
+
# with stochastic gradient descent optimization.
|
13
|
+
# For multiclass classification problem, it uses one-vs-the-rest strategy.
|
14
|
+
#
|
15
|
+
# @note
|
16
|
+
# Rumale::SVM provides linear support vector classifier based on LIBLINEAR.
|
17
|
+
# If you prefer execution speed, you should use Rumale::SVM::LinearSVC.
|
18
|
+
# https://github.com/yoshoku/rumale-svm
|
19
|
+
#
|
20
|
+
# @example
|
21
|
+
# require 'rumale/linear_model/svc'
|
22
|
+
#
|
23
|
+
# estimator =
|
24
|
+
# Rumale::LinearModel::SVC.new(reg_param: 1.0, max_iter: 1000, batch_size: 50, random_seed: 1)
|
25
|
+
# estimator.fit(training_samples, traininig_labels)
|
26
|
+
# results = estimator.predict(testing_samples)
|
27
|
+
#
|
28
|
+
# *Reference*
|
29
|
+
# - Shalev-Shwartz, S., and Singer, Y., "Pegasos: Primal Estimated sub-GrAdient SOlver for SVM," Proc. ICML'07, pp. 807--814, 2007.
|
30
|
+
# - Tsuruoka, Y., Tsujii, J., and Ananiadou, S., "Stochastic Gradient Descent Training for L1-regularized Log-linear Models with Cumulative Penalty," Proc. ACL'09, pp. 477--485, 2009.
|
31
|
+
# - Bottou, L., "Large-Scale Machine Learning with Stochastic Gradient Descent," Proc. COMPSTAT'10, pp. 177--186, 2010.
|
32
|
+
class SVC < BaseSGD
|
33
|
+
include ::Rumale::Base::Classifier
|
34
|
+
|
35
|
+
# Return the weight vector for SVC.
|
36
|
+
# @return [Numo::DFloat] (shape: [n_classes, n_features])
|
37
|
+
attr_reader :weight_vec
|
38
|
+
|
39
|
+
# Return the bias term (a.k.a. intercept) for SVC.
|
40
|
+
# @return [Numo::DFloat] (shape: [n_classes])
|
41
|
+
attr_reader :bias_term
|
42
|
+
|
43
|
+
# Return the class labels.
|
44
|
+
# @return [Numo::Int32] (shape: [n_classes])
|
45
|
+
attr_reader :classes
|
46
|
+
|
47
|
+
# Return the random generator for performing random sampling.
|
48
|
+
# @return [Random]
|
49
|
+
attr_reader :rng
|
50
|
+
|
51
|
+
# Create a new classifier with Support Vector Machine by the SGD optimization.
|
52
|
+
#
|
53
|
+
# @param learning_rate [Float] The initial value of learning rate.
|
54
|
+
# The learning rate decreases as the iteration proceeds according to the equation: learning_rate / (1 + decay * t).
|
55
|
+
# @param decay [Float] The smoothing parameter for decreasing learning rate as the iteration proceeds.
|
56
|
+
# If nil is given, the decay sets to 'reg_param * learning_rate'.
|
57
|
+
# @param momentum [Float] The momentum factor.
|
58
|
+
# @param penalty [String] The regularization type to be used ('l1', 'l2', and 'elasticnet').
|
59
|
+
# @param l1_ratio [Float] The elastic-net type regularization mixing parameter.
|
60
|
+
# If penalty set to 'l2' or 'l1', this parameter is ignored.
|
61
|
+
# If l1_ratio = 1, the regularization is similar to Lasso.
|
62
|
+
# If l1_ratio = 0, the regularization is similar to Ridge.
|
63
|
+
# If 0 < l1_ratio < 1, the regularization is a combination of L1 and L2.
|
64
|
+
# @param reg_param [Float] The regularization parameter.
|
65
|
+
# @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
|
66
|
+
# @param bias_scale [Float] The scale of the bias term.
|
67
|
+
# @param max_iter [Integer] The maximum number of epochs that indicates
|
68
|
+
# how many times the whole data is given to the training process.
|
69
|
+
# @param batch_size [Integer] The size of the mini batches.
|
70
|
+
# @param tol [Float] The tolerance of loss for terminating optimization.
|
71
|
+
# @param probability [Boolean] The flag indicating whether to perform probability estimation.
|
72
|
+
# @param n_jobs [Integer] The number of jobs for running the fit and predict methods in parallel.
|
73
|
+
# If nil is given, the methods do not execute in parallel.
|
74
|
+
# If zero or less is given, it becomes equal to the number of processors.
|
75
|
+
# This parameter is ignored if the Parallel gem is not loaded.
|
76
|
+
# @param verbose [Boolean] The flag indicating whether to output loss during iteration.
|
77
|
+
# @param random_seed [Integer] The seed value using to initialize the random generator.
|
78
|
+
def initialize(learning_rate: 0.01, decay: nil, momentum: 0.9,
|
79
|
+
penalty: 'l2', reg_param: 1.0, l1_ratio: 0.5,
|
80
|
+
fit_bias: true, bias_scale: 1.0,
|
81
|
+
max_iter: 1000, batch_size: 50, tol: 1e-4,
|
82
|
+
probability: false,
|
83
|
+
n_jobs: nil, verbose: false, random_seed: nil)
|
84
|
+
super()
|
85
|
+
@params.merge!(method(:initialize).parameters.to_h { |_t, arg| [arg, binding.local_variable_get(arg)] })
|
86
|
+
@params[:decay] ||= @params[:reg_param] * @params[:learning_rate]
|
87
|
+
@params[:random_seed] ||= srand
|
88
|
+
@rng = Random.new(@params[:random_seed])
|
89
|
+
@penalty_type = @params[:penalty]
|
90
|
+
@loss_func = ::Rumale::LinearModel::Loss::HingeLoss.new
|
91
|
+
end
|
92
|
+
|
93
|
+
# Fit the model with given training data.
|
94
|
+
#
|
95
|
+
# @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
|
96
|
+
# @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
|
97
|
+
# @return [SVC] The learned classifier itself.
|
98
|
+
def fit(x, y)
|
99
|
+
x = ::Rumale::Validation.check_convert_sample_array(x)
|
100
|
+
y = ::Rumale::Validation.check_convert_label_array(y)
|
101
|
+
::Rumale::Validation.check_sample_size(x, y)
|
102
|
+
|
103
|
+
@classes = Numo::Int32[*y.to_a.uniq.sort]
|
104
|
+
|
105
|
+
if multiclass_problem?
|
106
|
+
n_classes = @classes.size
|
107
|
+
n_features = x.shape[1]
|
108
|
+
# initialize model.
|
109
|
+
@weight_vec = Numo::DFloat.zeros(n_classes, n_features)
|
110
|
+
@bias_term = Numo::DFloat.zeros(n_classes)
|
111
|
+
@prob_param = Numo::DFloat.zeros(n_classes, 2)
|
112
|
+
# fit model.
|
113
|
+
models = if enable_parallel?
|
114
|
+
parallel_map(n_classes) do |n|
|
115
|
+
bin_y = Numo::Int32.cast(y.eq(@classes[n])) * 2 - 1
|
116
|
+
partial_fit(x, bin_y)
|
117
|
+
end
|
118
|
+
else
|
119
|
+
Array.new(n_classes) do |n|
|
120
|
+
bin_y = Numo::Int32.cast(y.eq(@classes[n])) * 2 - 1
|
121
|
+
partial_fit(x, bin_y)
|
122
|
+
end
|
123
|
+
end
|
124
|
+
# store model.
|
125
|
+
models.each_with_index { |model, n| @weight_vec[n, true], @bias_term[n], @prob_param[n, true] = model }
|
126
|
+
else
|
127
|
+
negative_label = @classes[0]
|
128
|
+
bin_y = Numo::Int32.cast(y.ne(negative_label)) * 2 - 1
|
129
|
+
@weight_vec, @bias_term, @prob_param = partial_fit(x, bin_y)
|
130
|
+
end
|
131
|
+
|
132
|
+
self
|
133
|
+
end
|
134
|
+
|
135
|
+
# Calculate confidence scores for samples.
|
136
|
+
#
|
137
|
+
# @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
|
138
|
+
# @return [Numo::DFloat] (shape: [n_samples, n_classes]) Confidence score per sample.
|
139
|
+
def decision_function(x)
|
140
|
+
x = ::Rumale::Validation.check_convert_sample_array(x)
|
141
|
+
|
142
|
+
x.dot(@weight_vec.transpose) + @bias_term
|
143
|
+
end
|
144
|
+
|
145
|
+
# Predict class labels for samples.
|
146
|
+
#
|
147
|
+
# @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
|
148
|
+
# @return [Numo::Int32] (shape: [n_samples]) Predicted class label per sample.
|
149
|
+
def predict(x)
|
150
|
+
x = ::Rumale::Validation.check_convert_sample_array(x)
|
151
|
+
|
152
|
+
n_samples = x.shape[0]
|
153
|
+
predicted = if multiclass_problem?
|
154
|
+
decision_values = decision_function(x)
|
155
|
+
if enable_parallel?
|
156
|
+
parallel_map(n_samples) { |n| @classes[decision_values[n, true].max_index] }
|
157
|
+
else
|
158
|
+
Array.new(n_samples) { |n| @classes[decision_values[n, true].max_index] }
|
159
|
+
end
|
160
|
+
else
|
161
|
+
decision_values = decision_function(x).ge(0.0).to_a
|
162
|
+
Array.new(n_samples) { |n| @classes[decision_values[n]] }
|
163
|
+
end
|
164
|
+
Numo::Int32.asarray(predicted)
|
165
|
+
end
|
166
|
+
|
167
|
+
# Predict probability for samples.
|
168
|
+
#
|
169
|
+
# @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probailities.
|
170
|
+
# @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
|
171
|
+
def predict_proba(x)
|
172
|
+
x = ::Rumale::Validation.check_convert_sample_array(x)
|
173
|
+
|
174
|
+
if multiclass_problem?
|
175
|
+
probs = 1.0 / (Numo::NMath.exp(@prob_param[true, 0] * decision_function(x) + @prob_param[true, 1]) + 1.0)
|
176
|
+
(probs.transpose / probs.sum(axis: 1)).transpose.dup
|
177
|
+
else
|
178
|
+
n_samples, = x.shape
|
179
|
+
probs = Numo::DFloat.zeros(n_samples, 2)
|
180
|
+
probs[true, 1] = 1.0 / (Numo::NMath.exp(@prob_param[0] * decision_function(x) + @prob_param[1]) + 1.0)
|
181
|
+
probs[true, 0] = 1.0 - probs[true, 1]
|
182
|
+
probs
|
183
|
+
end
|
184
|
+
end
|
185
|
+
|
186
|
+
private
|
187
|
+
|
188
|
+
def partial_fit(x, bin_y)
|
189
|
+
w, b = super
|
190
|
+
p = if @params[:probability]
|
191
|
+
::Rumale::ProbabilisticOutput.fit_sigmoid(x.dot(w.transpose) + b, bin_y)
|
192
|
+
else
|
193
|
+
Numo::DFloat[1, 0]
|
194
|
+
end
|
195
|
+
[w, b, p]
|
196
|
+
end
|
197
|
+
|
198
|
+
def multiclass_problem?
|
199
|
+
@classes.size > 2
|
200
|
+
end
|
201
|
+
end
|
202
|
+
end
|
203
|
+
end
|
@@ -0,0 +1,126 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'rumale/base/regressor'
|
4
|
+
require 'rumale/validation'
|
5
|
+
require 'rumale/linear_model/base_sgd'
|
6
|
+
|
7
|
+
module Rumale
|
8
|
+
module LinearModel
|
9
|
+
# SVR is a class that implements Support Vector Regressor
|
10
|
+
# with stochastic gradient descent optimization.
|
11
|
+
#
|
12
|
+
# @note
|
13
|
+
# Rumale::SVM provides linear and kernel support vector regressor based on LIBLINEAR and LIBSVM.
|
14
|
+
# If you prefer execution speed, you should use Rumale::SVM::LinearSVR.
|
15
|
+
# https://github.com/yoshoku/rumale-svm
|
16
|
+
#
|
17
|
+
# @example
|
18
|
+
# require 'rumale/linear_model/svr'
|
19
|
+
#
|
20
|
+
# estimator =
|
21
|
+
# Rumale::LinearModel::SVR.new(reg_param: 1.0, epsilon: 0.1, max_iter: 1000, batch_size: 50, random_seed: 1)
|
22
|
+
# estimator.fit(training_samples, traininig_target_values)
|
23
|
+
# results = estimator.predict(testing_samples)
|
24
|
+
#
|
25
|
+
# *Reference*
|
26
|
+
# - Shalev-Shwartz, S., and Singer, Y., "Pegasos: Primal Estimated sub-GrAdient SOlver for SVM," Proc. ICML'07, pp. 807--814, 2007.
|
27
|
+
# - Tsuruoka, Y., Tsujii, J., and Ananiadou, S., "Stochastic Gradient Descent Training for L1-regularized Log-linear Models with Cumulative Penalty," Proc. ACL'09, pp. 477--485, 2009.
|
28
|
+
# - Bottou, L., "Large-Scale Machine Learning with Stochastic Gradient Descent," Proc. COMPSTAT'10, pp. 177--186, 2010.
|
29
|
+
class SVR < BaseSGD
|
30
|
+
include ::Rumale::Base::Regressor
|
31
|
+
|
32
|
+
# Return the weight vector for SVR.
|
33
|
+
# @return [Numo::DFloat] (shape: [n_outputs, n_features])
|
34
|
+
attr_reader :weight_vec
|
35
|
+
|
36
|
+
# Return the bias term (a.k.a. intercept) for SVR.
|
37
|
+
# @return [Numo::DFloat] (shape: [n_outputs])
|
38
|
+
attr_reader :bias_term
|
39
|
+
|
40
|
+
# Return the random generator for performing random sampling.
|
41
|
+
# @return [Random]
|
42
|
+
attr_reader :rng
|
43
|
+
|
44
|
+
# Create a new regressor with Support Vector Machine by the SGD optimization.
|
45
|
+
#
|
46
|
+
# @param learning_rate [Float] The initial value of learning rate.
|
47
|
+
# The learning rate decreases as the iteration proceeds according to the equation: learning_rate / (1 + decay * t).
|
48
|
+
# @param decay [Float] The smoothing parameter for decreasing learning rate as the iteration proceeds.
|
49
|
+
# If nil is given, the decay sets to 'reg_param * learning_rate'.
|
50
|
+
# @param momentum [Float] The momentum factor.
|
51
|
+
# @param penalty [String] The regularization type to be used ('l1', 'l2', and 'elasticnet').
|
52
|
+
# @param l1_ratio [Float] The elastic-net type regularization mixing parameter.
|
53
|
+
# If penalty set to 'l2' or 'l1', this parameter is ignored.
|
54
|
+
# If l1_ratio = 1, the regularization is similar to Lasso.
|
55
|
+
# If l1_ratio = 0, the regularization is similar to Ridge.
|
56
|
+
# If 0 < l1_ratio < 1, the regularization is a combination of L1 and L2.
|
57
|
+
# @param reg_param [Float] The regularization parameter.
|
58
|
+
# @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
|
59
|
+
# @param bias_scale [Float] The scale of the bias term.
|
60
|
+
# @param epsilon [Float] The margin of tolerance.
|
61
|
+
# @param max_iter [Integer] The maximum number of epochs that indicates
|
62
|
+
# how many times the whole data is given to the training process.
|
63
|
+
# @param batch_size [Integer] The size of the mini batches.
|
64
|
+
# @param tol [Float] The tolerance of loss for terminating optimization.
|
65
|
+
# @param n_jobs [Integer] The number of jobs for running the fit method in parallel.
|
66
|
+
# If nil is given, the method does not execute in parallel.
|
67
|
+
# If zero or less is given, it becomes equal to the number of processors.
|
68
|
+
# This parameter is ignored if the Parallel gem is not loaded.
|
69
|
+
# @param verbose [Boolean] The flag indicating whether to output loss during iteration.
|
70
|
+
# @param random_seed [Integer] The seed value using to initialize the random generator.
|
71
|
+
def initialize(learning_rate: 0.01, decay: nil, momentum: 0.9,
|
72
|
+
penalty: 'l2', reg_param: 1.0, l1_ratio: 0.5,
|
73
|
+
fit_bias: true, bias_scale: 1.0,
|
74
|
+
epsilon: 0.1,
|
75
|
+
max_iter: 1000, batch_size: 50, tol: 1e-4,
|
76
|
+
n_jobs: nil, verbose: false, random_seed: nil)
|
77
|
+
super()
|
78
|
+
@params.merge!(method(:initialize).parameters.to_h { |_t, arg| [arg, binding.local_variable_get(arg)] })
|
79
|
+
@params[:decay] ||= @params[:reg_param] * @params[:learning_rate]
|
80
|
+
@params[:random_seed] ||= srand
|
81
|
+
@rng = Random.new(@params[:random_seed])
|
82
|
+
@penalty_type = @params[:penalty]
|
83
|
+
@loss_func = ::Rumale::LinearModel::Loss::EpsilonInsensitive.new(epsilon: @params[:epsilon])
|
84
|
+
end
|
85
|
+
|
86
|
+
# Fit the model with given training data.
|
87
|
+
#
|
88
|
+
# @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
|
89
|
+
# @param y [Numo::DFloat] (shape: [n_samples, n_outputs]) The target values to be used for fitting the model.
|
90
|
+
# @return [SVR] The learned regressor itself.
|
91
|
+
def fit(x, y)
|
92
|
+
x = ::Rumale::Validation.check_convert_sample_array(x)
|
93
|
+
y = ::Rumale::Validation.check_convert_target_value_array(y)
|
94
|
+
::Rumale::Validation.check_sample_size(x, y)
|
95
|
+
|
96
|
+
n_outputs = y.shape[1].nil? ? 1 : y.shape[1]
|
97
|
+
n_features = x.shape[1]
|
98
|
+
|
99
|
+
if n_outputs > 1
|
100
|
+
@weight_vec = Numo::DFloat.zeros(n_outputs, n_features)
|
101
|
+
@bias_term = Numo::DFloat.zeros(n_outputs)
|
102
|
+
if enable_parallel?
|
103
|
+
models = parallel_map(n_outputs) { |n| partial_fit(x, y[true, n]) }
|
104
|
+
n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = models[n] }
|
105
|
+
else
|
106
|
+
n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = partial_fit(x, y[true, n]) }
|
107
|
+
end
|
108
|
+
else
|
109
|
+
@weight_vec, @bias_term = partial_fit(x, y)
|
110
|
+
end
|
111
|
+
|
112
|
+
self
|
113
|
+
end
|
114
|
+
|
115
|
+
# Predict values for samples.
|
116
|
+
#
|
117
|
+
# @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the values.
|
118
|
+
# @return [Numo::DFloat] (shape: [n_samples, n_outputs]) Predicted values per sample.
|
119
|
+
def predict(x)
|
120
|
+
x = ::Rumale::Validation.check_convert_sample_array(x)
|
121
|
+
|
122
|
+
x.dot(@weight_vec.transpose) + @bias_term
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'numo/narray'
|
4
|
+
|
5
|
+
require_relative 'linear_model/base_sgd'
|
6
|
+
require_relative 'linear_model/elastic_net'
|
7
|
+
require_relative 'linear_model/lasso'
|
8
|
+
require_relative 'linear_model/linear_regression'
|
9
|
+
require_relative 'linear_model/logistic_regression'
|
10
|
+
require_relative 'linear_model/nnls'
|
11
|
+
require_relative 'linear_model/ridge'
|
12
|
+
require_relative 'linear_model/svc'
|
13
|
+
require_relative 'linear_model/svr'
|
14
|
+
require_relative 'linear_model/version'
|
metadata
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: rumale-linear_model
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.24.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- yoshoku
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2022-12-31 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: lbfgsb
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.3.0
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.3.0
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: numo-narray
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.9.1
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 0.9.1
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rumale-core
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.24.0
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.24.0
|
55
|
+
description: |
|
56
|
+
Rumale::LinearModel provides linear model algorithms,
|
57
|
+
such as Logistic Regression, Support Vector Machine, Lasso, and Ridge Regression
|
58
|
+
with Rumale interface.
|
59
|
+
email:
|
60
|
+
- yoshoku@outlook.com
|
61
|
+
executables: []
|
62
|
+
extensions: []
|
63
|
+
extra_rdoc_files: []
|
64
|
+
files:
|
65
|
+
- LICENSE.txt
|
66
|
+
- README.md
|
67
|
+
- lib/rumale/linear_model.rb
|
68
|
+
- lib/rumale/linear_model/base_sgd.rb
|
69
|
+
- lib/rumale/linear_model/elastic_net.rb
|
70
|
+
- lib/rumale/linear_model/lasso.rb
|
71
|
+
- lib/rumale/linear_model/linear_regression.rb
|
72
|
+
- lib/rumale/linear_model/logistic_regression.rb
|
73
|
+
- lib/rumale/linear_model/nnls.rb
|
74
|
+
- lib/rumale/linear_model/ridge.rb
|
75
|
+
- lib/rumale/linear_model/svc.rb
|
76
|
+
- lib/rumale/linear_model/svr.rb
|
77
|
+
- lib/rumale/linear_model/version.rb
|
78
|
+
homepage: https://github.com/yoshoku/rumale
|
79
|
+
licenses:
|
80
|
+
- BSD-3-Clause
|
81
|
+
metadata:
|
82
|
+
homepage_uri: https://github.com/yoshoku/rumale
|
83
|
+
source_code_uri: https://github.com/yoshoku/rumale/tree/main/rumale-linear_model
|
84
|
+
changelog_uri: https://github.com/yoshoku/rumale/blob/main/CHANGELOG.md
|
85
|
+
documentation_uri: https://yoshoku.github.io/rumale/doc/
|
86
|
+
rubygems_mfa_required: 'true'
|
87
|
+
post_install_message:
|
88
|
+
rdoc_options: []
|
89
|
+
require_paths:
|
90
|
+
- lib
|
91
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
92
|
+
requirements:
|
93
|
+
- - ">="
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: '0'
|
96
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
97
|
+
requirements:
|
98
|
+
- - ">="
|
99
|
+
- !ruby/object:Gem::Version
|
100
|
+
version: '0'
|
101
|
+
requirements: []
|
102
|
+
rubygems_version: 3.3.26
|
103
|
+
signing_key:
|
104
|
+
specification_version: 4
|
105
|
+
summary: Rumale::LinearModel provides linear model algorithms with Rumale interface.
|
106
|
+
test_files: []
|