nn 1.8 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/document.txt +23 -84
- data/lib/nn.rb +31 -36
- data/nn.gemspec +1 -1
- data/nn.rb +441 -0
- data/sample/cifar10_program.rb +38 -0
- data/sample/mnist_program.rb +38 -0
- data/sample/xor.rb +24 -0
- metadata +6 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8f77c817ea492d035851bf8552ad2a97928f6762acb455ae23de0e3ee8f40871
|
4
|
+
data.tar.gz: 1f162719087671733c8afd5279bca59859474dd366677acbcf79032a9fff5eba
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 492e639590f4b81083a669f51ee192cb9a758ee0bbe950539c74367322ab78a9ad77e6075f90e56f28eecbc57ba42df91455c88e109fe2ec5565ceb77730bafc
|
7
|
+
data.tar.gz: d8745f38ed5ca0d75da462c6a8cf1233ea8e9c69e10918b87f37c23c8344690075bba1f6c13cf16267f51a92993e1fe58c0297251c309adcbbd3c4c85f339221
|
data/README.md
CHANGED
data/document.txt
CHANGED
@@ -13,6 +13,11 @@ class NN
|
|
13
13
|
|
14
14
|
<クラスメソッド>
|
15
15
|
load(file_name) : NN
|
16
|
+
Marshal形式で保存された学習結果を読み込みます。
|
17
|
+
String file_name 読み込むMarshalファイル名
|
18
|
+
戻り値 NNのインスタンス
|
19
|
+
|
20
|
+
load_json(file_name) : NN
|
16
21
|
JSON形式で保存された学習結果を読み込みます。
|
17
22
|
String file_name 読み込むJSONファイル名
|
18
23
|
戻り値 NNのインスタンス
|
@@ -52,14 +57,7 @@ initialize(num_nodes,
|
|
52
57
|
Float dropout_ratio ドロップアウトさせるノードの比率
|
53
58
|
bool use_batch_norm バッチノーマライゼーションを使用するか否か
|
54
59
|
|
55
|
-
train(x_train, y_train, x_test, y_test, epochs,
|
56
|
-
learning_rate_decay: 0,
|
57
|
-
save_dir: nil,
|
58
|
-
save_interval: 1,
|
59
|
-
test: nil,
|
60
|
-
border: nil,
|
61
|
-
tolerance: 0.5,
|
62
|
-
&block) : void
|
60
|
+
train(x_train, y_train, x_test, y_test, epochs, func = nil, &block) : void
|
63
61
|
学習を行います。
|
64
62
|
Array<Array<Numeric>> | SFloat x_train トレーニング用入力データ。
|
65
63
|
Array<Array<Numeric>> | SFloat y_train トレーニング用正解データ。
|
@@ -71,8 +69,9 @@ train(x_train, y_train, x_test, y_test, epochs,
|
|
71
69
|
nilを指定すると、エポックごとにテストを行いません。
|
72
70
|
Float border 学習の早期終了判定に使用するテストデータの正答率。
|
73
71
|
nilの場合、学習の早期終了を行いません。
|
74
|
-
Proc
|
75
|
-
|
72
|
+
Proc func(SFloat x, SFloat y) : Array<SFloat> 入力層のミニバッチを取得します。ブロックの戻り値は、ミニバッチを[x, y]の
|
73
|
+
形で指定してください。入力層をミニバッチ単位で正規化したい場合に使用します。
|
74
|
+
Proc block(Integer epoch) : void 1エポックの学習が終わった後で行いたい処理を、ブロックで渡します。
|
76
75
|
|
77
76
|
test(x_test, y_test, tolerance = 0.5, &block) : Float
|
78
77
|
テストデータを用いて、テストを行います。
|
@@ -95,7 +94,7 @@ accurate(x_test, y_test, tolera)
|
|
95
94
|
戻り値 テストデータの正答率。
|
96
95
|
|
97
96
|
learn(x_train, y_train, &block) : Float
|
98
|
-
入力データを元に、1
|
97
|
+
入力データを元に、1回だけ学習を行います。柔軟な学習を行いたい場合に使用します。
|
99
98
|
Array<Array<Numeric>> | SFloat x_train 入力データ
|
100
99
|
Array<Array<Numeric>> | SFloat y_train 正解データ
|
101
100
|
Proc &block(SFloat x, SFloat y) : Array<SFloat> 入力層のミニバッチを取得します。ブロックの戻り値は、ミニバッチを[x, y]の
|
@@ -106,42 +105,23 @@ learn(x_train, y_train, &block) : Float
|
|
106
105
|
|
107
106
|
run(x) : Array<Array<Numeric>>
|
108
107
|
入力データから出力値を二次元配列で得ます。
|
109
|
-
Array<Array<Float>>
|
108
|
+
Array<Array<Float>> x 入力データ
|
109
|
+
戻り値 出力ノードの値
|
110
|
+
|
111
|
+
run(x) : SFloat
|
112
|
+
入力データから出力値をSFloat形式で得ます。
|
113
|
+
SFloat x 入力データ
|
110
114
|
戻り値 出力ノードの値
|
111
115
|
|
112
116
|
save(file_name) : void
|
117
|
+
学習結果をMarshal形式で保存します。
|
118
|
+
String file_name 書き込むMarshalファイル名
|
119
|
+
|
120
|
+
save_json(file_name) : void
|
113
121
|
学習結果をJSON形式で保存します。
|
114
122
|
String file_name 書き込むJSONファイル名
|
115
123
|
|
116
124
|
|
117
|
-
[サンプル1 XOR]
|
118
|
-
|
119
|
-
#ライブラリの読み込み
|
120
|
-
require "nn"
|
121
|
-
|
122
|
-
x = [
|
123
|
-
[0, 0],
|
124
|
-
[1, 0],
|
125
|
-
[0, 1],
|
126
|
-
[1, 1],
|
127
|
-
]
|
128
|
-
|
129
|
-
y = [[0], [1], [1], [0]]
|
130
|
-
|
131
|
-
#ニューラルネットワークの初期化
|
132
|
-
nn = NN.new([2, 4, 1], #ノード数
|
133
|
-
learning_rate: 0.1, #学習率
|
134
|
-
batch_size: 4, #ミニバッチの数
|
135
|
-
activation: [:sigmoid, :identity] #活性化関数
|
136
|
-
)
|
137
|
-
|
138
|
-
#学習を行う
|
139
|
-
nn.train(x, y, 20000)
|
140
|
-
|
141
|
-
#学習結果の確認
|
142
|
-
p nn.run(x)
|
143
|
-
|
144
|
-
|
145
125
|
[MNISTデータを読み込む]
|
146
126
|
MNISTをRubyでも簡単に試せるよう、MNISTを扱うためのモジュールを用意しました。
|
147
127
|
次のリンク(http://yann.lecun.com/exdb/mnist/)から、
|
@@ -156,54 +136,10 @@ MNIST.load_trainで学習用データを読み込み、MNIST.load_testでテス
|
|
156
136
|
(RubyでのMNISTの読み込みは、以下のリンクを参考にさせていただきました。)
|
157
137
|
http://d.hatena.ne.jp/n_shuyo/20090913/mnist
|
158
138
|
|
159
|
-
|
160
|
-
[サンプル2 MNIST]
|
161
|
-
|
162
|
-
#ライブラリの読み込み
|
163
|
-
require "nn"
|
164
|
-
require "nn/mnist"
|
165
|
-
|
166
|
-
#MNISTのトレーニング用データを読み込む
|
167
|
-
x_train, y_train = MNIST.load_train
|
168
|
-
|
169
|
-
#y_trainを10クラスに配列でカテゴライズする
|
170
|
-
y_train = MNIST.categorical(y_train)
|
171
|
-
|
172
|
-
#MNISTのテスト用データを読み込む
|
173
|
-
x_test, y_test = MNIST.load_test
|
174
|
-
|
175
|
-
#y_testを10クラスにカテゴライズする
|
176
|
-
y_test = MNIST.categorical(y_test)
|
177
|
-
|
178
|
-
puts "load mnist"
|
179
|
-
|
180
|
-
#ニューラルネットワークの初期化
|
181
|
-
nn = NN.new([784, 100, 100, 10], #ノード数
|
182
|
-
learning_rate: 0.1, #学習率
|
183
|
-
batch_size: 100, #ミニバッチの数
|
184
|
-
activation: [:relu, :softmax], #活性化関数
|
185
|
-
momentum: 0.9, #モーメンタム係数
|
186
|
-
use_batch_norm: true, #バッチノーマライゼーションを使用する
|
187
|
-
)
|
188
|
-
|
189
|
-
#学習を行う
|
190
|
-
nn.train(x_train, y_train, 10, test: [x_test, y_test]) do |x_batch, y_batch|
|
191
|
-
x_batch /= 255 #ミニバッチを0~1の範囲で正規化
|
192
|
-
[x_batch, y_batch]
|
193
|
-
end
|
194
|
-
|
195
|
-
#学習結果のテストを行う
|
196
|
-
nn.test(x_test, y_test) do |x_batch, y_batch|
|
197
|
-
x_batch /= 255 #ミニバッチを0~1の範囲で正規化
|
198
|
-
[x_batch, y_batch]
|
199
|
-
end
|
200
|
-
|
201
|
-
|
202
139
|
[お断り]
|
203
140
|
作者は、ニューラルネットワークを勉強し始めたばかりの初心者です。
|
204
141
|
そのため、バグや実装のミスもあるかと思いますが、温かい目で見守っていただけると、幸いでございます。
|
205
142
|
|
206
|
-
|
207
143
|
[更新履歴]
|
208
144
|
2018/3/8 バージョン1.0公開
|
209
145
|
2018/3/11 バージョン1.1公開
|
@@ -213,3 +149,6 @@ end
|
|
213
149
|
2018/3/22 バージョン1.5公開
|
214
150
|
2018/4/15 バージョン1.6公開
|
215
151
|
2018/5/4 バージョン1.8公開
|
152
|
+
2018/5/16 バージョン2.0公開
|
153
|
+
2018/6/10 バージョン2.0.1公開
|
154
|
+
2018/6/10 バージョン2.1.0公開
|
data/lib/nn.rb
CHANGED
@@ -2,7 +2,7 @@ require "numo/narray"
|
|
2
2
|
require "json"
|
3
3
|
|
4
4
|
class NN
|
5
|
-
VERSION = "
|
5
|
+
VERSION = "2.0"
|
6
6
|
|
7
7
|
include Numo
|
8
8
|
|
@@ -64,37 +64,19 @@ class NN
|
|
64
64
|
nn
|
65
65
|
end
|
66
66
|
|
67
|
-
def train(x_train, y_train, epochs,
|
68
|
-
learning_rate_decay: 0,
|
69
|
-
save_dir: nil,
|
70
|
-
save_interval: 1,
|
71
|
-
test: nil,
|
72
|
-
border: nil,
|
73
|
-
tolerance: 0.5,
|
74
|
-
&block)
|
67
|
+
def train(x_train, y_train, epochs, func = nil, &block)
|
75
68
|
num_train_data = x_train.is_a?(SFloat) ? x_train.shape[0] : x_train.length
|
76
69
|
(1..epochs).each do |epoch|
|
77
70
|
loss = nil
|
78
71
|
(num_train_data.to_f / @batch_size).ceil.times do
|
79
|
-
loss = learn(x_train, y_train, &
|
72
|
+
loss = learn(x_train, y_train, &func)
|
80
73
|
if loss.nan?
|
81
74
|
puts "loss is nan"
|
82
75
|
return
|
83
76
|
end
|
84
77
|
end
|
85
|
-
|
86
|
-
|
87
|
-
end
|
88
|
-
msg = "epoch #{epoch}/#{epochs} loss: #{loss}"
|
89
|
-
if test
|
90
|
-
acc = accurate(*test, tolerance, &block)
|
91
|
-
puts "#{msg} accurate: #{acc}"
|
92
|
-
break if border && acc >= border
|
93
|
-
else
|
94
|
-
puts msg
|
95
|
-
end
|
96
|
-
@learning_rate -= learning_rate_decay
|
97
|
-
@learning_rate = 1e-7 if @learning_rate < 1e-7
|
78
|
+
puts "epoch #{epoch}/#{epochs} loss: #{loss}"
|
79
|
+
block.call(epoch) if block
|
98
80
|
end
|
99
81
|
end
|
100
82
|
|
@@ -160,9 +142,11 @@ class NN
|
|
160
142
|
end
|
161
143
|
|
162
144
|
def run(x)
|
163
|
-
|
164
|
-
|
165
|
-
|
145
|
+
if x.is_a?(Array)
|
146
|
+
forward(SFloat.cast(x), false).to_a
|
147
|
+
else
|
148
|
+
forward(x, false)
|
149
|
+
end
|
166
150
|
end
|
167
151
|
|
168
152
|
def save(file_name)
|
@@ -257,22 +241,30 @@ class NN
|
|
257
241
|
def update_weight_and_bias
|
258
242
|
@layers.select{|layer| layer.is_a?(Affine)}.each.with_index do |layer, i|
|
259
243
|
weight_amount = layer.d_weight.mean(0) * @learning_rate
|
260
|
-
@weight_amounts[i] = weight_amount + @momentum * @weight_amounts[i]
|
261
|
-
@weights[i] -= @weight_amounts[i]
|
262
244
|
bias_amount = layer.d_bias.mean * @learning_rate
|
263
|
-
|
264
|
-
|
245
|
+
if @momentum > 0
|
246
|
+
weight_amount += @momentum * @weight_amounts[i]
|
247
|
+
@weight_amounts[i] = weight_amount
|
248
|
+
bias_amount += @momentum * @bias_amounts[i]
|
249
|
+
@bias_amounts[i] = bias_amount
|
250
|
+
end
|
251
|
+
@weights[i] -= weight_amount
|
252
|
+
@biases[i] -= bias_amount
|
265
253
|
end
|
266
254
|
end
|
267
255
|
|
268
256
|
def update_gamma_and_beta
|
269
257
|
@layers.select{|layer| layer.is_a?(BatchNorm)}.each.with_index do |layer, i|
|
270
258
|
gamma_amount = layer.d_gamma.mean * @learning_rate
|
271
|
-
@gamma_amounts[i] = gamma_amount + @momentum * @gamma_amounts[i]
|
272
|
-
@gammas[i] -= @gamma_amounts[i]
|
273
259
|
beta_amount = layer.d_beta.mean * @learning_rate
|
274
|
-
|
275
|
-
|
260
|
+
if @momentum > 0
|
261
|
+
gamma_amount += @momentum * @gamma_amounts[i]
|
262
|
+
@gamma_amounts[i] = gamma_amount
|
263
|
+
beta_amount += @momentum * @beta_amounts[i]
|
264
|
+
@beta_amounts[i] = beta_amount
|
265
|
+
end
|
266
|
+
@gammas[i] -= gamma_amount
|
267
|
+
@betas[i] -= gamma_amount
|
276
268
|
end
|
277
269
|
end
|
278
270
|
end
|
@@ -298,8 +290,11 @@ class NN::Affine
|
|
298
290
|
|
299
291
|
def backward(dout)
|
300
292
|
x = @x.reshape(*@x.shape, 1)
|
301
|
-
|
302
|
-
@
|
293
|
+
@d_weight = x.dot(dout.reshape(dout.shape[0], 1, dout.shape[1]))
|
294
|
+
if @nn.weight_decay > 0
|
295
|
+
dridge = @nn.weight_decay * @nn.weights[@index]
|
296
|
+
@d_weight += dridge
|
297
|
+
end
|
303
298
|
@d_bias = dout
|
304
299
|
dout.dot(@nn.weights[@index].transpose)
|
305
300
|
end
|
data/nn.gemspec
CHANGED
data/nn.rb
ADDED
@@ -0,0 +1,441 @@
|
|
1
|
+
require "numo/narray"
|
2
|
+
require "json"
|
3
|
+
|
4
|
+
class NN
|
5
|
+
VERSION = "2.1"
|
6
|
+
|
7
|
+
include Numo
|
8
|
+
|
9
|
+
attr_accessor :weights
|
10
|
+
attr_accessor :biases
|
11
|
+
attr_accessor :gammas
|
12
|
+
attr_accessor :betas
|
13
|
+
attr_accessor :learning_rate
|
14
|
+
attr_accessor :batch_size
|
15
|
+
attr_accessor :activation
|
16
|
+
attr_accessor :momentum
|
17
|
+
attr_accessor :weight_decay
|
18
|
+
attr_accessor :dropout_ratio
|
19
|
+
attr_reader :training
|
20
|
+
|
21
|
+
def initialize(num_nodes,
|
22
|
+
learning_rate: 0.01,
|
23
|
+
batch_size: 1,
|
24
|
+
activation: %i(relu identity),
|
25
|
+
momentum: 0,
|
26
|
+
weight_decay: 0,
|
27
|
+
use_dropout: false,
|
28
|
+
dropout_ratio: 0.5,
|
29
|
+
use_batch_norm: false)
|
30
|
+
SFloat.srand(rand(2 ** 64))
|
31
|
+
@num_nodes = num_nodes
|
32
|
+
@learning_rate = learning_rate
|
33
|
+
@batch_size = batch_size
|
34
|
+
@activation = activation
|
35
|
+
@momentum = momentum
|
36
|
+
@weight_decay = weight_decay
|
37
|
+
@use_dropout = use_dropout
|
38
|
+
@dropout_ratio = dropout_ratio
|
39
|
+
@use_batch_norm = use_batch_norm
|
40
|
+
init_weight_and_bias
|
41
|
+
init_gamma_and_beta if @use_batch_norm
|
42
|
+
@training = true
|
43
|
+
init_layers
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.load(file_name)
|
47
|
+
Marshal.load(File.binread(file_name))
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.load_json(file_name)
|
51
|
+
json = JSON.parse(File.read(file_name))
|
52
|
+
nn = self.new(json["num_nodes"],
|
53
|
+
learning_rate: json["learning_rate"],
|
54
|
+
batch_size: json["batch_size"],
|
55
|
+
activation: json["activation"].map(&:to_sym),
|
56
|
+
momentum: json["momentum"],
|
57
|
+
weight_decay: json["weight_decay"],
|
58
|
+
use_dropout: json["use_dropout"],
|
59
|
+
dropout_ratio: json["dropout_ratio"],
|
60
|
+
use_batch_norm: json["use_batch_norm"],
|
61
|
+
)
|
62
|
+
nn.weights = json["weights"].map{|weight| SFloat.cast(weight)}
|
63
|
+
nn.biases = json["biases"].map{|bias| SFloat.cast(bias)}
|
64
|
+
if json["use_batch_norm"]
|
65
|
+
nn.gammas = json["gammas"].map{|gamma| SFloat.cast(gamma)}
|
66
|
+
nn.betas = json["betas"].map{|beta| SFloat.cast(beta)}
|
67
|
+
end
|
68
|
+
nn
|
69
|
+
end
|
70
|
+
|
71
|
+
def train(x_train, y_train, epochs, func = nil, &block)
|
72
|
+
num_train_data = x_train.is_a?(SFloat) ? x_train.shape[0] : x_train.length
|
73
|
+
(1..epochs).each do |epoch|
|
74
|
+
loss = nil
|
75
|
+
(num_train_data.to_f / @batch_size).ceil.times do
|
76
|
+
loss = learn(x_train, y_train, &func)
|
77
|
+
if loss.nan?
|
78
|
+
puts "loss is nan"
|
79
|
+
return
|
80
|
+
end
|
81
|
+
end
|
82
|
+
puts "epoch #{epoch}/#{epochs} loss: #{loss}"
|
83
|
+
block.call(epoch) if block
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def test(x_test, y_test, tolerance = 0.5, &block)
|
88
|
+
acc = accurate(x_test, y_test, tolerance, &block)
|
89
|
+
puts "accurate: #{acc}"
|
90
|
+
acc
|
91
|
+
end
|
92
|
+
|
93
|
+
def accurate(x_test, y_test, tolerance = 0.5, &block)
|
94
|
+
correct = 0
|
95
|
+
num_test_data = x_test.is_a?(SFloat) ? x_test.shape[0] : x_test.length
|
96
|
+
(num_test_data.to_f / @batch_size).ceil.times do |i|
|
97
|
+
x = SFloat.zeros(@batch_size, @num_nodes.first)
|
98
|
+
y = SFloat.zeros(@batch_size, @num_nodes.last)
|
99
|
+
@batch_size.times do |j|
|
100
|
+
k = i * @batch_size + j
|
101
|
+
break if k >= num_test_data
|
102
|
+
if x_test.is_a?(SFloat)
|
103
|
+
x[j, true] = x_test[k, true]
|
104
|
+
y[j, true] = y_test[k, true]
|
105
|
+
else
|
106
|
+
x[j, true] = SFloat.cast(x_test[k])
|
107
|
+
y[j, true] = SFloat.cast(y_test[k])
|
108
|
+
end
|
109
|
+
end
|
110
|
+
x, y = block.call(x, y) if block
|
111
|
+
out = forward(x, false)
|
112
|
+
@batch_size.times do |j|
|
113
|
+
vout = out[j, true]
|
114
|
+
vy = y[j, true]
|
115
|
+
case @activation[1]
|
116
|
+
when :identity
|
117
|
+
correct += 1 unless (NMath.sqrt((vout - vy) ** 2) < tolerance).to_a.include?(0)
|
118
|
+
when :softmax
|
119
|
+
correct += 1 if vout.max_index == vy.max_index
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
correct.to_f / num_test_data
|
124
|
+
end
|
125
|
+
|
126
|
+
def learn(x_train, y_train, &block)
|
127
|
+
x = SFloat.zeros(@batch_size, @num_nodes.first)
|
128
|
+
y = SFloat.zeros(@batch_size, @num_nodes.last)
|
129
|
+
@batch_size.times do |i|
|
130
|
+
if x_train.is_a?(SFloat)
|
131
|
+
r = rand(x_train.shape[0])
|
132
|
+
x[i, true] = x_train[r, true]
|
133
|
+
y[i, true] = y_train[r, true]
|
134
|
+
else
|
135
|
+
r = rand(x_train.length)
|
136
|
+
x[i, true] = SFloat.cast(x_train[r])
|
137
|
+
y[i, true] = SFloat.cast(y_train[r])
|
138
|
+
end
|
139
|
+
end
|
140
|
+
x, y = block.call(x, y) if block
|
141
|
+
forward(x)
|
142
|
+
backward(y)
|
143
|
+
update_weight_and_bias
|
144
|
+
update_gamma_and_beta if @use_batch_norm
|
145
|
+
@layers[-1].loss(y)
|
146
|
+
end
|
147
|
+
|
148
|
+
def run(x)
|
149
|
+
if x.is_a?(Array)
|
150
|
+
forward(SFloat.cast(x), false).to_a
|
151
|
+
else
|
152
|
+
forward(x, false)
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
def save(file_name)
|
157
|
+
File.binwrite(file_name, Marshal.dump(self))
|
158
|
+
end
|
159
|
+
|
160
|
+
def save_json(file_name)
|
161
|
+
json = {
|
162
|
+
"version" => VERSION,
|
163
|
+
"num_nodes" => @num_nodes,
|
164
|
+
"learning_rate" => @learning_rate,
|
165
|
+
"batch_size" => @batch_size,
|
166
|
+
"activation" => @activation,
|
167
|
+
"momentum" => @momentum,
|
168
|
+
"weight_decay" => @weight_decay,
|
169
|
+
"use_dropout" => @use_dropout,
|
170
|
+
"dropout_ratio" => @dropout_ratio,
|
171
|
+
"use_batch_norm" => @use_batch_norm,
|
172
|
+
"weights" => @weights.map(&:to_a),
|
173
|
+
"biases" => @biases.map(&:to_a),
|
174
|
+
}
|
175
|
+
if @use_batch_norm
|
176
|
+
json_batch_norm = {
|
177
|
+
"gammas" => @gammas,
|
178
|
+
"betas" => @betas
|
179
|
+
}
|
180
|
+
json.merge!(json_batch_norm)
|
181
|
+
end
|
182
|
+
File.write(file_name, JSON.dump(json))
|
183
|
+
end
|
184
|
+
|
185
|
+
private
|
186
|
+
|
187
|
+
def init_weight_and_bias
|
188
|
+
@weights = Array.new(@num_nodes.length - 1)
|
189
|
+
@biases = Array.new(@num_nodes.length - 1)
|
190
|
+
@weight_amounts = Array.new(@num_nodes.length - 1, 0)
|
191
|
+
@bias_amounts = Array.new(@num_nodes.length - 1, 0)
|
192
|
+
@num_nodes[0...-1].each_index do |i|
|
193
|
+
weight = SFloat.new(@num_nodes[i], @num_nodes[i + 1]).rand_norm
|
194
|
+
bias = SFloat.new(@num_nodes[i + 1]).rand_norm
|
195
|
+
if @activation[0] == :relu
|
196
|
+
@weights[i] = weight / Math.sqrt(@num_nodes[i]) * Math.sqrt(2)
|
197
|
+
@biases[i] = bias / Math.sqrt(@num_nodes[i]) * Math.sqrt(2)
|
198
|
+
else
|
199
|
+
@weights[i] = weight / Math.sqrt(@num_nodes[i])
|
200
|
+
@biases[i] = bias / Math.sqrt(@num_nodes[i])
|
201
|
+
end
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
def init_gamma_and_beta
|
206
|
+
@gammas = Array.new(@num_nodes.length - 2, 1)
|
207
|
+
@betas = Array.new(@num_nodes.length - 2, 0)
|
208
|
+
@gamma_amounts = Array.new(@num_nodes.length - 2, 0)
|
209
|
+
@beta_amounts = Array.new(@num_nodes.length - 2, 0)
|
210
|
+
end
|
211
|
+
|
212
|
+
def init_layers
|
213
|
+
@layers = []
|
214
|
+
@num_nodes[0...-2].each_index do |i|
|
215
|
+
@layers << Affine.new(self, i)
|
216
|
+
@layers << BatchNorm.new(self, i) if @use_batch_norm
|
217
|
+
@layers << case @activation[0]
|
218
|
+
when :sigmoid
|
219
|
+
Sigmoid.new
|
220
|
+
when :relu
|
221
|
+
ReLU.new
|
222
|
+
end
|
223
|
+
@layers << Dropout.new(self) if @use_dropout
|
224
|
+
end
|
225
|
+
@layers << Affine.new(self, -1)
|
226
|
+
@layers << case @activation[1]
|
227
|
+
when :identity
|
228
|
+
Identity.new(self)
|
229
|
+
when :softmax
|
230
|
+
Softmax.new(self)
|
231
|
+
end
|
232
|
+
end
|
233
|
+
|
234
|
+
def forward(x, training = true)
|
235
|
+
@training = training
|
236
|
+
@layers.each do |layer|
|
237
|
+
x = layer.forward(x)
|
238
|
+
end
|
239
|
+
x
|
240
|
+
end
|
241
|
+
|
242
|
+
def backward(y)
|
243
|
+
dout = @layers[-1].backward(y)
|
244
|
+
@layers[0...-1].reverse.each do |layer|
|
245
|
+
dout = layer.backward(dout)
|
246
|
+
end
|
247
|
+
end
|
248
|
+
|
249
|
+
def update_weight_and_bias
|
250
|
+
@layers.select{|layer| layer.is_a?(Affine)}.each.with_index do |layer, i|
|
251
|
+
weight_amount = layer.d_weight * @learning_rate
|
252
|
+
bias_amount = layer.d_bias * @learning_rate
|
253
|
+
if @momentum > 0
|
254
|
+
weight_amount += @momentum * @weight_amounts[i]
|
255
|
+
@weight_amounts[i] = weight_amount
|
256
|
+
bias_amount += @momentum * @bias_amounts[i]
|
257
|
+
@bias_amounts[i] = bias_amount
|
258
|
+
end
|
259
|
+
@weights[i] -= weight_amount
|
260
|
+
@biases[i] -= bias_amount
|
261
|
+
end
|
262
|
+
end
|
263
|
+
|
264
|
+
def update_gamma_and_beta
|
265
|
+
@layers.select{|layer| layer.is_a?(BatchNorm)}.each.with_index do |layer, i|
|
266
|
+
gamma_amount = layer.d_gamma * @learning_rate
|
267
|
+
beta_amount = layer.d_beta * @learning_rate
|
268
|
+
if @momentum > 0
|
269
|
+
gamma_amount += @momentum * @gamma_amounts[i]
|
270
|
+
@gamma_amounts[i] = gamma_amount
|
271
|
+
beta_amount += @momentum * @beta_amounts[i]
|
272
|
+
@beta_amounts[i] = beta_amount
|
273
|
+
end
|
274
|
+
@gammas[i] -= gamma_amount
|
275
|
+
@betas[i] -= gamma_amount
|
276
|
+
end
|
277
|
+
end
|
278
|
+
end
|
279
|
+
|
280
|
+
|
281
|
+
class NN::Affine
|
282
|
+
include Numo
|
283
|
+
|
284
|
+
attr_reader :d_weight
|
285
|
+
attr_reader :d_bias
|
286
|
+
|
287
|
+
def initialize(nn, index)
|
288
|
+
@nn = nn
|
289
|
+
@index = index
|
290
|
+
@d_weight = nil
|
291
|
+
@d_bias = nil
|
292
|
+
end
|
293
|
+
|
294
|
+
def forward(x)
|
295
|
+
@x = x
|
296
|
+
@x.dot(@nn.weights[@index]) + @nn.biases[@index]
|
297
|
+
end
|
298
|
+
|
299
|
+
def backward(dout)
|
300
|
+
x = @x.reshape(*@x.shape, 1)
|
301
|
+
@d_weight = x.dot(dout.reshape(dout.shape[0], 1, dout.shape[1])).mean(0)
|
302
|
+
if @nn.weight_decay > 0
|
303
|
+
dridge = @nn.weight_decay * @nn.weights[@index]
|
304
|
+
@d_weight += dridge
|
305
|
+
end
|
306
|
+
@d_bias = dout.mean
|
307
|
+
dout.dot(@nn.weights[@index].transpose)
|
308
|
+
end
|
309
|
+
end
|
310
|
+
|
311
|
+
|
312
|
+
class NN::Sigmoid
|
313
|
+
include Numo
|
314
|
+
|
315
|
+
def forward(x)
|
316
|
+
@out = 1.0 / (1 + NMath.exp(-x))
|
317
|
+
end
|
318
|
+
|
319
|
+
def backward(dout)
|
320
|
+
dout * (1.0 - @out) * @out
|
321
|
+
end
|
322
|
+
end
|
323
|
+
|
324
|
+
|
325
|
+
class NN::ReLU
|
326
|
+
def forward(x)
|
327
|
+
@x = x.clone
|
328
|
+
x[x < 0] = 0
|
329
|
+
x
|
330
|
+
end
|
331
|
+
|
332
|
+
def backward(dout)
|
333
|
+
@x[@x > 0] = 1.0
|
334
|
+
@x[@x <= 0] = 0.0
|
335
|
+
dout * @x
|
336
|
+
end
|
337
|
+
end
|
338
|
+
|
339
|
+
|
340
|
+
class NN::Identity
|
341
|
+
def initialize(nn)
|
342
|
+
@nn = nn
|
343
|
+
end
|
344
|
+
|
345
|
+
def forward(x)
|
346
|
+
@out = x
|
347
|
+
end
|
348
|
+
|
349
|
+
def backward(y)
|
350
|
+
@out - y
|
351
|
+
end
|
352
|
+
|
353
|
+
def loss(y)
|
354
|
+
ridge = 0.5 * @nn.weight_decay * @nn.weights.reduce(0){|sum, weight| sum + (weight ** 2).sum}
|
355
|
+
0.5 * ((@out - y) ** 2).sum / @nn.batch_size + ridge
|
356
|
+
end
|
357
|
+
end
|
358
|
+
|
359
|
+
|
360
|
+
class NN::Softmax
|
361
|
+
include Numo
|
362
|
+
|
363
|
+
def initialize(nn)
|
364
|
+
@nn = nn
|
365
|
+
end
|
366
|
+
|
367
|
+
def forward(x)
|
368
|
+
@out = NMath.exp(x) / NMath.exp(x).sum(1).reshape(x.shape[0], 1)
|
369
|
+
end
|
370
|
+
|
371
|
+
def backward(y)
|
372
|
+
@out - y
|
373
|
+
end
|
374
|
+
|
375
|
+
def loss(y)
|
376
|
+
ridge = 0.5 * @nn.weight_decay * @nn.weights.reduce(0){|sum, weight| sum + (weight ** 2).sum}
|
377
|
+
-(y * NMath.log(@out + 1e-7)).sum / @nn.batch_size + ridge
|
378
|
+
end
|
379
|
+
end
|
380
|
+
|
381
|
+
|
382
|
+
class NN::Dropout
|
383
|
+
include Numo
|
384
|
+
|
385
|
+
def initialize(nn)
|
386
|
+
@nn = nn
|
387
|
+
@mask = nil
|
388
|
+
end
|
389
|
+
|
390
|
+
def forward(x)
|
391
|
+
if @nn.training
|
392
|
+
@mask = SFloat.ones(*x.shape).rand < @nn.dropout_ratio
|
393
|
+
x[@mask] = 0
|
394
|
+
else
|
395
|
+
x *= (1 - @nn.dropout_ratio)
|
396
|
+
end
|
397
|
+
x
|
398
|
+
end
|
399
|
+
|
400
|
+
def backward(dout)
|
401
|
+
dout[@mask] = 0 if @nn.training
|
402
|
+
dout
|
403
|
+
end
|
404
|
+
end
|
405
|
+
|
406
|
+
|
407
|
+
class NN::BatchNorm
|
408
|
+
include Numo
|
409
|
+
|
410
|
+
attr_reader :d_gamma
|
411
|
+
attr_reader :d_beta
|
412
|
+
|
413
|
+
def initialize(nn, index)
|
414
|
+
@nn = nn
|
415
|
+
@index = index
|
416
|
+
end
|
417
|
+
|
418
|
+
def forward(x)
|
419
|
+
@x = x
|
420
|
+
@mean = x.mean(0)
|
421
|
+
@xc = x - @mean
|
422
|
+
@var = (@xc ** 2).mean(0)
|
423
|
+
@std = NMath.sqrt(@var + 1e-7)
|
424
|
+
@xn = @xc / @std
|
425
|
+
out = @nn.gammas[@index] * @xn + @nn.betas[@index]
|
426
|
+
out.reshape(*@x.shape)
|
427
|
+
end
|
428
|
+
|
429
|
+
def backward(dout)
|
430
|
+
@d_beta = dout.sum(0).mean
|
431
|
+
@d_gamma = (@xn * dout).sum(0).mean
|
432
|
+
dxn = @nn.gammas[@index] * dout
|
433
|
+
dxc = dxn / @std
|
434
|
+
dstd = -((dxn * @xc) / (@std ** 2)).sum(0)
|
435
|
+
dvar = 0.5 * dstd / @std
|
436
|
+
dxc += (2.0 / @nn.batch_size) * @xc * dvar
|
437
|
+
dmean = dxc.sum(0)
|
438
|
+
dx = dxc - dmean / @nn.batch_size
|
439
|
+
dx.reshape(*@x.shape)
|
440
|
+
end
|
441
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require "nn"
|
2
|
+
require "nn/cifar10"
|
3
|
+
|
4
|
+
x_train = []
|
5
|
+
y_train = []
|
6
|
+
|
7
|
+
(1..5).each do |i|
|
8
|
+
x_train2, y_train2 = CIFAR10.load_train(i)
|
9
|
+
x_train.concat(x_train2)
|
10
|
+
y_train.concat(CIFAR10.categorical(y_train2))
|
11
|
+
end
|
12
|
+
GC.start
|
13
|
+
|
14
|
+
x_test, y_test = CIFAR10.load_test
|
15
|
+
y_test = CIFAR10.categorical(y_test)
|
16
|
+
GC.start
|
17
|
+
|
18
|
+
puts "load cifar10"
|
19
|
+
|
20
|
+
nn = NN.new([3072, 100, 100, 10],
|
21
|
+
learning_rate: 0.1,
|
22
|
+
batch_size: 32,
|
23
|
+
activation: [:relu, :softmax],
|
24
|
+
momentum: 0.9,
|
25
|
+
use_dropout: true,
|
26
|
+
dropout_ratio: 0.2,
|
27
|
+
use_batch_norm: true,
|
28
|
+
)
|
29
|
+
|
30
|
+
func = -> x, y do
|
31
|
+
x /= 255
|
32
|
+
[x, y]
|
33
|
+
end
|
34
|
+
|
35
|
+
nn.train(x_train, y_train, 20, func) do |epoch|
|
36
|
+
nn.test(x_test, y_test, &func)
|
37
|
+
nn.learning_rate *= 0.99
|
38
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
#ライブラリの読み込み
|
2
|
+
require "nn"
|
3
|
+
require "nn/mnist"
|
4
|
+
|
5
|
+
#MNISTのトレーニング用データを読み込む
|
6
|
+
x_train, y_train = MNIST.load_train
|
7
|
+
|
8
|
+
#y_trainを10クラスに配列でカテゴライズする
|
9
|
+
y_train = MNIST.categorical(y_train)
|
10
|
+
|
11
|
+
#MNISTのテスト用データを読み込む
|
12
|
+
x_test, y_test = MNIST.load_test
|
13
|
+
|
14
|
+
#y_testを10クラスにカテゴライズする
|
15
|
+
y_test = MNIST.categorical(y_test)
|
16
|
+
|
17
|
+
puts "load mnist"
|
18
|
+
|
19
|
+
#ニューラルネットワークの初期化
|
20
|
+
nn = NN.new([784, 100, 100, 10], #ノード数
|
21
|
+
learning_rate: 0.1, #学習率
|
22
|
+
batch_size: 100, #ミニバッチの数
|
23
|
+
activation: [:relu, :softmax], #活性化関数
|
24
|
+
momentum: 0.9, #モーメンタム係数
|
25
|
+
use_batch_norm: true, #バッチノーマライゼーションを使用する
|
26
|
+
)
|
27
|
+
|
28
|
+
#ミニバッチを0~1の範囲で正規化
|
29
|
+
func = -> x_batch, y_batch do
|
30
|
+
x_batch /= 255
|
31
|
+
[x_batch, y_batch]
|
32
|
+
end
|
33
|
+
|
34
|
+
#学習を行う
|
35
|
+
nn.train(x_train, y_train, 10, func) do
|
36
|
+
#学習結果のテストを行う
|
37
|
+
nn.test(x_test, y_test, &func)
|
38
|
+
end
|
data/sample/xor.rb
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
#ライブラリの読み込み
|
2
|
+
require "nn"
|
3
|
+
|
4
|
+
x = [
|
5
|
+
[0, 0],
|
6
|
+
[1, 0],
|
7
|
+
[0, 1],
|
8
|
+
[1, 1],
|
9
|
+
]
|
10
|
+
|
11
|
+
y = [[0], [1], [1], [0]]
|
12
|
+
|
13
|
+
#ニューラルネットワークの初期化
|
14
|
+
nn = NN.new([2, 4, 1], #ノード数
|
15
|
+
learning_rate: 0.1, #学習率
|
16
|
+
batch_size: 4, #ミニバッチの数
|
17
|
+
activation: [:sigmoid, :identity] #活性化関数
|
18
|
+
)
|
19
|
+
|
20
|
+
#学習を行う
|
21
|
+
nn.train(x, y, 20000)
|
22
|
+
|
23
|
+
#学習結果の確認
|
24
|
+
p nn.run(x)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nn
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- unagiootoro
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-06-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: numo-narray
|
@@ -70,6 +70,10 @@ files:
|
|
70
70
|
- lib/nn/cifar10.rb
|
71
71
|
- lib/nn/mnist.rb
|
72
72
|
- nn.gemspec
|
73
|
+
- nn.rb
|
74
|
+
- sample/cifar10_program.rb
|
75
|
+
- sample/mnist_program.rb
|
76
|
+
- sample/xor.rb
|
73
77
|
homepage: https://github.com/unagiootoro/nn.git
|
74
78
|
licenses:
|
75
79
|
- MIT
|