nn 2.0.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/document.txt +0 -10
- data/nn.gemspec +1 -1
- data/sample/cifar10_program.rb +1 -1
- metadata +1 -2
- data/nn.rb +0 -441
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5fd7f6dd2b015169de254f1fc68d8566a5ffa7e14bfc8bd9581239c96a6f6965
|
4
|
+
data.tar.gz: 232ee0dbbd7a16fce35e56e58ea40b79dc76b066a2c93dff576d48a34958ab66
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6323e2716a4ef835665b8955ce699ee5843649dfbaa469257d4b164c2702f39a508f2034510c001785e671dd781e36b4afc863f16d1305865114925876aa807a
|
7
|
+
data.tar.gz: da603d96fc83ab378cd34233624487b39fbd137037af5e4ca2d306a5c63cafc2193585d94405fb3f624ab7f9749e8587e86fc842a7cb5845e37001c70476f122
|
data/document.txt
CHANGED
@@ -13,11 +13,6 @@ class NN
|
|
13
13
|
|
14
14
|
<クラスメソッド>
|
15
15
|
load(file_name) : NN
|
16
|
-
Marshal形式で保存された学習結果を読み込みます。
|
17
|
-
String file_name 読み込むMarshalファイル名
|
18
|
-
戻り値 NNのインスタンス
|
19
|
-
|
20
|
-
load_json(file_name) : NN
|
21
16
|
JSON形式で保存された学習結果を読み込みます。
|
22
17
|
String file_name 読み込むJSONファイル名
|
23
18
|
戻り値 NNのインスタンス
|
@@ -114,10 +109,6 @@ run(x) : SFloat
|
|
114
109
|
戻り値 出力ノードの値
|
115
110
|
|
116
111
|
save(file_name) : void
|
117
|
-
学習結果をMarshal形式で保存します。
|
118
|
-
String file_name 書き込むMarshalファイル名
|
119
|
-
|
120
|
-
save_json(file_name) : void
|
121
112
|
学習結果をJSON形式で保存します。
|
122
113
|
String file_name 書き込むJSONファイル名
|
123
114
|
|
@@ -151,4 +142,3 @@ http://d.hatena.ne.jp/n_shuyo/20090913/mnist
|
|
151
142
|
2018/5/4 バージョン1.8公開
|
152
143
|
2018/5/16 バージョン2.0公開
|
153
144
|
2018/6/10 バージョン2.0.1公開
|
154
|
-
2018/6/10 バージョン2.1.0公開
|
data/nn.gemspec
CHANGED
data/sample/cifar10_program.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nn
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- unagiootoro
|
@@ -70,7 +70,6 @@ files:
|
|
70
70
|
- lib/nn/cifar10.rb
|
71
71
|
- lib/nn/mnist.rb
|
72
72
|
- nn.gemspec
|
73
|
-
- nn.rb
|
74
73
|
- sample/cifar10_program.rb
|
75
74
|
- sample/mnist_program.rb
|
76
75
|
- sample/xor.rb
|
data/nn.rb
DELETED
@@ -1,441 +0,0 @@
|
|
1
|
-
require "numo/narray"
|
2
|
-
require "json"
|
3
|
-
|
4
|
-
class NN
|
5
|
-
VERSION = "2.1"
|
6
|
-
|
7
|
-
include Numo
|
8
|
-
|
9
|
-
attr_accessor :weights
|
10
|
-
attr_accessor :biases
|
11
|
-
attr_accessor :gammas
|
12
|
-
attr_accessor :betas
|
13
|
-
attr_accessor :learning_rate
|
14
|
-
attr_accessor :batch_size
|
15
|
-
attr_accessor :activation
|
16
|
-
attr_accessor :momentum
|
17
|
-
attr_accessor :weight_decay
|
18
|
-
attr_accessor :dropout_ratio
|
19
|
-
attr_reader :training
|
20
|
-
|
21
|
-
def initialize(num_nodes,
|
22
|
-
learning_rate: 0.01,
|
23
|
-
batch_size: 1,
|
24
|
-
activation: %i(relu identity),
|
25
|
-
momentum: 0,
|
26
|
-
weight_decay: 0,
|
27
|
-
use_dropout: false,
|
28
|
-
dropout_ratio: 0.5,
|
29
|
-
use_batch_norm: false)
|
30
|
-
SFloat.srand(rand(2 ** 64))
|
31
|
-
@num_nodes = num_nodes
|
32
|
-
@learning_rate = learning_rate
|
33
|
-
@batch_size = batch_size
|
34
|
-
@activation = activation
|
35
|
-
@momentum = momentum
|
36
|
-
@weight_decay = weight_decay
|
37
|
-
@use_dropout = use_dropout
|
38
|
-
@dropout_ratio = dropout_ratio
|
39
|
-
@use_batch_norm = use_batch_norm
|
40
|
-
init_weight_and_bias
|
41
|
-
init_gamma_and_beta if @use_batch_norm
|
42
|
-
@training = true
|
43
|
-
init_layers
|
44
|
-
end
|
45
|
-
|
46
|
-
def self.load(file_name)
|
47
|
-
Marshal.load(File.binread(file_name))
|
48
|
-
end
|
49
|
-
|
50
|
-
def self.load_json(file_name)
|
51
|
-
json = JSON.parse(File.read(file_name))
|
52
|
-
nn = self.new(json["num_nodes"],
|
53
|
-
learning_rate: json["learning_rate"],
|
54
|
-
batch_size: json["batch_size"],
|
55
|
-
activation: json["activation"].map(&:to_sym),
|
56
|
-
momentum: json["momentum"],
|
57
|
-
weight_decay: json["weight_decay"],
|
58
|
-
use_dropout: json["use_dropout"],
|
59
|
-
dropout_ratio: json["dropout_ratio"],
|
60
|
-
use_batch_norm: json["use_batch_norm"],
|
61
|
-
)
|
62
|
-
nn.weights = json["weights"].map{|weight| SFloat.cast(weight)}
|
63
|
-
nn.biases = json["biases"].map{|bias| SFloat.cast(bias)}
|
64
|
-
if json["use_batch_norm"]
|
65
|
-
nn.gammas = json["gammas"].map{|gamma| SFloat.cast(gamma)}
|
66
|
-
nn.betas = json["betas"].map{|beta| SFloat.cast(beta)}
|
67
|
-
end
|
68
|
-
nn
|
69
|
-
end
|
70
|
-
|
71
|
-
def train(x_train, y_train, epochs, func = nil, &block)
|
72
|
-
num_train_data = x_train.is_a?(SFloat) ? x_train.shape[0] : x_train.length
|
73
|
-
(1..epochs).each do |epoch|
|
74
|
-
loss = nil
|
75
|
-
(num_train_data.to_f / @batch_size).ceil.times do
|
76
|
-
loss = learn(x_train, y_train, &func)
|
77
|
-
if loss.nan?
|
78
|
-
puts "loss is nan"
|
79
|
-
return
|
80
|
-
end
|
81
|
-
end
|
82
|
-
puts "epoch #{epoch}/#{epochs} loss: #{loss}"
|
83
|
-
block.call(epoch) if block
|
84
|
-
end
|
85
|
-
end
|
86
|
-
|
87
|
-
def test(x_test, y_test, tolerance = 0.5, &block)
|
88
|
-
acc = accurate(x_test, y_test, tolerance, &block)
|
89
|
-
puts "accurate: #{acc}"
|
90
|
-
acc
|
91
|
-
end
|
92
|
-
|
93
|
-
def accurate(x_test, y_test, tolerance = 0.5, &block)
|
94
|
-
correct = 0
|
95
|
-
num_test_data = x_test.is_a?(SFloat) ? x_test.shape[0] : x_test.length
|
96
|
-
(num_test_data.to_f / @batch_size).ceil.times do |i|
|
97
|
-
x = SFloat.zeros(@batch_size, @num_nodes.first)
|
98
|
-
y = SFloat.zeros(@batch_size, @num_nodes.last)
|
99
|
-
@batch_size.times do |j|
|
100
|
-
k = i * @batch_size + j
|
101
|
-
break if k >= num_test_data
|
102
|
-
if x_test.is_a?(SFloat)
|
103
|
-
x[j, true] = x_test[k, true]
|
104
|
-
y[j, true] = y_test[k, true]
|
105
|
-
else
|
106
|
-
x[j, true] = SFloat.cast(x_test[k])
|
107
|
-
y[j, true] = SFloat.cast(y_test[k])
|
108
|
-
end
|
109
|
-
end
|
110
|
-
x, y = block.call(x, y) if block
|
111
|
-
out = forward(x, false)
|
112
|
-
@batch_size.times do |j|
|
113
|
-
vout = out[j, true]
|
114
|
-
vy = y[j, true]
|
115
|
-
case @activation[1]
|
116
|
-
when :identity
|
117
|
-
correct += 1 unless (NMath.sqrt((vout - vy) ** 2) < tolerance).to_a.include?(0)
|
118
|
-
when :softmax
|
119
|
-
correct += 1 if vout.max_index == vy.max_index
|
120
|
-
end
|
121
|
-
end
|
122
|
-
end
|
123
|
-
correct.to_f / num_test_data
|
124
|
-
end
|
125
|
-
|
126
|
-
def learn(x_train, y_train, &block)
|
127
|
-
x = SFloat.zeros(@batch_size, @num_nodes.first)
|
128
|
-
y = SFloat.zeros(@batch_size, @num_nodes.last)
|
129
|
-
@batch_size.times do |i|
|
130
|
-
if x_train.is_a?(SFloat)
|
131
|
-
r = rand(x_train.shape[0])
|
132
|
-
x[i, true] = x_train[r, true]
|
133
|
-
y[i, true] = y_train[r, true]
|
134
|
-
else
|
135
|
-
r = rand(x_train.length)
|
136
|
-
x[i, true] = SFloat.cast(x_train[r])
|
137
|
-
y[i, true] = SFloat.cast(y_train[r])
|
138
|
-
end
|
139
|
-
end
|
140
|
-
x, y = block.call(x, y) if block
|
141
|
-
forward(x)
|
142
|
-
backward(y)
|
143
|
-
update_weight_and_bias
|
144
|
-
update_gamma_and_beta if @use_batch_norm
|
145
|
-
@layers[-1].loss(y)
|
146
|
-
end
|
147
|
-
|
148
|
-
def run(x)
|
149
|
-
if x.is_a?(Array)
|
150
|
-
forward(SFloat.cast(x), false).to_a
|
151
|
-
else
|
152
|
-
forward(x, false)
|
153
|
-
end
|
154
|
-
end
|
155
|
-
|
156
|
-
def save(file_name)
|
157
|
-
File.binwrite(file_name, Marshal.dump(self))
|
158
|
-
end
|
159
|
-
|
160
|
-
def save_json(file_name)
|
161
|
-
json = {
|
162
|
-
"version" => VERSION,
|
163
|
-
"num_nodes" => @num_nodes,
|
164
|
-
"learning_rate" => @learning_rate,
|
165
|
-
"batch_size" => @batch_size,
|
166
|
-
"activation" => @activation,
|
167
|
-
"momentum" => @momentum,
|
168
|
-
"weight_decay" => @weight_decay,
|
169
|
-
"use_dropout" => @use_dropout,
|
170
|
-
"dropout_ratio" => @dropout_ratio,
|
171
|
-
"use_batch_norm" => @use_batch_norm,
|
172
|
-
"weights" => @weights.map(&:to_a),
|
173
|
-
"biases" => @biases.map(&:to_a),
|
174
|
-
}
|
175
|
-
if @use_batch_norm
|
176
|
-
json_batch_norm = {
|
177
|
-
"gammas" => @gammas,
|
178
|
-
"betas" => @betas
|
179
|
-
}
|
180
|
-
json.merge!(json_batch_norm)
|
181
|
-
end
|
182
|
-
File.write(file_name, JSON.dump(json))
|
183
|
-
end
|
184
|
-
|
185
|
-
private
|
186
|
-
|
187
|
-
def init_weight_and_bias
|
188
|
-
@weights = Array.new(@num_nodes.length - 1)
|
189
|
-
@biases = Array.new(@num_nodes.length - 1)
|
190
|
-
@weight_amounts = Array.new(@num_nodes.length - 1, 0)
|
191
|
-
@bias_amounts = Array.new(@num_nodes.length - 1, 0)
|
192
|
-
@num_nodes[0...-1].each_index do |i|
|
193
|
-
weight = SFloat.new(@num_nodes[i], @num_nodes[i + 1]).rand_norm
|
194
|
-
bias = SFloat.new(@num_nodes[i + 1]).rand_norm
|
195
|
-
if @activation[0] == :relu
|
196
|
-
@weights[i] = weight / Math.sqrt(@num_nodes[i]) * Math.sqrt(2)
|
197
|
-
@biases[i] = bias / Math.sqrt(@num_nodes[i]) * Math.sqrt(2)
|
198
|
-
else
|
199
|
-
@weights[i] = weight / Math.sqrt(@num_nodes[i])
|
200
|
-
@biases[i] = bias / Math.sqrt(@num_nodes[i])
|
201
|
-
end
|
202
|
-
end
|
203
|
-
end
|
204
|
-
|
205
|
-
def init_gamma_and_beta
|
206
|
-
@gammas = Array.new(@num_nodes.length - 2, 1)
|
207
|
-
@betas = Array.new(@num_nodes.length - 2, 0)
|
208
|
-
@gamma_amounts = Array.new(@num_nodes.length - 2, 0)
|
209
|
-
@beta_amounts = Array.new(@num_nodes.length - 2, 0)
|
210
|
-
end
|
211
|
-
|
212
|
-
def init_layers
|
213
|
-
@layers = []
|
214
|
-
@num_nodes[0...-2].each_index do |i|
|
215
|
-
@layers << Affine.new(self, i)
|
216
|
-
@layers << BatchNorm.new(self, i) if @use_batch_norm
|
217
|
-
@layers << case @activation[0]
|
218
|
-
when :sigmoid
|
219
|
-
Sigmoid.new
|
220
|
-
when :relu
|
221
|
-
ReLU.new
|
222
|
-
end
|
223
|
-
@layers << Dropout.new(self) if @use_dropout
|
224
|
-
end
|
225
|
-
@layers << Affine.new(self, -1)
|
226
|
-
@layers << case @activation[1]
|
227
|
-
when :identity
|
228
|
-
Identity.new(self)
|
229
|
-
when :softmax
|
230
|
-
Softmax.new(self)
|
231
|
-
end
|
232
|
-
end
|
233
|
-
|
234
|
-
def forward(x, training = true)
|
235
|
-
@training = training
|
236
|
-
@layers.each do |layer|
|
237
|
-
x = layer.forward(x)
|
238
|
-
end
|
239
|
-
x
|
240
|
-
end
|
241
|
-
|
242
|
-
def backward(y)
|
243
|
-
dout = @layers[-1].backward(y)
|
244
|
-
@layers[0...-1].reverse.each do |layer|
|
245
|
-
dout = layer.backward(dout)
|
246
|
-
end
|
247
|
-
end
|
248
|
-
|
249
|
-
def update_weight_and_bias
|
250
|
-
@layers.select{|layer| layer.is_a?(Affine)}.each.with_index do |layer, i|
|
251
|
-
weight_amount = layer.d_weight * @learning_rate
|
252
|
-
bias_amount = layer.d_bias * @learning_rate
|
253
|
-
if @momentum > 0
|
254
|
-
weight_amount += @momentum * @weight_amounts[i]
|
255
|
-
@weight_amounts[i] = weight_amount
|
256
|
-
bias_amount += @momentum * @bias_amounts[i]
|
257
|
-
@bias_amounts[i] = bias_amount
|
258
|
-
end
|
259
|
-
@weights[i] -= weight_amount
|
260
|
-
@biases[i] -= bias_amount
|
261
|
-
end
|
262
|
-
end
|
263
|
-
|
264
|
-
def update_gamma_and_beta
|
265
|
-
@layers.select{|layer| layer.is_a?(BatchNorm)}.each.with_index do |layer, i|
|
266
|
-
gamma_amount = layer.d_gamma * @learning_rate
|
267
|
-
beta_amount = layer.d_beta * @learning_rate
|
268
|
-
if @momentum > 0
|
269
|
-
gamma_amount += @momentum * @gamma_amounts[i]
|
270
|
-
@gamma_amounts[i] = gamma_amount
|
271
|
-
beta_amount += @momentum * @beta_amounts[i]
|
272
|
-
@beta_amounts[i] = beta_amount
|
273
|
-
end
|
274
|
-
@gammas[i] -= gamma_amount
|
275
|
-
@betas[i] -= gamma_amount
|
276
|
-
end
|
277
|
-
end
|
278
|
-
end
|
279
|
-
|
280
|
-
|
281
|
-
class NN::Affine
|
282
|
-
include Numo
|
283
|
-
|
284
|
-
attr_reader :d_weight
|
285
|
-
attr_reader :d_bias
|
286
|
-
|
287
|
-
def initialize(nn, index)
|
288
|
-
@nn = nn
|
289
|
-
@index = index
|
290
|
-
@d_weight = nil
|
291
|
-
@d_bias = nil
|
292
|
-
end
|
293
|
-
|
294
|
-
def forward(x)
|
295
|
-
@x = x
|
296
|
-
@x.dot(@nn.weights[@index]) + @nn.biases[@index]
|
297
|
-
end
|
298
|
-
|
299
|
-
def backward(dout)
|
300
|
-
x = @x.reshape(*@x.shape, 1)
|
301
|
-
@d_weight = x.dot(dout.reshape(dout.shape[0], 1, dout.shape[1])).mean(0)
|
302
|
-
if @nn.weight_decay > 0
|
303
|
-
dridge = @nn.weight_decay * @nn.weights[@index]
|
304
|
-
@d_weight += dridge
|
305
|
-
end
|
306
|
-
@d_bias = dout.mean
|
307
|
-
dout.dot(@nn.weights[@index].transpose)
|
308
|
-
end
|
309
|
-
end
|
310
|
-
|
311
|
-
|
312
|
-
class NN::Sigmoid
|
313
|
-
include Numo
|
314
|
-
|
315
|
-
def forward(x)
|
316
|
-
@out = 1.0 / (1 + NMath.exp(-x))
|
317
|
-
end
|
318
|
-
|
319
|
-
def backward(dout)
|
320
|
-
dout * (1.0 - @out) * @out
|
321
|
-
end
|
322
|
-
end
|
323
|
-
|
324
|
-
|
325
|
-
class NN::ReLU
|
326
|
-
def forward(x)
|
327
|
-
@x = x.clone
|
328
|
-
x[x < 0] = 0
|
329
|
-
x
|
330
|
-
end
|
331
|
-
|
332
|
-
def backward(dout)
|
333
|
-
@x[@x > 0] = 1.0
|
334
|
-
@x[@x <= 0] = 0.0
|
335
|
-
dout * @x
|
336
|
-
end
|
337
|
-
end
|
338
|
-
|
339
|
-
|
340
|
-
class NN::Identity
|
341
|
-
def initialize(nn)
|
342
|
-
@nn = nn
|
343
|
-
end
|
344
|
-
|
345
|
-
def forward(x)
|
346
|
-
@out = x
|
347
|
-
end
|
348
|
-
|
349
|
-
def backward(y)
|
350
|
-
@out - y
|
351
|
-
end
|
352
|
-
|
353
|
-
def loss(y)
|
354
|
-
ridge = 0.5 * @nn.weight_decay * @nn.weights.reduce(0){|sum, weight| sum + (weight ** 2).sum}
|
355
|
-
0.5 * ((@out - y) ** 2).sum / @nn.batch_size + ridge
|
356
|
-
end
|
357
|
-
end
|
358
|
-
|
359
|
-
|
360
|
-
class NN::Softmax
|
361
|
-
include Numo
|
362
|
-
|
363
|
-
def initialize(nn)
|
364
|
-
@nn = nn
|
365
|
-
end
|
366
|
-
|
367
|
-
def forward(x)
|
368
|
-
@out = NMath.exp(x) / NMath.exp(x).sum(1).reshape(x.shape[0], 1)
|
369
|
-
end
|
370
|
-
|
371
|
-
def backward(y)
|
372
|
-
@out - y
|
373
|
-
end
|
374
|
-
|
375
|
-
def loss(y)
|
376
|
-
ridge = 0.5 * @nn.weight_decay * @nn.weights.reduce(0){|sum, weight| sum + (weight ** 2).sum}
|
377
|
-
-(y * NMath.log(@out + 1e-7)).sum / @nn.batch_size + ridge
|
378
|
-
end
|
379
|
-
end
|
380
|
-
|
381
|
-
|
382
|
-
class NN::Dropout
|
383
|
-
include Numo
|
384
|
-
|
385
|
-
def initialize(nn)
|
386
|
-
@nn = nn
|
387
|
-
@mask = nil
|
388
|
-
end
|
389
|
-
|
390
|
-
def forward(x)
|
391
|
-
if @nn.training
|
392
|
-
@mask = SFloat.ones(*x.shape).rand < @nn.dropout_ratio
|
393
|
-
x[@mask] = 0
|
394
|
-
else
|
395
|
-
x *= (1 - @nn.dropout_ratio)
|
396
|
-
end
|
397
|
-
x
|
398
|
-
end
|
399
|
-
|
400
|
-
def backward(dout)
|
401
|
-
dout[@mask] = 0 if @nn.training
|
402
|
-
dout
|
403
|
-
end
|
404
|
-
end
|
405
|
-
|
406
|
-
|
407
|
-
class NN::BatchNorm
|
408
|
-
include Numo
|
409
|
-
|
410
|
-
attr_reader :d_gamma
|
411
|
-
attr_reader :d_beta
|
412
|
-
|
413
|
-
def initialize(nn, index)
|
414
|
-
@nn = nn
|
415
|
-
@index = index
|
416
|
-
end
|
417
|
-
|
418
|
-
def forward(x)
|
419
|
-
@x = x
|
420
|
-
@mean = x.mean(0)
|
421
|
-
@xc = x - @mean
|
422
|
-
@var = (@xc ** 2).mean(0)
|
423
|
-
@std = NMath.sqrt(@var + 1e-7)
|
424
|
-
@xn = @xc / @std
|
425
|
-
out = @nn.gammas[@index] * @xn + @nn.betas[@index]
|
426
|
-
out.reshape(*@x.shape)
|
427
|
-
end
|
428
|
-
|
429
|
-
def backward(dout)
|
430
|
-
@d_beta = dout.sum(0).mean
|
431
|
-
@d_gamma = (@xn * dout).sum(0).mean
|
432
|
-
dxn = @nn.gammas[@index] * dout
|
433
|
-
dxc = dxn / @std
|
434
|
-
dstd = -((dxn * @xc) / (@std ** 2)).sum(0)
|
435
|
-
dvar = 0.5 * dstd / @std
|
436
|
-
dxc += (2.0 / @nn.batch_size) * @xc * dvar
|
437
|
-
dmean = dxc.sum(0)
|
438
|
-
dx = dxc - dmean / @nn.batch_size
|
439
|
-
dx.reshape(*@x.shape)
|
440
|
-
end
|
441
|
-
end
|