nn 2.0.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8f77c817ea492d035851bf8552ad2a97928f6762acb455ae23de0e3ee8f40871
4
- data.tar.gz: 1f162719087671733c8afd5279bca59859474dd366677acbcf79032a9fff5eba
3
+ metadata.gz: 5fd7f6dd2b015169de254f1fc68d8566a5ffa7e14bfc8bd9581239c96a6f6965
4
+ data.tar.gz: 232ee0dbbd7a16fce35e56e58ea40b79dc76b066a2c93dff576d48a34958ab66
5
5
  SHA512:
6
- metadata.gz: 492e639590f4b81083a669f51ee192cb9a758ee0bbe950539c74367322ab78a9ad77e6075f90e56f28eecbc57ba42df91455c88e109fe2ec5565ceb77730bafc
7
- data.tar.gz: d8745f38ed5ca0d75da462c6a8cf1233ea8e9c69e10918b87f37c23c8344690075bba1f6c13cf16267f51a92993e1fe58c0297251c309adcbbd3c4c85f339221
6
+ metadata.gz: 6323e2716a4ef835665b8955ce699ee5843649dfbaa469257d4b164c2702f39a508f2034510c001785e671dd781e36b4afc863f16d1305865114925876aa807a
7
+ data.tar.gz: da603d96fc83ab378cd34233624487b39fbd137037af5e4ca2d306a5c63cafc2193585d94405fb3f624ab7f9749e8587e86fc842a7cb5845e37001c70476f122
data/document.txt CHANGED
@@ -13,11 +13,6 @@ class NN
13
13
 
14
14
  <クラスメソッド>
15
15
  load(file_name) : NN
16
- Marshal形式で保存された学習結果を読み込みます。
17
- String file_name 読み込むMarshalファイル名
18
- 戻り値 NNのインスタンス
19
-
20
- load_json(file_name) : NN
21
16
  JSON形式で保存された学習結果を読み込みます。
22
17
  String file_name 読み込むJSONファイル名
23
18
  戻り値 NNのインスタンス
@@ -114,10 +109,6 @@ run(x) : SFloat
114
109
  戻り値 出力ノードの値
115
110
 
116
111
  save(file_name) : void
117
- 学習結果をMarshal形式で保存します。
118
- String file_name 書き込むMarshalファイル名
119
-
120
- save_json(file_name) : void
121
112
  学習結果をJSON形式で保存します。
122
113
  String file_name 書き込むJSONファイル名
123
114
 
@@ -151,4 +142,3 @@ http://d.hatena.ne.jp/n_shuyo/20090913/mnist
151
142
  2018/5/4 バージョン1.8公開
152
143
  2018/5/16 バージョン2.0公開
153
144
  2018/6/10 バージョン2.0.1公開
154
- 2018/6/10 バージョン2.1.0公開
data/nn.gemspec CHANGED
@@ -5,7 +5,7 @@ require "nn"
5
5
 
6
6
  Gem::Specification.new do |spec|
7
7
  spec.name = "nn"
8
- spec.version = NN::VERSION + ".0"
8
+ spec.version = NN::VERSION + ".1"
9
9
  spec.authors = ["unagiootoro"]
10
10
  spec.email = ["ootoro838861@outlook.jp"]
11
11
 
@@ -32,7 +32,7 @@ func = -> x, y do
32
32
  [x, y]
33
33
  end
34
34
 
35
- nn.train(x_train, y_train, 20, func) do |epoch|
35
+ nn.train(x_train, y_train, 50, func) do |epoch|
36
36
  nn.test(x_test, y_test, &func)
37
37
  nn.learning_rate *= 0.99
38
38
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: nn
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0
4
+ version: 2.0.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - unagiootoro
@@ -70,7 +70,6 @@ files:
70
70
  - lib/nn/cifar10.rb
71
71
  - lib/nn/mnist.rb
72
72
  - nn.gemspec
73
- - nn.rb
74
73
  - sample/cifar10_program.rb
75
74
  - sample/mnist_program.rb
76
75
  - sample/xor.rb
data/nn.rb DELETED
@@ -1,441 +0,0 @@
1
- require "numo/narray"
2
- require "json"
3
-
4
- class NN
5
- VERSION = "2.1"
6
-
7
- include Numo
8
-
9
- attr_accessor :weights
10
- attr_accessor :biases
11
- attr_accessor :gammas
12
- attr_accessor :betas
13
- attr_accessor :learning_rate
14
- attr_accessor :batch_size
15
- attr_accessor :activation
16
- attr_accessor :momentum
17
- attr_accessor :weight_decay
18
- attr_accessor :dropout_ratio
19
- attr_reader :training
20
-
21
- def initialize(num_nodes,
22
- learning_rate: 0.01,
23
- batch_size: 1,
24
- activation: %i(relu identity),
25
- momentum: 0,
26
- weight_decay: 0,
27
- use_dropout: false,
28
- dropout_ratio: 0.5,
29
- use_batch_norm: false)
30
- SFloat.srand(rand(2 ** 64))
31
- @num_nodes = num_nodes
32
- @learning_rate = learning_rate
33
- @batch_size = batch_size
34
- @activation = activation
35
- @momentum = momentum
36
- @weight_decay = weight_decay
37
- @use_dropout = use_dropout
38
- @dropout_ratio = dropout_ratio
39
- @use_batch_norm = use_batch_norm
40
- init_weight_and_bias
41
- init_gamma_and_beta if @use_batch_norm
42
- @training = true
43
- init_layers
44
- end
45
-
46
- def self.load(file_name)
47
- Marshal.load(File.binread(file_name))
48
- end
49
-
50
- def self.load_json(file_name)
51
- json = JSON.parse(File.read(file_name))
52
- nn = self.new(json["num_nodes"],
53
- learning_rate: json["learning_rate"],
54
- batch_size: json["batch_size"],
55
- activation: json["activation"].map(&:to_sym),
56
- momentum: json["momentum"],
57
- weight_decay: json["weight_decay"],
58
- use_dropout: json["use_dropout"],
59
- dropout_ratio: json["dropout_ratio"],
60
- use_batch_norm: json["use_batch_norm"],
61
- )
62
- nn.weights = json["weights"].map{|weight| SFloat.cast(weight)}
63
- nn.biases = json["biases"].map{|bias| SFloat.cast(bias)}
64
- if json["use_batch_norm"]
65
- nn.gammas = json["gammas"].map{|gamma| SFloat.cast(gamma)}
66
- nn.betas = json["betas"].map{|beta| SFloat.cast(beta)}
67
- end
68
- nn
69
- end
70
-
71
- def train(x_train, y_train, epochs, func = nil, &block)
72
- num_train_data = x_train.is_a?(SFloat) ? x_train.shape[0] : x_train.length
73
- (1..epochs).each do |epoch|
74
- loss = nil
75
- (num_train_data.to_f / @batch_size).ceil.times do
76
- loss = learn(x_train, y_train, &func)
77
- if loss.nan?
78
- puts "loss is nan"
79
- return
80
- end
81
- end
82
- puts "epoch #{epoch}/#{epochs} loss: #{loss}"
83
- block.call(epoch) if block
84
- end
85
- end
86
-
87
- def test(x_test, y_test, tolerance = 0.5, &block)
88
- acc = accurate(x_test, y_test, tolerance, &block)
89
- puts "accurate: #{acc}"
90
- acc
91
- end
92
-
93
- def accurate(x_test, y_test, tolerance = 0.5, &block)
94
- correct = 0
95
- num_test_data = x_test.is_a?(SFloat) ? x_test.shape[0] : x_test.length
96
- (num_test_data.to_f / @batch_size).ceil.times do |i|
97
- x = SFloat.zeros(@batch_size, @num_nodes.first)
98
- y = SFloat.zeros(@batch_size, @num_nodes.last)
99
- @batch_size.times do |j|
100
- k = i * @batch_size + j
101
- break if k >= num_test_data
102
- if x_test.is_a?(SFloat)
103
- x[j, true] = x_test[k, true]
104
- y[j, true] = y_test[k, true]
105
- else
106
- x[j, true] = SFloat.cast(x_test[k])
107
- y[j, true] = SFloat.cast(y_test[k])
108
- end
109
- end
110
- x, y = block.call(x, y) if block
111
- out = forward(x, false)
112
- @batch_size.times do |j|
113
- vout = out[j, true]
114
- vy = y[j, true]
115
- case @activation[1]
116
- when :identity
117
- correct += 1 unless (NMath.sqrt((vout - vy) ** 2) < tolerance).to_a.include?(0)
118
- when :softmax
119
- correct += 1 if vout.max_index == vy.max_index
120
- end
121
- end
122
- end
123
- correct.to_f / num_test_data
124
- end
125
-
126
- def learn(x_train, y_train, &block)
127
- x = SFloat.zeros(@batch_size, @num_nodes.first)
128
- y = SFloat.zeros(@batch_size, @num_nodes.last)
129
- @batch_size.times do |i|
130
- if x_train.is_a?(SFloat)
131
- r = rand(x_train.shape[0])
132
- x[i, true] = x_train[r, true]
133
- y[i, true] = y_train[r, true]
134
- else
135
- r = rand(x_train.length)
136
- x[i, true] = SFloat.cast(x_train[r])
137
- y[i, true] = SFloat.cast(y_train[r])
138
- end
139
- end
140
- x, y = block.call(x, y) if block
141
- forward(x)
142
- backward(y)
143
- update_weight_and_bias
144
- update_gamma_and_beta if @use_batch_norm
145
- @layers[-1].loss(y)
146
- end
147
-
148
- def run(x)
149
- if x.is_a?(Array)
150
- forward(SFloat.cast(x), false).to_a
151
- else
152
- forward(x, false)
153
- end
154
- end
155
-
156
- def save(file_name)
157
- File.binwrite(file_name, Marshal.dump(self))
158
- end
159
-
160
- def save_json(file_name)
161
- json = {
162
- "version" => VERSION,
163
- "num_nodes" => @num_nodes,
164
- "learning_rate" => @learning_rate,
165
- "batch_size" => @batch_size,
166
- "activation" => @activation,
167
- "momentum" => @momentum,
168
- "weight_decay" => @weight_decay,
169
- "use_dropout" => @use_dropout,
170
- "dropout_ratio" => @dropout_ratio,
171
- "use_batch_norm" => @use_batch_norm,
172
- "weights" => @weights.map(&:to_a),
173
- "biases" => @biases.map(&:to_a),
174
- }
175
- if @use_batch_norm
176
- json_batch_norm = {
177
- "gammas" => @gammas,
178
- "betas" => @betas
179
- }
180
- json.merge!(json_batch_norm)
181
- end
182
- File.write(file_name, JSON.dump(json))
183
- end
184
-
185
- private
186
-
187
- def init_weight_and_bias
188
- @weights = Array.new(@num_nodes.length - 1)
189
- @biases = Array.new(@num_nodes.length - 1)
190
- @weight_amounts = Array.new(@num_nodes.length - 1, 0)
191
- @bias_amounts = Array.new(@num_nodes.length - 1, 0)
192
- @num_nodes[0...-1].each_index do |i|
193
- weight = SFloat.new(@num_nodes[i], @num_nodes[i + 1]).rand_norm
194
- bias = SFloat.new(@num_nodes[i + 1]).rand_norm
195
- if @activation[0] == :relu
196
- @weights[i] = weight / Math.sqrt(@num_nodes[i]) * Math.sqrt(2)
197
- @biases[i] = bias / Math.sqrt(@num_nodes[i]) * Math.sqrt(2)
198
- else
199
- @weights[i] = weight / Math.sqrt(@num_nodes[i])
200
- @biases[i] = bias / Math.sqrt(@num_nodes[i])
201
- end
202
- end
203
- end
204
-
205
- def init_gamma_and_beta
206
- @gammas = Array.new(@num_nodes.length - 2, 1)
207
- @betas = Array.new(@num_nodes.length - 2, 0)
208
- @gamma_amounts = Array.new(@num_nodes.length - 2, 0)
209
- @beta_amounts = Array.new(@num_nodes.length - 2, 0)
210
- end
211
-
212
- def init_layers
213
- @layers = []
214
- @num_nodes[0...-2].each_index do |i|
215
- @layers << Affine.new(self, i)
216
- @layers << BatchNorm.new(self, i) if @use_batch_norm
217
- @layers << case @activation[0]
218
- when :sigmoid
219
- Sigmoid.new
220
- when :relu
221
- ReLU.new
222
- end
223
- @layers << Dropout.new(self) if @use_dropout
224
- end
225
- @layers << Affine.new(self, -1)
226
- @layers << case @activation[1]
227
- when :identity
228
- Identity.new(self)
229
- when :softmax
230
- Softmax.new(self)
231
- end
232
- end
233
-
234
- def forward(x, training = true)
235
- @training = training
236
- @layers.each do |layer|
237
- x = layer.forward(x)
238
- end
239
- x
240
- end
241
-
242
- def backward(y)
243
- dout = @layers[-1].backward(y)
244
- @layers[0...-1].reverse.each do |layer|
245
- dout = layer.backward(dout)
246
- end
247
- end
248
-
249
- def update_weight_and_bias
250
- @layers.select{|layer| layer.is_a?(Affine)}.each.with_index do |layer, i|
251
- weight_amount = layer.d_weight * @learning_rate
252
- bias_amount = layer.d_bias * @learning_rate
253
- if @momentum > 0
254
- weight_amount += @momentum * @weight_amounts[i]
255
- @weight_amounts[i] = weight_amount
256
- bias_amount += @momentum * @bias_amounts[i]
257
- @bias_amounts[i] = bias_amount
258
- end
259
- @weights[i] -= weight_amount
260
- @biases[i] -= bias_amount
261
- end
262
- end
263
-
264
- def update_gamma_and_beta
265
- @layers.select{|layer| layer.is_a?(BatchNorm)}.each.with_index do |layer, i|
266
- gamma_amount = layer.d_gamma * @learning_rate
267
- beta_amount = layer.d_beta * @learning_rate
268
- if @momentum > 0
269
- gamma_amount += @momentum * @gamma_amounts[i]
270
- @gamma_amounts[i] = gamma_amount
271
- beta_amount += @momentum * @beta_amounts[i]
272
- @beta_amounts[i] = beta_amount
273
- end
274
- @gammas[i] -= gamma_amount
275
- @betas[i] -= gamma_amount
276
- end
277
- end
278
- end
279
-
280
-
281
- class NN::Affine
282
- include Numo
283
-
284
- attr_reader :d_weight
285
- attr_reader :d_bias
286
-
287
- def initialize(nn, index)
288
- @nn = nn
289
- @index = index
290
- @d_weight = nil
291
- @d_bias = nil
292
- end
293
-
294
- def forward(x)
295
- @x = x
296
- @x.dot(@nn.weights[@index]) + @nn.biases[@index]
297
- end
298
-
299
- def backward(dout)
300
- x = @x.reshape(*@x.shape, 1)
301
- @d_weight = x.dot(dout.reshape(dout.shape[0], 1, dout.shape[1])).mean(0)
302
- if @nn.weight_decay > 0
303
- dridge = @nn.weight_decay * @nn.weights[@index]
304
- @d_weight += dridge
305
- end
306
- @d_bias = dout.mean
307
- dout.dot(@nn.weights[@index].transpose)
308
- end
309
- end
310
-
311
-
312
- class NN::Sigmoid
313
- include Numo
314
-
315
- def forward(x)
316
- @out = 1.0 / (1 + NMath.exp(-x))
317
- end
318
-
319
- def backward(dout)
320
- dout * (1.0 - @out) * @out
321
- end
322
- end
323
-
324
-
325
- class NN::ReLU
326
- def forward(x)
327
- @x = x.clone
328
- x[x < 0] = 0
329
- x
330
- end
331
-
332
- def backward(dout)
333
- @x[@x > 0] = 1.0
334
- @x[@x <= 0] = 0.0
335
- dout * @x
336
- end
337
- end
338
-
339
-
340
- class NN::Identity
341
- def initialize(nn)
342
- @nn = nn
343
- end
344
-
345
- def forward(x)
346
- @out = x
347
- end
348
-
349
- def backward(y)
350
- @out - y
351
- end
352
-
353
- def loss(y)
354
- ridge = 0.5 * @nn.weight_decay * @nn.weights.reduce(0){|sum, weight| sum + (weight ** 2).sum}
355
- 0.5 * ((@out - y) ** 2).sum / @nn.batch_size + ridge
356
- end
357
- end
358
-
359
-
360
- class NN::Softmax
361
- include Numo
362
-
363
- def initialize(nn)
364
- @nn = nn
365
- end
366
-
367
- def forward(x)
368
- @out = NMath.exp(x) / NMath.exp(x).sum(1).reshape(x.shape[0], 1)
369
- end
370
-
371
- def backward(y)
372
- @out - y
373
- end
374
-
375
- def loss(y)
376
- ridge = 0.5 * @nn.weight_decay * @nn.weights.reduce(0){|sum, weight| sum + (weight ** 2).sum}
377
- -(y * NMath.log(@out + 1e-7)).sum / @nn.batch_size + ridge
378
- end
379
- end
380
-
381
-
382
- class NN::Dropout
383
- include Numo
384
-
385
- def initialize(nn)
386
- @nn = nn
387
- @mask = nil
388
- end
389
-
390
- def forward(x)
391
- if @nn.training
392
- @mask = SFloat.ones(*x.shape).rand < @nn.dropout_ratio
393
- x[@mask] = 0
394
- else
395
- x *= (1 - @nn.dropout_ratio)
396
- end
397
- x
398
- end
399
-
400
- def backward(dout)
401
- dout[@mask] = 0 if @nn.training
402
- dout
403
- end
404
- end
405
-
406
-
407
- class NN::BatchNorm
408
- include Numo
409
-
410
- attr_reader :d_gamma
411
- attr_reader :d_beta
412
-
413
- def initialize(nn, index)
414
- @nn = nn
415
- @index = index
416
- end
417
-
418
- def forward(x)
419
- @x = x
420
- @mean = x.mean(0)
421
- @xc = x - @mean
422
- @var = (@xc ** 2).mean(0)
423
- @std = NMath.sqrt(@var + 1e-7)
424
- @xn = @xc / @std
425
- out = @nn.gammas[@index] * @xn + @nn.betas[@index]
426
- out.reshape(*@x.shape)
427
- end
428
-
429
- def backward(dout)
430
- @d_beta = dout.sum(0).mean
431
- @d_gamma = (@xn * dout).sum(0).mean
432
- dxn = @nn.gammas[@index] * dout
433
- dxc = dxn / @std
434
- dstd = -((dxn * @xc) / (@std ** 2)).sum(0)
435
- dvar = 0.5 * dstd / @std
436
- dxc += (2.0 / @nn.batch_size) * @xc * dvar
437
- dmean = dxc.sum(0)
438
- dx = dxc - dmean / @nn.batch_size
439
- dx.reshape(*@x.shape)
440
- end
441
- end