ruby-dnn 0.9.1 → 0.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '02929b358bbd4ff8c3107c54be0ce37ae86c20a3d345c0090f4be5bcd2ad8b32'
4
- data.tar.gz: 7641e5072f9bcdd4eb1bd93d173f78fe0fa11769ecca614ce44155dc7e310b96
3
+ metadata.gz: dc2897efaefa857bc21c4a4237c4c8d6fac3ec508708ba0935874050f8dab7f9
4
+ data.tar.gz: 2eaea58d620043e47c197f1ace3ec94f8108d84dbe3883a0455b64f39c8d6f82
5
5
  SHA512:
6
- metadata.gz: df47d323eda15b0f11dcf2153083bf05a8bd6c158227c3c93c0c9e0ab1f4679769fa8ad031a8bcbb886e5610880b2c95e26ad425a49ccdfe1b79b0ee280628de
7
- data.tar.gz: 86766ea8873229cb665d3e93e70142c246ad8b3e197543093d06ce674d66319a2f9a94a78cf089a0677df4a4966fdbfcfb0ffc95337e6465eabd2ab4d9ba7a1c
6
+ metadata.gz: e74b37d9d31af87cc833b4237b2f90051199a2020d645b733e5af2eb6f07b26b87debaac651818a95fca27ffeedb3c08a62c55191e3213b6b7a8c85762ba91fd
7
+ data.tar.gz: b4b25e79cd9ac57e464d70248178048473e4e6a26bcae34da9b5b63f109c336be5f3ccec700a23dc9dce427b7014be81b41ba547fa59ad3f5d9ebb5fa5d881c6
data/API-Reference.ja.md CHANGED
@@ -2,7 +2,7 @@
2
2
  ruby-dnnのAPIリファレンスです。このリファレンスでは、APIを利用するうえで必要となるクラスとメソッドしか記載していません。
3
3
  そのため、プログラムの詳細が必要な場合は、ソースコードを参照してください。
4
4
 
5
- 最終更新バージョン:0.9.0
5
+ 最終更新バージョン:0.9.2
6
6
 
7
7
  # module DNN
8
8
  ruby-dnnの名前空間をなすモジュールです。
@@ -119,6 +119,16 @@ Loss
119
119
  ### return
120
120
  なし。
121
121
 
122
+ ## def compile(optimizer, loss)
123
+ モデルを再コンパイルします。ただし、レイヤーのビルドは行いません。
124
+ ### arguments
125
+ * Optimizer optimizer
126
+ モデルが学習に使用するオプティマイザー。
127
+ * Loss loss
128
+ モデルが学習に使用する損失関数。
129
+ ### return
130
+ なし。
131
+
122
132
  ## def compiled?
123
133
  モデルがコンパイル済みであるか否かを取得します。
124
134
  ### arguments
@@ -485,6 +495,11 @@ Integer
485
495
  bool
486
496
  レイヤーがステートフルであるか否かを返します。
487
497
 
498
+ ## attr_reader :return_sequences
499
+ bool
500
+ trueを指定した場合、レイヤーのforward出力値において、時系列データ全てを返します。
501
+ falseを指定した場合、レイヤーのforward出力値において、時系列データの最後の値を返します。
502
+
488
503
  ## 【Instance methods】
489
504
 
490
505
  ## def initialize(num_nodes, stateful: false, return_sequences: true, weight_initializer: Initializers::RandomNormal.new, bias_initializer: Initializers::Zeros.new, l1_lamda: 0, l2_lambda: 0)
@@ -4,33 +4,34 @@ module DNN
4
4
  module Conv2DModule
5
5
  private
6
6
 
7
+ # img[bsize, out_h, out_w, channel] to col[bsize * out_h * out_w, fil_h * fil_w * ch]
7
8
  def im2col(img, out_h, out_w, fil_h, fil_w, strides)
8
9
  bsize = img.shape[0]
9
10
  ch = img.shape[3]
10
- col = Xumo::SFloat.zeros(bsize, ch, fil_h, fil_w, out_h, out_w)
11
- img = img.transpose(0, 3, 1, 2)
11
+ col = Xumo::SFloat.zeros(bsize, out_h, out_w, fil_h, fil_w, ch)
12
12
  (0...fil_h).each do |i|
13
13
  i_range = (i...(i + strides[0] * out_h)).step(strides[0]).to_a
14
14
  (0...fil_w).each do |j|
15
15
  j_range = (j...(j + strides[1] * out_w)).step(strides[1]).to_a
16
- col[true, true, i, j, true, true] = img[true, true, i_range, j_range]
16
+ col[true, true, true, i, j, true] = img[true, i_range, j_range, true]
17
17
  end
18
18
  end
19
- col.transpose(0, 4, 5, 2, 3, 1).reshape(bsize * out_h * out_w, fil_h * fil_w * ch)
19
+ col.reshape(bsize * out_h * out_w, fil_h * fil_w * ch)
20
20
  end
21
21
 
22
+ # col[bsize * out_h * out_w, fil_h * fil_w * ch] to img[bsize, out_h, out_w, channel]
22
23
  def col2im(col, img_shape, out_h, out_w, fil_h, fil_w, strides)
23
24
  bsize, img_h, img_w, ch = img_shape
24
- col = col.reshape(bsize, out_h, out_w, fil_h, fil_w, ch).transpose(0, 5, 3, 4, 1, 2)
25
- img = Xumo::SFloat.zeros(bsize, ch, img_h, img_w)
25
+ col = col.reshape(bsize, out_h, out_w, fil_h, fil_w, ch)
26
+ img = Xumo::SFloat.zeros(bsize, img_h, img_w, ch)
26
27
  (0...fil_h).each do |i|
27
28
  i_range = (i...(i + strides[0] * out_h)).step(strides[0]).to_a
28
29
  (0...fil_w).each do |j|
29
30
  j_range = (j...(j + strides[1] * out_w)).step(strides[1]).to_a
30
- img[true, true, i_range, j_range] += col[true, true, i, j, true, true]
31
+ img[true, i_range, j_range, true] += col[true, true, true, i, j, true]
31
32
  end
32
33
  end
33
- img.transpose(0, 2, 3, 1)
34
+ img
34
35
  end
35
36
 
36
37
  def padding(img, pad)
@@ -57,16 +58,39 @@ module DNN
57
58
  out_w = (prev_w - fil_w) / strides[1] + 1
58
59
  [out_h, out_w]
59
60
  end
61
+
62
+ def padding_size(prev_h, prev_w, out_h, out_w, strides)
63
+ pad_h = (prev_h.to_f / strides[0]).ceil - out_h
64
+ pad_w = (prev_w.to_f / strides[1]).ceil - out_w
65
+ [pad_h, pad_w]
66
+ end
60
67
  end
61
68
 
62
69
 
63
70
  class Conv2D < Connection
64
71
  include Conv2DModule
65
72
 
73
+ # @return [Integer] number of filters.
66
74
  attr_reader :num_filters
75
+ # @return [Array] Return filter size. filter size is of the form [height, width].
67
76
  attr_reader :filter_size
77
+ # @return [Array] Return stride length. stride length is of the form [height, width].
68
78
  attr_reader :strides
69
-
79
+
80
+ def self.load_hash(hash)
81
+ Conv2D.new(hash[:num_filters], hash[:filter_size],
82
+ weight_initializer: Utils.load_hash(hash[:weight_initializer]),
83
+ bias_initializer: Utils.load_hash(hash[:bias_initializer]),
84
+ strides: hash[:strides],
85
+ padding: hash[:padding],
86
+ l1_lambda: hash[:l1_lambda],
87
+ l2_lambda: hash[:l2_lambda])
88
+ end
89
+
90
+ # @param [Integer] num_filters number of filters.
91
+ # @param [Array or Integer] filter_size filter size. filter size is of the form [height, width].
92
+ # @param [Array or Integer] strides stride length. stride length is of the form [height, width].
93
+ # @param [Bool] padding Whether to padding.
70
94
  def initialize(num_filters, filter_size,
71
95
  weight_initializer: Initializers::RandomNormal.new,
72
96
  bias_initializer: Initializers::RandomNormal.new,
@@ -82,29 +106,18 @@ module DNN
82
106
  @padding = padding
83
107
  end
84
108
 
85
- def self.load_hash(hash)
86
- Conv2D.new(hash[:num_filters], hash[:filter_size],
87
- weight_initializer: Utils.load_hash(hash[:weight_initializer]),
88
- bias_initializer: Utils.load_hash(hash[:bias_initializer]),
89
- strides: hash[:strides],
90
- padding: hash[:padding],
91
- l1_lambda: hash[:l1_lambda],
92
- l2_lambda: hash[:l2_lambda])
93
- end
94
-
95
109
  def build(input_shape)
96
110
  super
97
111
  prev_h, prev_w = input_shape[0..1]
98
112
  @out_size = out_size(prev_h, prev_w, *@filter_size, @strides)
99
- out_w, out_h = @out_size
100
113
  if @padding
101
- @pad = [prev_h - out_h, prev_w - out_w]
102
- @out_size = [prev_h, prev_w]
114
+ @pad_size = padding_size(prev_h, prev_w, *@out_size, @strides)
115
+ @out_size = [@out_size[0] + @pad_size[0], @out_size[1] + @pad_size[1]]
103
116
  end
104
117
  end
105
118
 
106
119
  def forward(x)
107
- x = padding(x, @pad) if @padding
120
+ x = padding(x, @pad_size) if @padding
108
121
  @x_shape = x.shape
109
122
  @col = im2col(x, *@out_size, *@filter_size, @strides)
110
123
  out = @col.dot(@weight.data) + @bias.data
@@ -117,13 +130,30 @@ module DNN
117
130
  @bias.grad = dout.sum(0)
118
131
  dcol = dout.dot(@weight.data.transpose)
119
132
  dx = col2im(dcol, @x_shape, *@out_size, *@filter_size, @strides)
120
- @padding ? back_padding(dx, @pad) : dx
133
+ @padding ? back_padding(dx, @pad_size) : dx
121
134
  end
122
135
 
123
136
  def output_shape
124
137
  [*@out_size, @num_filters]
125
138
  end
126
139
 
140
+ # @return [Bool] whether to padding.
141
+ def padding?
142
+ @padding
143
+ end
144
+
145
+ # @return [Numo::SFloat] Convert weight to filter and return.
146
+ def filters
147
+ num_prev_filter = @input_shape[2]
148
+ @weight.data.reshape(*@filter_size, num_prev_filter, @num_filters)
149
+ end
150
+
151
+ # @param [Numo::SFloat] filters Convert weight to filters and set.
152
+ def filters=(filters)
153
+ num_prev_filter = @input_shape[2]
154
+ @weight.data = filters.reshape(@filter_size.reduce(:*) * num_prev_filter, @num_filters)
155
+ end
156
+
127
157
  def to_hash
128
158
  super({num_filters: @num_filters,
129
159
  filter_size: @filter_size,
@@ -135,7 +165,7 @@ module DNN
135
165
 
136
166
  def init_params
137
167
  num_prev_filter = @input_shape[2]
138
- @weight.data = Xumo::SFloat.new(num_prev_filter * @filter_size.reduce(:*), @num_filters)
168
+ @weight.data = Xumo::SFloat.new(@filter_size.reduce(:*) * num_prev_filter, @num_filters)
139
169
  @bias.data = Xumo::SFloat.new(@num_filters)
140
170
  super()
141
171
  end
@@ -146,13 +176,19 @@ module DNN
146
176
  class Pool2D < Layer
147
177
  include Conv2DModule
148
178
 
179
+ # @return [Array] Return pooling size. pooling size is of the form [height, width].
149
180
  attr_reader :pool_size
181
+ # @return [Array] Return stride length. stride length is of the form [height, width].
150
182
  attr_reader :strides
151
183
 
152
184
  def self.load_hash(pool2d_class, hash)
153
185
  pool2d_class.new(hash[:pool_size], strides: hash[:strides], padding: hash[:padding])
154
186
  end
155
187
 
188
+ # @param [Array or Integer] pool_size pooling size. pooling size is of the form [height, width].
189
+ # @param [Array or Integer or NilClass] strides stride length. stride length is of the form [height, width].
190
+ # If you set nil, treat pool_size as strides.
191
+ # @param [Bool] padding Whether to padding.
156
192
  def initialize(pool_size, strides: nil, padding: false)
157
193
  super()
158
194
  @pool_size = pool_size.is_a?(Integer) ? [pool_size, pool_size] : pool_size
@@ -169,10 +205,9 @@ module DNN
169
205
  prev_h, prev_w = input_shape[0..1]
170
206
  @num_channel = input_shape[2]
171
207
  @out_size = out_size(prev_h, prev_w, *@pool_size, @strides)
172
- out_w, out_h = @out_size
173
208
  if @padding
174
- @pad = [prev_h - out_h, prev_w - out_w]
175
- @out_size = [prev_h, prev_w]
209
+ @pad_size = padding_size(prev_h, prev_w, *@out_size, @strides)
210
+ @out_size = [@out_size[0] + @pad_size[0], @out_size[1] + @pad_size[1]]
176
211
  end
177
212
  end
178
213
 
@@ -180,6 +215,11 @@ module DNN
180
215
  [*@out_size, @num_channel]
181
216
  end
182
217
 
218
+ # @return [Bool] whether to padding.
219
+ def padding?
220
+ @padding
221
+ end
222
+
183
223
  def to_hash
184
224
  super({pool_size: @pool_size,
185
225
  strides: @strides,
@@ -194,10 +234,11 @@ module DNN
194
234
  end
195
235
 
196
236
  def forward(x)
197
- x = padding(x, @pad) if @padding
237
+ x = padding(x, @pad_size) if @padding
198
238
  @x_shape = x.shape
199
239
  col = im2col(x, *@out_size, *@pool_size, @strides)
200
- col = col.reshape(x.shape[0] * @out_size.reduce(:*) * x.shape[3], @pool_size.reduce(:*))
240
+ col = col.reshape(x.shape[0] * @out_size.reduce(:*), @pool_size.reduce(:*), x.shape[3]).transpose(0, 2, 1)
241
+ .reshape(x.shape[0] * @out_size.reduce(:*) * x.shape[3], @pool_size.reduce(:*))
201
242
  @max_index = col.max_index(1)
202
243
  col.max(1).reshape(x.shape[0], *@out_size, x.shape[3])
203
244
  end
@@ -205,9 +246,9 @@ module DNN
205
246
  def backward(dout)
206
247
  dmax = Xumo::SFloat.zeros(dout.size * @pool_size.reduce(:*))
207
248
  dmax[@max_index] = dout.flatten
208
- dcol = dmax.reshape(dout.shape[0..2].reduce(:*), dout.shape[3] * @pool_size.reduce(:*))
249
+ dcol = dmax.reshape(dout.shape[0..2].reduce(:*), @pool_size.reduce(:*) * dout.shape[3])
209
250
  dx = col2im(dcol, @x_shape, *@out_size, *@pool_size, @strides)
210
- @padding ? back_padding(dx, @pad) : dx
251
+ @padding ? back_padding(dx, @pad_size) : dx
211
252
  end
212
253
  end
213
254
 
@@ -218,10 +259,11 @@ module DNN
218
259
  end
219
260
 
220
261
  def forward(x)
221
- x = padding(x, @pad) if @padding
262
+ x = padding(x, @pad_size) if @padding
222
263
  @x_shape = x.shape
223
264
  col = im2col(x, *@out_size, *@pool_size, @strides)
224
- col = col.reshape(x.shape[0] * @out_size.reduce(:*) * x.shape[3], @pool_size.reduce(:*))
265
+ col = col.reshape(x.shape[0] * @out_size.reduce(:*), @pool_size.reduce(:*), x.shape[3]).transpose(0, 2, 1)
266
+ .reshape(x.shape[0] * @out_size.reduce(:*) * x.shape[3], @pool_size.reduce(:*))
225
267
  col.mean(1).reshape(x.shape[0], *@out_size, x.shape[3])
226
268
  end
227
269
 
@@ -234,14 +276,16 @@ module DNN
234
276
  end
235
277
  dcol = davg.reshape(dout.shape[0..2].reduce(:*), dout.shape[3] * @pool_size.reduce(:*))
236
278
  dx = col2im(dcol, @x_shape, *@out_size, *@pool_size, @strides)
237
- @padding ? back_padding(dx, @pad) : dx
279
+ @padding ? back_padding(dx, @pad_size) : dx
238
280
  end
239
281
  end
240
282
 
241
283
 
242
284
  class UnPool2D < Layer
285
+ # @return [Array] Return unpooling size. unpooling size is of the form [height, width].
243
286
  attr_reader :unpool_size
244
287
 
288
+ # @param [Array or Integer] unpool_size Unpooling size. unpooling size is of the form [height, width].
245
289
  def initialize(unpool_size)
246
290
  super()
247
291
  @unpool_size = unpool_size.is_a?(Integer) ? [unpool_size, unpool_size] : unpool_size
@@ -45,8 +45,10 @@ module DNN
45
45
 
46
46
  # This class is a superclass of all classes with learning parameters.
47
47
  class HasParamLayer < Layer
48
- attr_accessor :trainable # Setting false prevents learning of parameters.
49
- attr_reader :params # The parameters of the layer.
48
+ # @return [Bool] trainable Setting false prevents learning of parameters.
49
+ attr_accessor :trainable
50
+ # @return [Array] The parameters of the layer.
51
+ attr_reader :params
50
52
 
51
53
  def initialize
52
54
  super()
@@ -107,11 +109,19 @@ module DNN
107
109
 
108
110
  # It is a superclass of all connection layers.
109
111
  class Connection < HasParamLayer
110
- attr_reader :l1_lambda # L1 regularization
111
- attr_reader :l2_lambda # L2 regularization
112
+ # @return [DNN::Initializers] weight initializer.
112
113
  attr_reader :weight_initializer
114
+ # @return [DNN::Initializers] bias initializer.
113
115
  attr_reader :bias_initializer
114
-
116
+ # @return [Float] L1 regularization
117
+ attr_reader :l1_lambda
118
+ # @return [Float] L2 regularization
119
+ attr_reader :l2_lambda
120
+
121
+ # @param [DNN::Initializers] weight_initializer weight initializer.
122
+ # @param [DNN::Initializers] bias_initializer bias initializer.
123
+ # @param [Float] l1_lambda L1 regularization
124
+ # @param [Float] l2_lambda L2 regularization
115
125
  def initialize(weight_initializer: Initializers::RandomNormal.new,
116
126
  bias_initializer: Initializers::Zeros.new,
117
127
  l1_lambda: 0,
@@ -171,7 +181,9 @@ module DNN
171
181
  end
172
182
 
173
183
 
184
+ # Full connnection layer.
174
185
  class Dense < Connection
186
+ # @return [Integer] number of nodes.
175
187
  attr_reader :num_nodes
176
188
 
177
189
  def self.load_hash(hash)
@@ -181,7 +193,8 @@ module DNN
181
193
  l1_lambda: hash[:l1_lambda],
182
194
  l2_lambda: hash[:l2_lambda])
183
195
  end
184
-
196
+
197
+ # @param [Integer] num_nodes number of nodes.
185
198
  def initialize(num_nodes,
186
199
  weight_initializer: Initializers::RandomNormal.new,
187
200
  bias_initializer: Initializers::Zeros.new,
@@ -213,6 +226,8 @@ module DNN
213
226
 
214
227
  private
215
228
 
229
+ # TODO
230
+ # Change writing super() other than the first.
216
231
  def init_params
217
232
  num_prev_nodes = @input_shape[0]
218
233
  @weight.data = Xumo::SFloat.new(num_prev_nodes, @num_nodes)
@@ -266,16 +281,20 @@ module DNN
266
281
 
267
282
 
268
283
  class Dropout < Layer
284
+ # @return [Float] dropout ratio.
269
285
  attr_reader :dropout_ratio
286
+ # @return [Float] Use 'weight scaling inference rule'.
287
+ attr_reader :use_scale
270
288
 
271
289
  def self.load_hash(hash)
272
- self.new(hash[:dropout_ratio], hash[:seed])
290
+ self.new(hash[:dropout_ratio], seed: hash[:seed], use_scale: hash[:use_scale])
273
291
  end
274
292
 
275
- def initialize(dropout_ratio = 0.5, seed = rand(1 << 31))
293
+ def initialize(dropout_ratio = 0.5, seed: rand(1 << 31), use_scale: true)
276
294
  super()
277
295
  @dropout_ratio = dropout_ratio
278
296
  @seed = seed
297
+ @use_scale = use_scale
279
298
  @mask = nil
280
299
  end
281
300
 
@@ -285,29 +304,31 @@ module DNN
285
304
  @mask = Xumo::SFloat.ones(*x.shape).rand < @dropout_ratio
286
305
  x[@mask] = 0
287
306
  else
288
- x *= (1 - @dropout_ratio)
307
+ x *= (1 - @dropout_ratio) if @use_scale
289
308
  end
290
309
  x
291
310
  end
292
311
 
293
- def backward(dout, learning_phase)
294
- dout[@mask] = 0 if learning_phase
312
+ def backward(dout)
313
+ dout[@mask] = 0
295
314
  dout
296
315
  end
297
316
 
298
317
  def to_hash
299
- super({dropout_ratio: @dropout_ratio, seed: @seed})
318
+ super({dropout_ratio: @dropout_ratio, seed: @seed, use_scale: @use_scale})
300
319
  end
301
320
  end
302
321
 
303
322
 
304
323
  class BatchNormalization < HasParamLayer
324
+ # @return [Float] Exponential moving average of mean and variance.
305
325
  attr_reader :momentum
306
326
 
307
327
  def self.load_hash(hash)
308
328
  self.new(momentum: hash[:momentum])
309
329
  end
310
330
 
331
+ # @param [Float] momentum Exponential moving average of mean and variance.
311
332
  def initialize(momentum: 0.9)
312
333
  super()
313
334
  @momentum = momentum
@@ -330,7 +351,7 @@ module DNN
330
351
  @gamma.data * xn + @beta.data
331
352
  end
332
353
 
333
- def backward(dout, learning_phase)
354
+ def backward(dout)
334
355
  batch_size = dout.shape[0]
335
356
  @beta.grad = dout.sum(0)
336
357
  @gamma.grad = (@xn * dout).sum(0)
@@ -2,19 +2,16 @@ module DNN
2
2
  module Losses
3
3
 
4
4
  class Loss
5
- def forward(out, y)
6
- raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'forward'")
5
+ def forward(out, y, layers)
6
+ regularize = layers.select { |layer| layer.is_a?(Connection) }
7
+ .reduce(0) { |sum, layer| sum + layer.lasso + layer.ridge }
8
+ loss(out, y) + regularize
7
9
  end
8
10
 
9
11
  def backward(y)
10
12
  raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'backward'")
11
13
  end
12
14
 
13
- def regularize(layers)
14
- layers.select { |layer| layer.is_a?(Connection) }
15
- .reduce(0) { |sum, layer| sum + layer.lasso + layer.ridge }
16
- end
17
-
18
15
  def d_regularize(layers)
19
16
  layers.select { |layer| layer.is_a?(Connection) }.each do |layer|
20
17
  layer.d_lasso
@@ -25,10 +22,16 @@ module DNN
25
22
  def to_hash
26
23
  {class: self.class.name}
27
24
  end
25
+
26
+ private
27
+
28
+ def loss(out, y)
29
+ raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'loss'")
30
+ end
28
31
  end
29
32
 
30
33
  class MeanSquaredError < Loss
31
- def forward(out, y)
34
+ def loss(out, y)
32
35
  @out = out
33
36
  batch_size = y.shape[0]
34
37
  0.5 * ((out - y)**2).sum / batch_size
@@ -41,7 +44,7 @@ module DNN
41
44
 
42
45
 
43
46
  class MeanAbsoluteError < Loss
44
- def forward(out, y)
47
+ def loss(out, y)
45
48
  @out = out
46
49
  batch_size = y.shape[0]
47
50
  (out - y).abs.sum / batch_size
@@ -58,16 +61,18 @@ module DNN
58
61
 
59
62
  class HuberLoss < Loss
60
63
  def forward(out, y, layers)
64
+ @loss_value = super(out, y, layers)
65
+ end
66
+
67
+ def loss(out, y)
61
68
  @out = out
62
- loss = loss_l1(y)
63
- loss = loss > 1 ? loss : loss_l2(y)
64
- #@loss = loss + regularize(layers)
65
- @loss = loss
69
+ loss_value = loss_l1(y)
70
+ loss_value > 1 ? loss_value : loss_l2(y)
66
71
  end
67
72
 
68
73
  def backward(y)
69
74
  dout = @out - y
70
- if @loss > 1
75
+ if @loss_value > 1
71
76
  dout[dout >= 0] = 1
72
77
  dout[dout < 0] = -1
73
78
  end
@@ -89,7 +94,7 @@ module DNN
89
94
 
90
95
 
91
96
  class SoftmaxCrossEntropy < Loss
92
- def forward(x, y)
97
+ def loss(x, y)
93
98
  @out = Utils.softmax(x)
94
99
  batch_size = y.shape[0]
95
100
  -(y * NMath.log(@out + 1e-7)).sum / batch_size
@@ -102,7 +107,7 @@ module DNN
102
107
 
103
108
 
104
109
  class SigmoidCrossEntropy < Loss
105
- def forward(x, y)
110
+ def loss(x, y)
106
111
  @out = Utils.sigmoid(x)
107
112
  batch_size = y.shape[0]
108
113
  -(y * NMath.log(@out + 1e-7) + (1 - y) * NMath.log(1 - @out + 1e-7)).sum / batch_size
@@ -6,13 +6,20 @@ module DNN
6
6
 
7
7
  # This class deals with the model of the network.
8
8
  class Model
9
- attr_accessor :layers # All layers possessed by the model
10
- attr_accessor :trainable # Setting false prevents learning of parameters.
9
+ # @return [Array] All layers possessed by the model.
10
+ attr_accessor :layers
11
+ # @return [Bool] Setting false prevents learning of parameters.
12
+ attr_accessor :trainable
11
13
 
14
+ # Load marshal model.
15
+ # @param [String] file_name File name of marshal model to load.
12
16
  def self.load(file_name)
13
17
  Marshal.load(Zlib::Inflate.inflate(File.binread(file_name)))
14
18
  end
15
19
 
20
+ # Load json model.
21
+ # @param [String] json_str json string to load model.
22
+ # @return [DNN::Model]
16
23
  def self.load_json(json_str)
17
24
  hash = JSON.parse(json_str, symbolize_names: true)
18
25
  model = self.load_hash(hash)
@@ -33,6 +40,8 @@ module DNN
33
40
  @compiled = false
34
41
  end
35
42
 
43
+ # Load json model parameters.
44
+ # @param [String] json_str json string to load model parameters.
36
45
  def load_json_params(json_str)
37
46
  hash = JSON.parse(json_str, symbolize_names: true)
38
47
  has_param_layers_params = hash[:params]
@@ -49,6 +58,8 @@ module DNN
49
58
  end
50
59
  end
51
60
 
61
+ # Save the model in marshal format.
62
+ # @param [String] file_name name to save model.
52
63
  def save(file_name)
53
64
  bin = Zlib::Deflate.deflate(Marshal.dump(self))
54
65
  begin
@@ -60,12 +71,16 @@ module DNN
60
71
  end
61
72
  end
62
73
 
74
+ # Convert model to json string.
75
+ # @return [String] json string.
63
76
  def to_json
64
77
  hash = self.to_hash
65
78
  hash[:version] = VERSION
66
79
  JSON.pretty_generate(hash)
67
80
  end
68
81
 
82
+ # Convert model parameters to json string.
83
+ # @return [String] json string.
69
84
  def params_to_json
70
85
  has_param_layers = get_all_layers.select { |layer| layer.is_a?(Layers::HasParamLayer) }
71
86
  has_param_layers_params = has_param_layers.map do |layer|
@@ -78,6 +93,9 @@ module DNN
78
93
  JSON.dump(hash)
79
94
  end
80
95
 
96
+ # Add layer to the model.
97
+ # @param [DNN::Layers::Layer] layer Layer to add to the model.
98
+ # @return [DNN::Model] return self.
81
99
  def <<(layer)
82
100
  # Due to a bug in saving nested models, temporarily prohibit model nesting.
83
101
  # if !layer.is_a?(Layers::Layer) && !layer.is_a?(Model)
@@ -90,7 +108,11 @@ module DNN
90
108
  self
91
109
  end
92
110
 
111
+ # Set optimizer and loss to model and build all layers.
112
+ # @param [DNN::Optimizers::Optimizer] optimizer Optimizer to use for learning.
113
+ # @param [DNN::Losses::Loss] loss Lptimizer to use for learning.
93
114
  def compile(optimizer, loss)
115
+ raise DNN_Error.new("The model is already compiled.") if compiled?
94
116
  unless optimizer.is_a?(Optimizers::Optimizer)
95
117
  raise TypeError.new("optimizer:#{optimizer.class} is not an instance of DNN::Optimizers::Optimizer class.")
96
118
  end
@@ -105,6 +127,23 @@ module DNN
105
127
  layers_shape_check
106
128
  end
107
129
 
130
+ # Set optimizer and loss to model and recompile. But does not build layers.
131
+ # @param [DNN::Optimizers::Optimizer] optimizer Optimizer to use for learning.
132
+ # @param [DNN::Losses::Loss] loss Lptimizer to use for learning.
133
+ def recompile(optimizer, loss)
134
+ unless optimizer.is_a?(Optimizers::Optimizer)
135
+ raise TypeError.new("optimizer:#{optimizer.class} is not an instance of DNN::Optimizers::Optimizer class.")
136
+ end
137
+ unless loss.is_a?(Losses::Loss)
138
+ raise TypeError.new("loss:#{loss.class} is not an instance of DNN::Losses::Loss class.")
139
+ end
140
+ @compiled = true
141
+ layers_check
142
+ @optimizer = optimizer
143
+ @loss = loss
144
+ layers_shape_check
145
+ end
146
+
108
147
  def build(super_model = nil)
109
148
  @super_model = super_model
110
149
  shape = if super_model
@@ -122,28 +161,44 @@ module DNN
122
161
  end
123
162
  end
124
163
 
164
+ # @return [Array] Return the input shape of the model.
125
165
  def input_shape
126
166
  @layers.first.input_shape
127
167
  end
128
168
 
169
+ # @return [Array] Return the output shape of the model.
129
170
  def output_shape
130
171
  @layers.last.output_shape
131
172
  end
132
173
 
174
+ # @return [DNN::Optimizers::Optimizer] optimizer Return the optimizer to use for learning.
133
175
  def optimizer
134
176
  raise DNN_Error.new("The model is not compiled.") unless compiled?
135
177
  @optimizer ? @optimizer : @super_model.optimizer
136
178
  end
137
179
 
180
+ # @return [DNN::Losses::Loss] loss Return the loss to use for learning.
138
181
  def loss
139
182
  raise DNN_Error.new("The model is not compiled.") unless compiled?
140
183
  @loss ? @loss : @super_model.loss
141
184
  end
142
185
 
186
+ # @return [Bool] Returns whether the model is learning.
143
187
  def compiled?
144
188
  @compiled
145
189
  end
146
190
 
191
+ # Start training.
192
+ # Compile the model before use this method.
193
+ # @param [Numo::SFloat] x Input training data.
194
+ # @param [Numo::SFloat] y Output training data.
195
+ # @param [Integer] epochs Number of training.
196
+ # @param [Integer] batch_size Batch size used for one training.
197
+ # @param [Array or NilClass] test If you to test the model for every 1 epoch,
198
+ # specify [x_test, y_test]. Don't test to the model, specify nil.
199
+ # @param [Bool] verbose Set true to display the log. If false is set, the log is not displayed.
200
+ # @param [Proc] batch_proc Set proc to process per batch.
201
+ # @yield [epoch] Process performed before one training.
147
202
  def train(x, y, epochs,
148
203
  batch_size: 1,
149
204
  test: nil,
@@ -187,24 +242,29 @@ module DNN
187
242
  end
188
243
  end
189
244
 
245
+ # Training once.
246
+ # Compile the model before use this method.
247
+ # @param [Numo::SFloat] x Input training data.
248
+ # @param [Numo::SFloat] y Output training data.
249
+ # @yield [x, y] batch_proc Set proc to process per batch.
190
250
  def train_on_batch(x, y, &batch_proc)
191
251
  raise DNN_Error.new("The model is not compiled.") unless compiled?
192
252
  check_xy_type(x, y)
193
253
  input_data_shape_check(x, y)
194
254
  x, y = batch_proc.call(x, y) if batch_proc
195
255
  out = forward(x, true)
196
- loss_value = if @loss.is_a?(HuberLoss)
197
- @loss.forward(out, y, get_all_layers)
198
- else
199
- @loss.forward(out, y) + @loss.regularize(get_all_layers)
200
- end
256
+ loss_value = @loss.forward(out, y, get_all_layers)
201
257
  dout = @loss.backward(y)
202
- backward(dout, true)
258
+ backward(dout)
203
259
  @loss.d_regularize(get_all_layers)
204
260
  update
205
261
  loss_value
206
262
  end
207
263
 
264
+ # Evaluate model and get accurate of test data.
265
+ # @param [Numo::SFloat] x Input test data.
266
+ # @param [Numo::SFloat] y Output test data.
267
+ # @yield [x, y] batch_proc Set proc to process per batch.
208
268
  def accurate(x, y, batch_size = 100, &batch_proc)
209
269
  check_xy_type(x, y)
210
270
  input_data_shape_check(x, y)
@@ -231,22 +291,28 @@ module DNN
231
291
  end
232
292
  correct.to_f / x.shape[0]
233
293
  end
234
-
294
+
295
+ # Predict data.
296
+ # @param [Numo::SFloat] x Input data.
235
297
  def predict(x)
236
298
  check_xy_type(x)
237
299
  input_data_shape_check(x)
238
300
  forward(x, false)
239
301
  end
240
302
 
303
+ # Predict one data.
304
+ # @param [Numo::SFloat] x Input data. However, x is single data.
241
305
  def predict1(x)
242
306
  check_xy_type(x)
243
307
  predict(Xumo::SFloat.cast([x]))[0, false]
244
308
  end
245
309
 
310
+ # @return [DNN::Model] Copy this model.
246
311
  def copy
247
312
  Marshal.load(Marshal.dump(self))
248
313
  end
249
314
 
315
+ # Get the layer that the model has.
250
316
  def get_layer(*args)
251
317
  if args.length == 1
252
318
  index = args[0]
@@ -257,13 +323,17 @@ module DNN
257
323
  end
258
324
  end
259
325
 
326
+ # Get the all layers.
327
+ # @return [Array] all layers array.
260
328
  def get_all_layers
261
329
  @layers.map { |layer|
262
330
  layer.is_a?(Model) ? layer.get_all_layers : layer
263
331
  }.flatten
264
332
  end
265
333
 
266
- def forward(x, learning_phase)
334
+ # TODO
335
+ # It is not good to write the Layer class name directly in the Model class. I will fix it later.
336
+ def forward(x, learning_phase)01
267
337
  @layers.each do |layer|
268
338
  x = if layer.is_a?(Layers::Dropout) || layer.is_a?(Layers::BatchNormalization) || layer.is_a?(Model)
269
339
  layer.forward(x, learning_phase)
@@ -274,13 +344,9 @@ module DNN
274
344
  x
275
345
  end
276
346
 
277
- def backward(dout, learning_phase)
347
+ def backward(dout)
278
348
  @layers.reverse.each do |layer|
279
- if layer.is_a?(Layers::Dropout) || layer.is_a?(Layers::BatchNormalization) || layer.is_a?(Model)
280
- dout = layer.backward(dout, learning_phase)
281
- else
282
- dout = layer.backward(dout)
283
- end
349
+ dout = layer.backward(dout)
284
350
  end
285
351
  dout
286
352
  end
@@ -364,12 +430,6 @@ module DNN
364
430
  raise TypeError.new("y:#{y.class.name} is not an instance of #{Xumo::SFloat.name} class.")
365
431
  end
366
432
  end
367
-
368
- def type_check(var_name, var, type)
369
- unless var.is_a?(type)
370
- raise TypeError.new("#{var_name}:#{var.class} is not an instance of #{type} class.")
371
- end
372
- end
373
433
  end
374
434
 
375
435
  end
@@ -3,19 +3,20 @@ module DNN
3
3
 
4
4
  # Super class of all RNN classes.
5
5
  class RNN < Connection
6
- include Activations
6
+ include Initializers
7
7
 
8
+ # @return [Integer] number of nodes.
8
9
  attr_reader :num_nodes
10
+ # @return [Bool] Maintain state between batches.
9
11
  attr_reader :stateful
10
- attr_reader :weight
11
- attr_reader :weight2
12
- attr_reader :bias
12
+ # @return [Bool] Only the last of each cell of RNN is left.
13
+ attr_reader :return_sequences
13
14
 
14
15
  def initialize(num_nodes,
15
16
  stateful: false,
16
17
  return_sequences: true,
17
- weight_initializer: Initializers::RandomNormal.new,
18
- bias_initializer: Initializers::Zeros.new,
18
+ weight_initializer: RandomNormal.new,
19
+ bias_initializer: Zeros.new,
19
20
  l1_lambda: 0,
20
21
  l2_lambda: 0)
21
22
  super(weight_initializer: weight_initializer, bias_initializer: bias_initializer,
@@ -25,6 +26,8 @@ module DNN
25
26
  @return_sequences = return_sequences
26
27
  @layers = []
27
28
  @hidden = @params[:h] = Param.new
29
+ # TODO
30
+ # Change to a good name.
28
31
  @params[:weight2] = @weight2 = Param.new
29
32
  end
30
33
 
@@ -74,10 +77,7 @@ module DNN
74
77
  super(hash)
75
78
  end
76
79
 
77
- def shape
78
- @return_sequences ? [@time_length, @num_nodes] : [@num_nodes]
79
- end
80
-
80
+ # Reset the state of RNN.
81
81
  def reset_state
82
82
  @hidden.data = @hidden.data.fill(0) if @hidden.data
83
83
  end
@@ -152,6 +152,8 @@ module DNN
152
152
 
153
153
 
154
154
  class SimpleRNN < RNN
155
+ include Activations
156
+
155
157
  attr_reader :activation
156
158
 
157
159
  def self.load_hash(hash)
@@ -170,8 +172,8 @@ module DNN
170
172
  stateful: false,
171
173
  return_sequences: true,
172
174
  activation: Tanh.new,
173
- weight_initializer: Initializers::RandomNormal.new,
174
- bias_initializer: Initializers::Zeros.new,
175
+ weight_initializer: RandomNormal.new,
176
+ bias_initializer: Zeros.new,
175
177
  l1_lambda: 0,
176
178
  l2_lambda: 0)
177
179
  super(num_nodes,
@@ -273,8 +275,8 @@ module DNN
273
275
  def initialize(num_nodes,
274
276
  stateful: false,
275
277
  return_sequences: true,
276
- weight_initializer: Initializers::RandomNormal.new,
277
- bias_initializer: Initializers::Zeros.new,
278
+ weight_initializer: RandomNormal.new,
279
+ bias_initializer: Zeros.new,
278
280
  l1_lambda: 0,
279
281
  l2_lambda: 0)
280
282
  super
@@ -416,8 +418,8 @@ module DNN
416
418
  def initialize(num_nodes,
417
419
  stateful: false,
418
420
  return_sequences: true,
419
- weight_initializer: Initializers::RandomNormal.new,
420
- bias_initializer: Initializers::Zeros.new,
421
+ weight_initializer: RandomNormal.new,
422
+ bias_initializer: Zeros.new,
421
423
  l1_lambda: 0,
422
424
  l2_lambda: 0)
423
425
  super
@@ -20,6 +20,8 @@ module DNN
20
20
  dnn_class.new
21
21
  end
22
22
 
23
+ # TODO
24
+ # Don't want to write an implementation of the activation function in utils, so we will consider it later.
23
25
  def self.sigmoid(x)
24
26
  1 / (1 + NMath.exp(-x))
25
27
  end
data/lib/dnn/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module DNN
2
- VERSION = "0.9.1"
2
+ VERSION = "0.9.2"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ruby-dnn
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.1
4
+ version: 0.9.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - unagiootoro
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2019-05-04 00:00:00.000000000 Z
11
+ date: 2019-05-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: numo-narray