ruby-dnn 0.8.8 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -84,17 +84,17 @@ module DNN
84
84
 
85
85
  def self.load_hash(hash)
86
86
  Conv2D.new(hash[:num_filters], hash[:filter_size],
87
- weight_initializer: Util.load_hash(hash[:weight_initializer]),
88
- bias_initializer: Util.load_hash(hash[:bias_initializer]),
87
+ weight_initializer: Utils.load_hash(hash[:weight_initializer]),
88
+ bias_initializer: Utils.load_hash(hash[:bias_initializer]),
89
89
  strides: hash[:strides],
90
90
  padding: hash[:padding],
91
91
  l1_lambda: hash[:l1_lambda],
92
92
  l2_lambda: hash[:l2_lambda])
93
93
  end
94
94
 
95
- def build(model)
95
+ def build(input_shape)
96
96
  super
97
- prev_h, prev_w = prev_layer.shape[0..1]
97
+ prev_h, prev_w = input_shape[0..1]
98
98
  @out_size = out_size(prev_h, prev_w, *@filter_size, @strides)
99
99
  out_w, out_h = @out_size
100
100
  if @padding
@@ -120,7 +120,7 @@ module DNN
120
120
  @padding ? back_padding(dx, @pad) : dx
121
121
  end
122
122
 
123
- def shape
123
+ def output_shape
124
124
  [*@out_size, @num_filters]
125
125
  end
126
126
 
@@ -134,7 +134,7 @@ module DNN
134
134
  private
135
135
 
136
136
  def init_params
137
- num_prev_filter = prev_layer.shape[2]
137
+ num_prev_filter = @input_shape[2]
138
138
  @weight.data = Xumo::SFloat.new(num_prev_filter * @filter_size.reduce(:*), @num_filters)
139
139
  @bias.data = Xumo::SFloat.new(@num_filters)
140
140
  super()
@@ -164,10 +164,10 @@ module DNN
164
164
  @padding = padding
165
165
  end
166
166
 
167
- def build(model)
167
+ def build(input_shape)
168
168
  super
169
- prev_w, prev_h = prev_layer.shape[0..1]
170
- @num_channel = prev_layer.shape[2]
169
+ prev_h, prev_w = input_shape[0..1]
170
+ @num_channel = input_shape[2]
171
171
  @out_size = out_size(prev_h, prev_w, *@pool_size, @strides)
172
172
  out_w, out_h = @out_size
173
173
  if @padding
@@ -176,7 +176,7 @@ module DNN
176
176
  end
177
177
  end
178
178
 
179
- def shape
179
+ def output_shape
180
180
  [*@out_size, @num_channel]
181
181
  end
182
182
 
@@ -251,14 +251,14 @@ module DNN
251
251
  UnPool2D.new(hash[:unpool_size])
252
252
  end
253
253
 
254
- def build(model)
254
+ def build(input_shape)
255
255
  super
256
- prev_h, prev_w = prev_layer.shape[0..1]
256
+ prev_h, prev_w = input_shape[0..1]
257
257
  unpool_h, unpool_w = @unpool_size
258
258
  out_h = prev_h * unpool_h
259
259
  out_w = prev_w * unpool_w
260
260
  @out_size = [out_h, out_w]
261
- @num_channel = prev_layer.shape[2]
261
+ @num_channel = input_shape[2]
262
262
  end
263
263
 
264
264
  def forward(x)
@@ -275,7 +275,7 @@ module DNN
275
275
  dout[true, true, 0, true, 0, true].clone
276
276
  end
277
277
 
278
- def shape
278
+ def output_shape
279
279
  [*@out_size, @num_channel]
280
280
  end
281
281
 
@@ -0,0 +1,18 @@
1
+ class DNN::Dataset
2
+ def initialize(x_datas, y_datas)
3
+ @x_datas = x_datas
4
+ @y_datas = y_datas
5
+ @num_datas = x_datas.shape[0]
6
+ @indexes = @num_datas.times.to_a.shuffle
7
+ end
8
+
9
+ def get_batch(batch_size)
10
+ if @indexes.length < batch_size
11
+ @indexes = @num_datas.times.to_a.shuffle
12
+ end
13
+ batch_indexes = @indexes.shift(batch_size)
14
+ x_batch = @x_datas[batch_indexes, false]
15
+ y_batch = @y_datas[batch_indexes, false]
16
+ [x_batch, y_batch]
17
+ end
18
+ end
@@ -2,13 +2,16 @@ module DNN
2
2
  module Initializers
3
3
 
4
4
  class Initializer
5
- # Classes that inherit from this class must implement this method.
5
+ def initialize(seed = false)
6
+ @seed = seed == true ? rand(1 << 31) : seed
7
+ end
8
+
6
9
  def init_param(layer, param)
7
10
  raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'init_params'")
8
11
  end
9
12
 
10
13
  def to_hash(merge_hash = nil)
11
- hash = {class: self.class.name}
14
+ hash = {class: self.class.name, seed: @seed}
12
15
  hash.merge!(merge_hash) if merge_hash
13
16
  hash
14
17
  end
@@ -23,11 +26,14 @@ module DNN
23
26
 
24
27
 
25
28
  class Const < Initializer
29
+ attr_reader :const
30
+
26
31
  def self.load_hash(hash)
27
32
  self.new(hash[:const])
28
33
  end
29
34
 
30
35
  def initialize(const)
36
+ super()
31
37
  @const = const
32
38
  end
33
39
 
@@ -46,15 +52,17 @@ module DNN
46
52
  attr_reader :std
47
53
 
48
54
  def self.load_hash(hash)
49
- self.new(hash[:mean], hash[:std])
55
+ self.new(hash[:mean], hash[:std], hash[:seed])
50
56
  end
51
57
 
52
- def initialize(mean = 0, std = 0.05)
58
+ def initialize(mean = 0, std = 0.05, seed = true)
59
+ super(seed)
53
60
  @mean = mean
54
61
  @std = std
55
62
  end
56
63
 
57
64
  def init_param(layer, param)
65
+ Xumo::SFloat.srand(@seed)
58
66
  param.data = param.data.rand_norm(@mean, @std)
59
67
  end
60
68
 
@@ -69,15 +77,17 @@ module DNN
69
77
  attr_reader :max
70
78
 
71
79
  def self.load_hash(hash)
72
- self.new(hash[:min], hash[:max])
80
+ self.new(hash[:min], hash[:max], hash[:seed])
73
81
  end
74
82
 
75
- def initialize(min = -0.05, max = 0.05)
83
+ def initialize(min = -0.05, max = 0.05, seed = true)
84
+ super(seed)
76
85
  @min = min
77
86
  @max = max
78
87
  end
79
88
 
80
89
  def init_param(layer, param)
90
+ Xumo::SFloat.srand(@seed)
81
91
  param.data = param.data.rand(@min, @max)
82
92
  end
83
93
 
@@ -88,16 +98,26 @@ module DNN
88
98
 
89
99
 
90
100
  class Xavier < Initializer
101
+ def initialize(seed = true)
102
+ super
103
+ end
104
+
91
105
  def init_param(layer, param)
92
- num_prev_nodes = layer.prev_layer.shape.reduce(:*)
106
+ Xumo::SFloat.srand(@seed)
107
+ num_prev_nodes = layer.input_shape.reduce(:*)
93
108
  param.data = param.data.rand_norm / Math.sqrt(num_prev_nodes)
94
109
  end
95
110
  end
96
111
 
97
112
 
98
113
  class He < Initializer
114
+ def initialize(seed = true)
115
+ super
116
+ end
117
+
99
118
  def init_param(layer, param)
100
- num_prev_nodes = layer.prev_layer.shape.reduce(:*)
119
+ Xumo::SFloat.srand(@seed)
120
+ num_prev_nodes = layer.input_shape.reduce(:*)
101
121
  param.data = param.data.rand_norm / Math.sqrt(num_prev_nodes) * Math.sqrt(2)
102
122
  end
103
123
  end
@@ -3,13 +3,15 @@ module DNN
3
3
 
4
4
  # Super class of all optimizer classes.
5
5
  class Layer
6
+ attr_reader :input_shape
7
+
6
8
  def initialize
7
9
  @built = false
8
10
  end
9
11
 
10
12
  # Build the layer.
11
- def build(model)
12
- @model = model
13
+ def build(input_shape)
14
+ @input_shape = input_shape
13
15
  @built = true
14
16
  end
15
17
 
@@ -19,20 +21,17 @@ module DNN
19
21
  end
20
22
 
21
23
  # Forward propagation.
22
- # Classes that inherit from this class must implement this method.
23
24
  def forward(x)
24
25
  raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'forward'")
25
26
  end
26
27
 
27
28
  # Backward propagation.
28
- # Classes that inherit from this class must implement this method.
29
29
  def backward(dout)
30
30
  raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'update'")
31
31
  end
32
-
33
- # Get the shape of the layer.
34
- def shape
35
- prev_layer.shape
32
+
33
+ def output_shape
34
+ @input_shape
36
35
  end
37
36
 
38
37
  # Layer to a hash.
@@ -41,11 +40,6 @@ module DNN
41
40
  hash.merge!(merge_hash) if merge_hash
42
41
  hash
43
42
  end
44
-
45
- # Get the previous layer.
46
- def prev_layer
47
- @model.get_prev_layer(self)
48
- end
49
43
  end
50
44
 
51
45
 
@@ -60,8 +54,8 @@ module DNN
60
54
  @trainable = true
61
55
  end
62
56
 
63
- def build(model)
64
- @model = model
57
+ def build(input_shape)
58
+ @input_shape = input_shape
65
59
  unless @built
66
60
  @built = true
67
61
  init_params
@@ -69,14 +63,13 @@ module DNN
69
63
  end
70
64
 
71
65
  # Update the parameters.
72
- def update
73
- @model.optimizer.update(@params) if @trainable
66
+ def update(optimizer)
67
+ optimizer.update(@params) if @trainable
74
68
  end
75
69
 
76
70
  private
77
71
 
78
72
  # Initialize of the parameters.
79
- # Classes that inherit from this class must implement this method.
80
73
  def init_params
81
74
  raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'init_params'")
82
75
  end
@@ -84,15 +77,18 @@ module DNN
84
77
 
85
78
 
86
79
  class InputLayer < Layer
87
- attr_reader :shape
88
-
89
80
  def self.load_hash(hash)
90
- self.new(hash[:shape])
81
+ self.new(hash[:input_shape])
91
82
  end
92
83
 
93
- def initialize(dim_or_shape)
84
+ def initialize(input_dim_or_shape)
94
85
  super()
95
- @shape = dim_or_shape.is_a?(Array) ? dim_or_shape : [dim_or_shape]
86
+ @input_shape = input_dim_or_shape.is_a?(Array) ? input_dim_or_shape : [input_dim_or_shape]
87
+ end
88
+
89
+ def build
90
+ @built = true
91
+ @input_shape
96
92
  end
97
93
 
98
94
  def forward(x)
@@ -104,7 +100,7 @@ module DNN
104
100
  end
105
101
 
106
102
  def to_hash
107
- super({shape: @shape})
103
+ super({input_shape: @input_shape})
108
104
  end
109
105
  end
110
106
 
@@ -113,6 +109,8 @@ module DNN
113
109
  class Connection < HasParamLayer
114
110
  attr_reader :l1_lambda # L1 regularization
115
111
  attr_reader :l2_lambda # L2 regularization
112
+ attr_reader :weight_initializer
113
+ attr_reader :bias_initializer
116
114
 
117
115
  def initialize(weight_initializer: Initializers::RandomNormal.new,
118
116
  bias_initializer: Initializers::Zeros.new,
@@ -143,7 +141,7 @@ module DNN
143
141
  end
144
142
  end
145
143
 
146
- def dlasso
144
+ def d_lasso
147
145
  if @l1_lambda > 0
148
146
  dlasso = Xumo::SFloat.ones(*@weight.data.shape)
149
147
  dlasso[@weight.data < 0] = -1
@@ -151,7 +149,7 @@ module DNN
151
149
  end
152
150
  end
153
151
 
154
- def dridge
152
+ def d_ridge
155
153
  if @l2_lambda > 0
156
154
  @weight.grad += @l2_lambda * @weight.data
157
155
  end
@@ -178,8 +176,8 @@ module DNN
178
176
 
179
177
  def self.load_hash(hash)
180
178
  self.new(hash[:num_nodes],
181
- weight_initializer: Util.load_hash(hash[:weight_initializer]),
182
- bias_initializer: Util.load_hash(hash[:bias_initializer]),
179
+ weight_initializer: Utils.load_hash(hash[:weight_initializer]),
180
+ bias_initializer: Utils.load_hash(hash[:bias_initializer]),
183
181
  l1_lambda: hash[:l1_lambda],
184
182
  l2_lambda: hash[:l2_lambda])
185
183
  end
@@ -205,7 +203,7 @@ module DNN
205
203
  dout.dot(@weight.data.transpose)
206
204
  end
207
205
 
208
- def shape
206
+ def output_shape
209
207
  [@num_nodes]
210
208
  end
211
209
 
@@ -216,7 +214,7 @@ module DNN
216
214
  private
217
215
 
218
216
  def init_params
219
- num_prev_nodes = prev_layer.shape[0]
217
+ num_prev_nodes = @input_shape[0]
220
218
  @weight.data = Xumo::SFloat.new(num_prev_nodes, @num_nodes)
221
219
  @bias.data = Xumo::SFloat.new(@num_nodes)
222
220
  super()
@@ -226,90 +224,64 @@ module DNN
226
224
 
227
225
  class Flatten < Layer
228
226
  def forward(x)
229
- @shape = x.shape
230
- x.reshape(x.shape[0], x.shape[1..-1].reduce(:*))
227
+ x.reshape(x.shape[0], *output_shape)
231
228
  end
232
229
 
233
230
  def backward(dout)
234
- dout.reshape(*@shape)
231
+ dout.reshape(dout.shape[0], *@input_shape)
235
232
  end
236
-
237
- def shape
238
- [prev_layer.shape.reduce(:*)]
233
+
234
+ def output_shape
235
+ [@input_shape.reduce(:*)]
239
236
  end
240
237
  end
241
238
 
242
239
 
243
240
  class Reshape < Layer
244
- attr_reader :shape
245
-
246
- def initialize(shape)
247
- super()
248
- @shape = shape
249
- @x_shape = nil
241
+ def self.load_hash(hash)
242
+ self.new(hash[:output_shape])
250
243
  end
251
244
 
252
- def self.load_hash(hash)
253
- self.new(hash[:shape])
245
+ def initialize(output_shape)
246
+ super()
247
+ @output_shape = output_shape
254
248
  end
255
249
 
256
250
  def forward(x)
257
- @x_shape = x.shape
258
- x.reshape(x.shape[0], *@shape)
251
+ x.reshape(x.shape[0], *@output_shape)
259
252
  end
260
253
 
261
254
  def backward(dout)
262
- dout.reshape(*@x_shape)
263
- end
264
-
265
- def to_hash
266
- super({shape: @shape})
267
- end
268
- end
269
-
270
-
271
- class OutputLayer < Layer
272
- # Classes that inherit from this class must implement this method.
273
- def loss(x)
274
- raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'forward'")
255
+ dout.reshape(dout.shape[0], *@input_shape)
275
256
  end
276
257
 
277
- def dloss
278
- @model.get_all_layers.select { |layer| layer.is_a?(Connection) }.each do |layer|
279
- layer.dlasso
280
- layer.dridge
281
- end
258
+ def output_shape
259
+ @output_shape
282
260
  end
283
-
284
- private
285
261
 
286
- def lasso
287
- @model.get_all_layers.select { |layer| layer.is_a?(Connection) }
288
- .reduce(0) { |sum, layer| sum + layer.lasso }
289
- end
290
-
291
- def ridge
292
- @model.get_all_layers.select { |layer| layer.is_a?(Connection) }
293
- .reduce(0) { |sum, layer| sum + layer.ridge }
262
+ def to_hash
263
+ super({output_shape: @output_shape})
294
264
  end
295
265
  end
296
-
266
+
297
267
 
298
268
  class Dropout < Layer
299
269
  attr_reader :dropout_ratio
300
270
 
301
271
  def self.load_hash(hash)
302
- self.new(hash[:dropout_ratio])
272
+ self.new(hash[:dropout_ratio], hash[:seed])
303
273
  end
304
274
 
305
- def initialize(dropout_ratio = 0.5)
275
+ def initialize(dropout_ratio = 0.5, seed = rand(1 << 31))
306
276
  super()
307
277
  @dropout_ratio = dropout_ratio
278
+ @seed = seed
308
279
  @mask = nil
309
280
  end
310
-
311
- def forward(x)
312
- if @model.training?
281
+
282
+ def forward(x, learning_phase)
283
+ if learning_phase
284
+ Xumo::SFloat.srand(@seed)
313
285
  @mask = Xumo::SFloat.ones(*x.shape).rand < @dropout_ratio
314
286
  x[@mask] = 0
315
287
  else
@@ -318,13 +290,13 @@ module DNN
318
290
  x
319
291
  end
320
292
 
321
- def backward(dout)
322
- dout[@mask] = 0 if @model.training?
293
+ def backward(dout, learning_phase)
294
+ dout[@mask] = 0 if learning_phase
323
295
  dout
324
296
  end
325
297
 
326
298
  def to_hash
327
- super({dropout_ratio: @dropout_ratio})
299
+ super({dropout_ratio: @dropout_ratio, seed: @seed})
328
300
  end
329
301
  end
330
302
 
@@ -341,8 +313,8 @@ module DNN
341
313
  @momentum = momentum
342
314
  end
343
315
 
344
- def forward(x)
345
- if @model.training?
316
+ def forward(x, learning_phase)
317
+ if learning_phase
346
318
  mean = x.mean(0)
347
319
  @xc = x - mean
348
320
  var = (@xc**2).mean(0)
@@ -358,7 +330,7 @@ module DNN
358
330
  @gamma.data * xn + @beta.data
359
331
  end
360
332
 
361
- def backward(dout)
333
+ def backward(dout, learning_phase)
362
334
  batch_size = dout.shape[0]
363
335
  @beta.grad = dout.sum(0)
364
336
  @gamma.grad = (@xn * dout).sum(0)
@@ -378,10 +350,10 @@ module DNN
378
350
  private
379
351
 
380
352
  def init_params
381
- @params[:gamma] = @gamma = Param.new(Xumo::SFloat.ones(*shape))
382
- @params[:beta] = @beta = Param.new(Xumo::SFloat.zeros(*shape))
383
- @params[:running_mean] = @running_mean = Param.new(Xumo::SFloat.zeros(*shape))
384
- @params[:running_var] = @running_var = Param.new(Xumo::SFloat.zeros(*shape))
353
+ @params[:gamma] = @gamma = Param.new(Xumo::SFloat.ones(*output_shape))
354
+ @params[:beta] = @beta = Param.new(Xumo::SFloat.zeros(*output_shape))
355
+ @params[:running_mean] = @running_mean = Param.new(Xumo::SFloat.zeros(*output_shape))
356
+ @params[:running_var] = @running_var = Param.new(Xumo::SFloat.zeros(*output_shape))
385
357
  end
386
358
  end
387
359
  end