ruby-dnn 0.9.1 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/API-Reference.ja.md +16 -1
- data/lib/dnn/core/cnn_layers.rb +79 -35
- data/lib/dnn/core/layers.rb +34 -13
- data/lib/dnn/core/losses.rb +21 -16
- data/lib/dnn/core/model.rb +82 -22
- data/lib/dnn/core/rnn_layers.rb +18 -16
- data/lib/dnn/core/utils.rb +2 -0
- data/lib/dnn/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: dc2897efaefa857bc21c4a4237c4c8d6fac3ec508708ba0935874050f8dab7f9
|
4
|
+
data.tar.gz: 2eaea58d620043e47c197f1ace3ec94f8108d84dbe3883a0455b64f39c8d6f82
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e74b37d9d31af87cc833b4237b2f90051199a2020d645b733e5af2eb6f07b26b87debaac651818a95fca27ffeedb3c08a62c55191e3213b6b7a8c85762ba91fd
|
7
|
+
data.tar.gz: b4b25e79cd9ac57e464d70248178048473e4e6a26bcae34da9b5b63f109c336be5f3ccec700a23dc9dce427b7014be81b41ba547fa59ad3f5d9ebb5fa5d881c6
|
data/API-Reference.ja.md
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
ruby-dnnのAPIリファレンスです。このリファレンスでは、APIを利用するうえで必要となるクラスとメソッドしか記載していません。
|
3
3
|
そのため、プログラムの詳細が必要な場合は、ソースコードを参照してください。
|
4
4
|
|
5
|
-
最終更新バージョン:0.9.
|
5
|
+
最終更新バージョン:0.9.2
|
6
6
|
|
7
7
|
# module DNN
|
8
8
|
ruby-dnnの名前空間をなすモジュールです。
|
@@ -119,6 +119,16 @@ Loss
|
|
119
119
|
### return
|
120
120
|
なし。
|
121
121
|
|
122
|
+
## def compile(optimizer, loss)
|
123
|
+
モデルを再コンパイルします。ただし、レイヤーのビルドは行いません。
|
124
|
+
### arguments
|
125
|
+
* Optimizer optimizer
|
126
|
+
モデルが学習に使用するオプティマイザー。
|
127
|
+
* Loss loss
|
128
|
+
モデルが学習に使用する損失関数。
|
129
|
+
### return
|
130
|
+
なし。
|
131
|
+
|
122
132
|
## def compiled?
|
123
133
|
モデルがコンパイル済みであるか否かを取得します。
|
124
134
|
### arguments
|
@@ -485,6 +495,11 @@ Integer
|
|
485
495
|
bool
|
486
496
|
レイヤーがステートフルであるか否かを返します。
|
487
497
|
|
498
|
+
## attr_reader :return_sequences
|
499
|
+
bool
|
500
|
+
trueを指定した場合、レイヤーのforward出力値において、時系列データ全てを返します。
|
501
|
+
falseを指定した場合、レイヤーのforward出力値において、時系列データの最後の値を返します。
|
502
|
+
|
488
503
|
## 【Instance methods】
|
489
504
|
|
490
505
|
## def initialize(num_nodes, stateful: false, return_sequences: true, weight_initializer: Initializers::RandomNormal.new, bias_initializer: Initializers::Zeros.new, l1_lamda: 0, l2_lambda: 0)
|
data/lib/dnn/core/cnn_layers.rb
CHANGED
@@ -4,33 +4,34 @@ module DNN
|
|
4
4
|
module Conv2DModule
|
5
5
|
private
|
6
6
|
|
7
|
+
# img[bsize, out_h, out_w, channel] to col[bsize * out_h * out_w, fil_h * fil_w * ch]
|
7
8
|
def im2col(img, out_h, out_w, fil_h, fil_w, strides)
|
8
9
|
bsize = img.shape[0]
|
9
10
|
ch = img.shape[3]
|
10
|
-
col = Xumo::SFloat.zeros(bsize,
|
11
|
-
img = img.transpose(0, 3, 1, 2)
|
11
|
+
col = Xumo::SFloat.zeros(bsize, out_h, out_w, fil_h, fil_w, ch)
|
12
12
|
(0...fil_h).each do |i|
|
13
13
|
i_range = (i...(i + strides[0] * out_h)).step(strides[0]).to_a
|
14
14
|
(0...fil_w).each do |j|
|
15
15
|
j_range = (j...(j + strides[1] * out_w)).step(strides[1]).to_a
|
16
|
-
col[true, true, i, j, true
|
16
|
+
col[true, true, true, i, j, true] = img[true, i_range, j_range, true]
|
17
17
|
end
|
18
18
|
end
|
19
|
-
col.
|
19
|
+
col.reshape(bsize * out_h * out_w, fil_h * fil_w * ch)
|
20
20
|
end
|
21
21
|
|
22
|
+
# col[bsize * out_h * out_w, fil_h * fil_w * ch] to img[bsize, out_h, out_w, channel]
|
22
23
|
def col2im(col, img_shape, out_h, out_w, fil_h, fil_w, strides)
|
23
24
|
bsize, img_h, img_w, ch = img_shape
|
24
|
-
col = col.reshape(bsize, out_h, out_w, fil_h, fil_w, ch)
|
25
|
-
img = Xumo::SFloat.zeros(bsize,
|
25
|
+
col = col.reshape(bsize, out_h, out_w, fil_h, fil_w, ch)
|
26
|
+
img = Xumo::SFloat.zeros(bsize, img_h, img_w, ch)
|
26
27
|
(0...fil_h).each do |i|
|
27
28
|
i_range = (i...(i + strides[0] * out_h)).step(strides[0]).to_a
|
28
29
|
(0...fil_w).each do |j|
|
29
30
|
j_range = (j...(j + strides[1] * out_w)).step(strides[1]).to_a
|
30
|
-
img[true,
|
31
|
+
img[true, i_range, j_range, true] += col[true, true, true, i, j, true]
|
31
32
|
end
|
32
33
|
end
|
33
|
-
img
|
34
|
+
img
|
34
35
|
end
|
35
36
|
|
36
37
|
def padding(img, pad)
|
@@ -57,16 +58,39 @@ module DNN
|
|
57
58
|
out_w = (prev_w - fil_w) / strides[1] + 1
|
58
59
|
[out_h, out_w]
|
59
60
|
end
|
61
|
+
|
62
|
+
def padding_size(prev_h, prev_w, out_h, out_w, strides)
|
63
|
+
pad_h = (prev_h.to_f / strides[0]).ceil - out_h
|
64
|
+
pad_w = (prev_w.to_f / strides[1]).ceil - out_w
|
65
|
+
[pad_h, pad_w]
|
66
|
+
end
|
60
67
|
end
|
61
68
|
|
62
69
|
|
63
70
|
class Conv2D < Connection
|
64
71
|
include Conv2DModule
|
65
72
|
|
73
|
+
# @return [Integer] number of filters.
|
66
74
|
attr_reader :num_filters
|
75
|
+
# @return [Array] Return filter size. filter size is of the form [height, width].
|
67
76
|
attr_reader :filter_size
|
77
|
+
# @return [Array] Return stride length. stride length is of the form [height, width].
|
68
78
|
attr_reader :strides
|
69
|
-
|
79
|
+
|
80
|
+
def self.load_hash(hash)
|
81
|
+
Conv2D.new(hash[:num_filters], hash[:filter_size],
|
82
|
+
weight_initializer: Utils.load_hash(hash[:weight_initializer]),
|
83
|
+
bias_initializer: Utils.load_hash(hash[:bias_initializer]),
|
84
|
+
strides: hash[:strides],
|
85
|
+
padding: hash[:padding],
|
86
|
+
l1_lambda: hash[:l1_lambda],
|
87
|
+
l2_lambda: hash[:l2_lambda])
|
88
|
+
end
|
89
|
+
|
90
|
+
# @param [Integer] num_filters number of filters.
|
91
|
+
# @param [Array or Integer] filter_size filter size. filter size is of the form [height, width].
|
92
|
+
# @param [Array or Integer] strides stride length. stride length is of the form [height, width].
|
93
|
+
# @param [Bool] padding Whether to padding.
|
70
94
|
def initialize(num_filters, filter_size,
|
71
95
|
weight_initializer: Initializers::RandomNormal.new,
|
72
96
|
bias_initializer: Initializers::RandomNormal.new,
|
@@ -82,29 +106,18 @@ module DNN
|
|
82
106
|
@padding = padding
|
83
107
|
end
|
84
108
|
|
85
|
-
def self.load_hash(hash)
|
86
|
-
Conv2D.new(hash[:num_filters], hash[:filter_size],
|
87
|
-
weight_initializer: Utils.load_hash(hash[:weight_initializer]),
|
88
|
-
bias_initializer: Utils.load_hash(hash[:bias_initializer]),
|
89
|
-
strides: hash[:strides],
|
90
|
-
padding: hash[:padding],
|
91
|
-
l1_lambda: hash[:l1_lambda],
|
92
|
-
l2_lambda: hash[:l2_lambda])
|
93
|
-
end
|
94
|
-
|
95
109
|
def build(input_shape)
|
96
110
|
super
|
97
111
|
prev_h, prev_w = input_shape[0..1]
|
98
112
|
@out_size = out_size(prev_h, prev_w, *@filter_size, @strides)
|
99
|
-
out_w, out_h = @out_size
|
100
113
|
if @padding
|
101
|
-
@
|
102
|
-
@out_size = [
|
114
|
+
@pad_size = padding_size(prev_h, prev_w, *@out_size, @strides)
|
115
|
+
@out_size = [@out_size[0] + @pad_size[0], @out_size[1] + @pad_size[1]]
|
103
116
|
end
|
104
117
|
end
|
105
118
|
|
106
119
|
def forward(x)
|
107
|
-
x = padding(x, @
|
120
|
+
x = padding(x, @pad_size) if @padding
|
108
121
|
@x_shape = x.shape
|
109
122
|
@col = im2col(x, *@out_size, *@filter_size, @strides)
|
110
123
|
out = @col.dot(@weight.data) + @bias.data
|
@@ -117,13 +130,30 @@ module DNN
|
|
117
130
|
@bias.grad = dout.sum(0)
|
118
131
|
dcol = dout.dot(@weight.data.transpose)
|
119
132
|
dx = col2im(dcol, @x_shape, *@out_size, *@filter_size, @strides)
|
120
|
-
@padding ? back_padding(dx, @
|
133
|
+
@padding ? back_padding(dx, @pad_size) : dx
|
121
134
|
end
|
122
135
|
|
123
136
|
def output_shape
|
124
137
|
[*@out_size, @num_filters]
|
125
138
|
end
|
126
139
|
|
140
|
+
# @return [Bool] whether to padding.
|
141
|
+
def padding?
|
142
|
+
@padding
|
143
|
+
end
|
144
|
+
|
145
|
+
# @return [Numo::SFloat] Convert weight to filter and return.
|
146
|
+
def filters
|
147
|
+
num_prev_filter = @input_shape[2]
|
148
|
+
@weight.data.reshape(*@filter_size, num_prev_filter, @num_filters)
|
149
|
+
end
|
150
|
+
|
151
|
+
# @param [Numo::SFloat] filters Convert weight to filters and set.
|
152
|
+
def filters=(filters)
|
153
|
+
num_prev_filter = @input_shape[2]
|
154
|
+
@weight.data = filters.reshape(@filter_size.reduce(:*) * num_prev_filter, @num_filters)
|
155
|
+
end
|
156
|
+
|
127
157
|
def to_hash
|
128
158
|
super({num_filters: @num_filters,
|
129
159
|
filter_size: @filter_size,
|
@@ -135,7 +165,7 @@ module DNN
|
|
135
165
|
|
136
166
|
def init_params
|
137
167
|
num_prev_filter = @input_shape[2]
|
138
|
-
@weight.data = Xumo::SFloat.new(
|
168
|
+
@weight.data = Xumo::SFloat.new(@filter_size.reduce(:*) * num_prev_filter, @num_filters)
|
139
169
|
@bias.data = Xumo::SFloat.new(@num_filters)
|
140
170
|
super()
|
141
171
|
end
|
@@ -146,13 +176,19 @@ module DNN
|
|
146
176
|
class Pool2D < Layer
|
147
177
|
include Conv2DModule
|
148
178
|
|
179
|
+
# @return [Array] Return pooling size. pooling size is of the form [height, width].
|
149
180
|
attr_reader :pool_size
|
181
|
+
# @return [Array] Return stride length. stride length is of the form [height, width].
|
150
182
|
attr_reader :strides
|
151
183
|
|
152
184
|
def self.load_hash(pool2d_class, hash)
|
153
185
|
pool2d_class.new(hash[:pool_size], strides: hash[:strides], padding: hash[:padding])
|
154
186
|
end
|
155
187
|
|
188
|
+
# @param [Array or Integer] pool_size pooling size. pooling size is of the form [height, width].
|
189
|
+
# @param [Array or Integer or NilClass] strides stride length. stride length is of the form [height, width].
|
190
|
+
# If you set nil, treat pool_size as strides.
|
191
|
+
# @param [Bool] padding Whether to padding.
|
156
192
|
def initialize(pool_size, strides: nil, padding: false)
|
157
193
|
super()
|
158
194
|
@pool_size = pool_size.is_a?(Integer) ? [pool_size, pool_size] : pool_size
|
@@ -169,10 +205,9 @@ module DNN
|
|
169
205
|
prev_h, prev_w = input_shape[0..1]
|
170
206
|
@num_channel = input_shape[2]
|
171
207
|
@out_size = out_size(prev_h, prev_w, *@pool_size, @strides)
|
172
|
-
out_w, out_h = @out_size
|
173
208
|
if @padding
|
174
|
-
@
|
175
|
-
@out_size = [
|
209
|
+
@pad_size = padding_size(prev_h, prev_w, *@out_size, @strides)
|
210
|
+
@out_size = [@out_size[0] + @pad_size[0], @out_size[1] + @pad_size[1]]
|
176
211
|
end
|
177
212
|
end
|
178
213
|
|
@@ -180,6 +215,11 @@ module DNN
|
|
180
215
|
[*@out_size, @num_channel]
|
181
216
|
end
|
182
217
|
|
218
|
+
# @return [Bool] whether to padding.
|
219
|
+
def padding?
|
220
|
+
@padding
|
221
|
+
end
|
222
|
+
|
183
223
|
def to_hash
|
184
224
|
super({pool_size: @pool_size,
|
185
225
|
strides: @strides,
|
@@ -194,10 +234,11 @@ module DNN
|
|
194
234
|
end
|
195
235
|
|
196
236
|
def forward(x)
|
197
|
-
x = padding(x, @
|
237
|
+
x = padding(x, @pad_size) if @padding
|
198
238
|
@x_shape = x.shape
|
199
239
|
col = im2col(x, *@out_size, *@pool_size, @strides)
|
200
|
-
col = col.reshape(x.shape[0] * @out_size.reduce(:*)
|
240
|
+
col = col.reshape(x.shape[0] * @out_size.reduce(:*), @pool_size.reduce(:*), x.shape[3]).transpose(0, 2, 1)
|
241
|
+
.reshape(x.shape[0] * @out_size.reduce(:*) * x.shape[3], @pool_size.reduce(:*))
|
201
242
|
@max_index = col.max_index(1)
|
202
243
|
col.max(1).reshape(x.shape[0], *@out_size, x.shape[3])
|
203
244
|
end
|
@@ -205,9 +246,9 @@ module DNN
|
|
205
246
|
def backward(dout)
|
206
247
|
dmax = Xumo::SFloat.zeros(dout.size * @pool_size.reduce(:*))
|
207
248
|
dmax[@max_index] = dout.flatten
|
208
|
-
dcol = dmax.reshape(dout.shape[0..2].reduce(:*),
|
249
|
+
dcol = dmax.reshape(dout.shape[0..2].reduce(:*), @pool_size.reduce(:*) * dout.shape[3])
|
209
250
|
dx = col2im(dcol, @x_shape, *@out_size, *@pool_size, @strides)
|
210
|
-
@padding ? back_padding(dx, @
|
251
|
+
@padding ? back_padding(dx, @pad_size) : dx
|
211
252
|
end
|
212
253
|
end
|
213
254
|
|
@@ -218,10 +259,11 @@ module DNN
|
|
218
259
|
end
|
219
260
|
|
220
261
|
def forward(x)
|
221
|
-
x = padding(x, @
|
262
|
+
x = padding(x, @pad_size) if @padding
|
222
263
|
@x_shape = x.shape
|
223
264
|
col = im2col(x, *@out_size, *@pool_size, @strides)
|
224
|
-
col = col.reshape(x.shape[0] * @out_size.reduce(:*)
|
265
|
+
col = col.reshape(x.shape[0] * @out_size.reduce(:*), @pool_size.reduce(:*), x.shape[3]).transpose(0, 2, 1)
|
266
|
+
.reshape(x.shape[0] * @out_size.reduce(:*) * x.shape[3], @pool_size.reduce(:*))
|
225
267
|
col.mean(1).reshape(x.shape[0], *@out_size, x.shape[3])
|
226
268
|
end
|
227
269
|
|
@@ -234,14 +276,16 @@ module DNN
|
|
234
276
|
end
|
235
277
|
dcol = davg.reshape(dout.shape[0..2].reduce(:*), dout.shape[3] * @pool_size.reduce(:*))
|
236
278
|
dx = col2im(dcol, @x_shape, *@out_size, *@pool_size, @strides)
|
237
|
-
@padding ? back_padding(dx, @
|
279
|
+
@padding ? back_padding(dx, @pad_size) : dx
|
238
280
|
end
|
239
281
|
end
|
240
282
|
|
241
283
|
|
242
284
|
class UnPool2D < Layer
|
285
|
+
# @return [Array] Return unpooling size. unpooling size is of the form [height, width].
|
243
286
|
attr_reader :unpool_size
|
244
287
|
|
288
|
+
# @param [Array or Integer] unpool_size Unpooling size. unpooling size is of the form [height, width].
|
245
289
|
def initialize(unpool_size)
|
246
290
|
super()
|
247
291
|
@unpool_size = unpool_size.is_a?(Integer) ? [unpool_size, unpool_size] : unpool_size
|
data/lib/dnn/core/layers.rb
CHANGED
@@ -45,8 +45,10 @@ module DNN
|
|
45
45
|
|
46
46
|
# This class is a superclass of all classes with learning parameters.
|
47
47
|
class HasParamLayer < Layer
|
48
|
-
|
49
|
-
|
48
|
+
# @return [Bool] trainable Setting false prevents learning of parameters.
|
49
|
+
attr_accessor :trainable
|
50
|
+
# @return [Array] The parameters of the layer.
|
51
|
+
attr_reader :params
|
50
52
|
|
51
53
|
def initialize
|
52
54
|
super()
|
@@ -107,11 +109,19 @@ module DNN
|
|
107
109
|
|
108
110
|
# It is a superclass of all connection layers.
|
109
111
|
class Connection < HasParamLayer
|
110
|
-
|
111
|
-
attr_reader :l2_lambda # L2 regularization
|
112
|
+
# @return [DNN::Initializers] weight initializer.
|
112
113
|
attr_reader :weight_initializer
|
114
|
+
# @return [DNN::Initializers] bias initializer.
|
113
115
|
attr_reader :bias_initializer
|
114
|
-
|
116
|
+
# @return [Float] L1 regularization
|
117
|
+
attr_reader :l1_lambda
|
118
|
+
# @return [Float] L2 regularization
|
119
|
+
attr_reader :l2_lambda
|
120
|
+
|
121
|
+
# @param [DNN::Initializers] weight_initializer weight initializer.
|
122
|
+
# @param [DNN::Initializers] bias_initializer bias initializer.
|
123
|
+
# @param [Float] l1_lambda L1 regularization
|
124
|
+
# @param [Float] l2_lambda L2 regularization
|
115
125
|
def initialize(weight_initializer: Initializers::RandomNormal.new,
|
116
126
|
bias_initializer: Initializers::Zeros.new,
|
117
127
|
l1_lambda: 0,
|
@@ -171,7 +181,9 @@ module DNN
|
|
171
181
|
end
|
172
182
|
|
173
183
|
|
184
|
+
# Full connnection layer.
|
174
185
|
class Dense < Connection
|
186
|
+
# @return [Integer] number of nodes.
|
175
187
|
attr_reader :num_nodes
|
176
188
|
|
177
189
|
def self.load_hash(hash)
|
@@ -181,7 +193,8 @@ module DNN
|
|
181
193
|
l1_lambda: hash[:l1_lambda],
|
182
194
|
l2_lambda: hash[:l2_lambda])
|
183
195
|
end
|
184
|
-
|
196
|
+
|
197
|
+
# @param [Integer] num_nodes number of nodes.
|
185
198
|
def initialize(num_nodes,
|
186
199
|
weight_initializer: Initializers::RandomNormal.new,
|
187
200
|
bias_initializer: Initializers::Zeros.new,
|
@@ -213,6 +226,8 @@ module DNN
|
|
213
226
|
|
214
227
|
private
|
215
228
|
|
229
|
+
# TODO
|
230
|
+
# Change writing super() other than the first.
|
216
231
|
def init_params
|
217
232
|
num_prev_nodes = @input_shape[0]
|
218
233
|
@weight.data = Xumo::SFloat.new(num_prev_nodes, @num_nodes)
|
@@ -266,16 +281,20 @@ module DNN
|
|
266
281
|
|
267
282
|
|
268
283
|
class Dropout < Layer
|
284
|
+
# @return [Float] dropout ratio.
|
269
285
|
attr_reader :dropout_ratio
|
286
|
+
# @return [Float] Use 'weight scaling inference rule'.
|
287
|
+
attr_reader :use_scale
|
270
288
|
|
271
289
|
def self.load_hash(hash)
|
272
|
-
self.new(hash[:dropout_ratio], hash[:seed])
|
290
|
+
self.new(hash[:dropout_ratio], seed: hash[:seed], use_scale: hash[:use_scale])
|
273
291
|
end
|
274
292
|
|
275
|
-
def initialize(dropout_ratio = 0.5, seed
|
293
|
+
def initialize(dropout_ratio = 0.5, seed: rand(1 << 31), use_scale: true)
|
276
294
|
super()
|
277
295
|
@dropout_ratio = dropout_ratio
|
278
296
|
@seed = seed
|
297
|
+
@use_scale = use_scale
|
279
298
|
@mask = nil
|
280
299
|
end
|
281
300
|
|
@@ -285,29 +304,31 @@ module DNN
|
|
285
304
|
@mask = Xumo::SFloat.ones(*x.shape).rand < @dropout_ratio
|
286
305
|
x[@mask] = 0
|
287
306
|
else
|
288
|
-
x *= (1 - @dropout_ratio)
|
307
|
+
x *= (1 - @dropout_ratio) if @use_scale
|
289
308
|
end
|
290
309
|
x
|
291
310
|
end
|
292
311
|
|
293
|
-
def backward(dout
|
294
|
-
dout[@mask] = 0
|
312
|
+
def backward(dout)
|
313
|
+
dout[@mask] = 0
|
295
314
|
dout
|
296
315
|
end
|
297
316
|
|
298
317
|
def to_hash
|
299
|
-
super({dropout_ratio: @dropout_ratio, seed: @seed})
|
318
|
+
super({dropout_ratio: @dropout_ratio, seed: @seed, use_scale: @use_scale})
|
300
319
|
end
|
301
320
|
end
|
302
321
|
|
303
322
|
|
304
323
|
class BatchNormalization < HasParamLayer
|
324
|
+
# @return [Float] Exponential moving average of mean and variance.
|
305
325
|
attr_reader :momentum
|
306
326
|
|
307
327
|
def self.load_hash(hash)
|
308
328
|
self.new(momentum: hash[:momentum])
|
309
329
|
end
|
310
330
|
|
331
|
+
# @param [Float] momentum Exponential moving average of mean and variance.
|
311
332
|
def initialize(momentum: 0.9)
|
312
333
|
super()
|
313
334
|
@momentum = momentum
|
@@ -330,7 +351,7 @@ module DNN
|
|
330
351
|
@gamma.data * xn + @beta.data
|
331
352
|
end
|
332
353
|
|
333
|
-
def backward(dout
|
354
|
+
def backward(dout)
|
334
355
|
batch_size = dout.shape[0]
|
335
356
|
@beta.grad = dout.sum(0)
|
336
357
|
@gamma.grad = (@xn * dout).sum(0)
|
data/lib/dnn/core/losses.rb
CHANGED
@@ -2,19 +2,16 @@ module DNN
|
|
2
2
|
module Losses
|
3
3
|
|
4
4
|
class Loss
|
5
|
-
def forward(out, y)
|
6
|
-
|
5
|
+
def forward(out, y, layers)
|
6
|
+
regularize = layers.select { |layer| layer.is_a?(Connection) }
|
7
|
+
.reduce(0) { |sum, layer| sum + layer.lasso + layer.ridge }
|
8
|
+
loss(out, y) + regularize
|
7
9
|
end
|
8
10
|
|
9
11
|
def backward(y)
|
10
12
|
raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'backward'")
|
11
13
|
end
|
12
14
|
|
13
|
-
def regularize(layers)
|
14
|
-
layers.select { |layer| layer.is_a?(Connection) }
|
15
|
-
.reduce(0) { |sum, layer| sum + layer.lasso + layer.ridge }
|
16
|
-
end
|
17
|
-
|
18
15
|
def d_regularize(layers)
|
19
16
|
layers.select { |layer| layer.is_a?(Connection) }.each do |layer|
|
20
17
|
layer.d_lasso
|
@@ -25,10 +22,16 @@ module DNN
|
|
25
22
|
def to_hash
|
26
23
|
{class: self.class.name}
|
27
24
|
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def loss(out, y)
|
29
|
+
raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'loss'")
|
30
|
+
end
|
28
31
|
end
|
29
32
|
|
30
33
|
class MeanSquaredError < Loss
|
31
|
-
def
|
34
|
+
def loss(out, y)
|
32
35
|
@out = out
|
33
36
|
batch_size = y.shape[0]
|
34
37
|
0.5 * ((out - y)**2).sum / batch_size
|
@@ -41,7 +44,7 @@ module DNN
|
|
41
44
|
|
42
45
|
|
43
46
|
class MeanAbsoluteError < Loss
|
44
|
-
def
|
47
|
+
def loss(out, y)
|
45
48
|
@out = out
|
46
49
|
batch_size = y.shape[0]
|
47
50
|
(out - y).abs.sum / batch_size
|
@@ -58,16 +61,18 @@ module DNN
|
|
58
61
|
|
59
62
|
class HuberLoss < Loss
|
60
63
|
def forward(out, y, layers)
|
64
|
+
@loss_value = super(out, y, layers)
|
65
|
+
end
|
66
|
+
|
67
|
+
def loss(out, y)
|
61
68
|
@out = out
|
62
|
-
|
63
|
-
|
64
|
-
#@loss = loss + regularize(layers)
|
65
|
-
@loss = loss
|
69
|
+
loss_value = loss_l1(y)
|
70
|
+
loss_value > 1 ? loss_value : loss_l2(y)
|
66
71
|
end
|
67
72
|
|
68
73
|
def backward(y)
|
69
74
|
dout = @out - y
|
70
|
-
if @
|
75
|
+
if @loss_value > 1
|
71
76
|
dout[dout >= 0] = 1
|
72
77
|
dout[dout < 0] = -1
|
73
78
|
end
|
@@ -89,7 +94,7 @@ module DNN
|
|
89
94
|
|
90
95
|
|
91
96
|
class SoftmaxCrossEntropy < Loss
|
92
|
-
def
|
97
|
+
def loss(x, y)
|
93
98
|
@out = Utils.softmax(x)
|
94
99
|
batch_size = y.shape[0]
|
95
100
|
-(y * NMath.log(@out + 1e-7)).sum / batch_size
|
@@ -102,7 +107,7 @@ module DNN
|
|
102
107
|
|
103
108
|
|
104
109
|
class SigmoidCrossEntropy < Loss
|
105
|
-
def
|
110
|
+
def loss(x, y)
|
106
111
|
@out = Utils.sigmoid(x)
|
107
112
|
batch_size = y.shape[0]
|
108
113
|
-(y * NMath.log(@out + 1e-7) + (1 - y) * NMath.log(1 - @out + 1e-7)).sum / batch_size
|
data/lib/dnn/core/model.rb
CHANGED
@@ -6,13 +6,20 @@ module DNN
|
|
6
6
|
|
7
7
|
# This class deals with the model of the network.
|
8
8
|
class Model
|
9
|
-
|
10
|
-
attr_accessor :
|
9
|
+
# @return [Array] All layers possessed by the model.
|
10
|
+
attr_accessor :layers
|
11
|
+
# @return [Bool] Setting false prevents learning of parameters.
|
12
|
+
attr_accessor :trainable
|
11
13
|
|
14
|
+
# Load marshal model.
|
15
|
+
# @param [String] file_name File name of marshal model to load.
|
12
16
|
def self.load(file_name)
|
13
17
|
Marshal.load(Zlib::Inflate.inflate(File.binread(file_name)))
|
14
18
|
end
|
15
19
|
|
20
|
+
# Load json model.
|
21
|
+
# @param [String] json_str json string to load model.
|
22
|
+
# @return [DNN::Model]
|
16
23
|
def self.load_json(json_str)
|
17
24
|
hash = JSON.parse(json_str, symbolize_names: true)
|
18
25
|
model = self.load_hash(hash)
|
@@ -33,6 +40,8 @@ module DNN
|
|
33
40
|
@compiled = false
|
34
41
|
end
|
35
42
|
|
43
|
+
# Load json model parameters.
|
44
|
+
# @param [String] json_str json string to load model parameters.
|
36
45
|
def load_json_params(json_str)
|
37
46
|
hash = JSON.parse(json_str, symbolize_names: true)
|
38
47
|
has_param_layers_params = hash[:params]
|
@@ -49,6 +58,8 @@ module DNN
|
|
49
58
|
end
|
50
59
|
end
|
51
60
|
|
61
|
+
# Save the model in marshal format.
|
62
|
+
# @param [String] file_name name to save model.
|
52
63
|
def save(file_name)
|
53
64
|
bin = Zlib::Deflate.deflate(Marshal.dump(self))
|
54
65
|
begin
|
@@ -60,12 +71,16 @@ module DNN
|
|
60
71
|
end
|
61
72
|
end
|
62
73
|
|
74
|
+
# Convert model to json string.
|
75
|
+
# @return [String] json string.
|
63
76
|
def to_json
|
64
77
|
hash = self.to_hash
|
65
78
|
hash[:version] = VERSION
|
66
79
|
JSON.pretty_generate(hash)
|
67
80
|
end
|
68
81
|
|
82
|
+
# Convert model parameters to json string.
|
83
|
+
# @return [String] json string.
|
69
84
|
def params_to_json
|
70
85
|
has_param_layers = get_all_layers.select { |layer| layer.is_a?(Layers::HasParamLayer) }
|
71
86
|
has_param_layers_params = has_param_layers.map do |layer|
|
@@ -78,6 +93,9 @@ module DNN
|
|
78
93
|
JSON.dump(hash)
|
79
94
|
end
|
80
95
|
|
96
|
+
# Add layer to the model.
|
97
|
+
# @param [DNN::Layers::Layer] layer Layer to add to the model.
|
98
|
+
# @return [DNN::Model] return self.
|
81
99
|
def <<(layer)
|
82
100
|
# Due to a bug in saving nested models, temporarily prohibit model nesting.
|
83
101
|
# if !layer.is_a?(Layers::Layer) && !layer.is_a?(Model)
|
@@ -90,7 +108,11 @@ module DNN
|
|
90
108
|
self
|
91
109
|
end
|
92
110
|
|
111
|
+
# Set optimizer and loss to model and build all layers.
|
112
|
+
# @param [DNN::Optimizers::Optimizer] optimizer Optimizer to use for learning.
|
113
|
+
# @param [DNN::Losses::Loss] loss Lptimizer to use for learning.
|
93
114
|
def compile(optimizer, loss)
|
115
|
+
raise DNN_Error.new("The model is already compiled.") if compiled?
|
94
116
|
unless optimizer.is_a?(Optimizers::Optimizer)
|
95
117
|
raise TypeError.new("optimizer:#{optimizer.class} is not an instance of DNN::Optimizers::Optimizer class.")
|
96
118
|
end
|
@@ -105,6 +127,23 @@ module DNN
|
|
105
127
|
layers_shape_check
|
106
128
|
end
|
107
129
|
|
130
|
+
# Set optimizer and loss to model and recompile. But does not build layers.
|
131
|
+
# @param [DNN::Optimizers::Optimizer] optimizer Optimizer to use for learning.
|
132
|
+
# @param [DNN::Losses::Loss] loss Lptimizer to use for learning.
|
133
|
+
def recompile(optimizer, loss)
|
134
|
+
unless optimizer.is_a?(Optimizers::Optimizer)
|
135
|
+
raise TypeError.new("optimizer:#{optimizer.class} is not an instance of DNN::Optimizers::Optimizer class.")
|
136
|
+
end
|
137
|
+
unless loss.is_a?(Losses::Loss)
|
138
|
+
raise TypeError.new("loss:#{loss.class} is not an instance of DNN::Losses::Loss class.")
|
139
|
+
end
|
140
|
+
@compiled = true
|
141
|
+
layers_check
|
142
|
+
@optimizer = optimizer
|
143
|
+
@loss = loss
|
144
|
+
layers_shape_check
|
145
|
+
end
|
146
|
+
|
108
147
|
def build(super_model = nil)
|
109
148
|
@super_model = super_model
|
110
149
|
shape = if super_model
|
@@ -122,28 +161,44 @@ module DNN
|
|
122
161
|
end
|
123
162
|
end
|
124
163
|
|
164
|
+
# @return [Array] Return the input shape of the model.
|
125
165
|
def input_shape
|
126
166
|
@layers.first.input_shape
|
127
167
|
end
|
128
168
|
|
169
|
+
# @return [Array] Return the output shape of the model.
|
129
170
|
def output_shape
|
130
171
|
@layers.last.output_shape
|
131
172
|
end
|
132
173
|
|
174
|
+
# @return [DNN::Optimizers::Optimizer] optimizer Return the optimizer to use for learning.
|
133
175
|
def optimizer
|
134
176
|
raise DNN_Error.new("The model is not compiled.") unless compiled?
|
135
177
|
@optimizer ? @optimizer : @super_model.optimizer
|
136
178
|
end
|
137
179
|
|
180
|
+
# @return [DNN::Losses::Loss] loss Return the loss to use for learning.
|
138
181
|
def loss
|
139
182
|
raise DNN_Error.new("The model is not compiled.") unless compiled?
|
140
183
|
@loss ? @loss : @super_model.loss
|
141
184
|
end
|
142
185
|
|
186
|
+
# @return [Bool] Returns whether the model is learning.
|
143
187
|
def compiled?
|
144
188
|
@compiled
|
145
189
|
end
|
146
190
|
|
191
|
+
# Start training.
|
192
|
+
# Compile the model before use this method.
|
193
|
+
# @param [Numo::SFloat] x Input training data.
|
194
|
+
# @param [Numo::SFloat] y Output training data.
|
195
|
+
# @param [Integer] epochs Number of training.
|
196
|
+
# @param [Integer] batch_size Batch size used for one training.
|
197
|
+
# @param [Array or NilClass] test If you to test the model for every 1 epoch,
|
198
|
+
# specify [x_test, y_test]. Don't test to the model, specify nil.
|
199
|
+
# @param [Bool] verbose Set true to display the log. If false is set, the log is not displayed.
|
200
|
+
# @param [Proc] batch_proc Set proc to process per batch.
|
201
|
+
# @yield [epoch] Process performed before one training.
|
147
202
|
def train(x, y, epochs,
|
148
203
|
batch_size: 1,
|
149
204
|
test: nil,
|
@@ -187,24 +242,29 @@ module DNN
|
|
187
242
|
end
|
188
243
|
end
|
189
244
|
|
245
|
+
# Training once.
|
246
|
+
# Compile the model before use this method.
|
247
|
+
# @param [Numo::SFloat] x Input training data.
|
248
|
+
# @param [Numo::SFloat] y Output training data.
|
249
|
+
# @yield [x, y] batch_proc Set proc to process per batch.
|
190
250
|
def train_on_batch(x, y, &batch_proc)
|
191
251
|
raise DNN_Error.new("The model is not compiled.") unless compiled?
|
192
252
|
check_xy_type(x, y)
|
193
253
|
input_data_shape_check(x, y)
|
194
254
|
x, y = batch_proc.call(x, y) if batch_proc
|
195
255
|
out = forward(x, true)
|
196
|
-
loss_value =
|
197
|
-
@loss.forward(out, y, get_all_layers)
|
198
|
-
else
|
199
|
-
@loss.forward(out, y) + @loss.regularize(get_all_layers)
|
200
|
-
end
|
256
|
+
loss_value = @loss.forward(out, y, get_all_layers)
|
201
257
|
dout = @loss.backward(y)
|
202
|
-
backward(dout
|
258
|
+
backward(dout)
|
203
259
|
@loss.d_regularize(get_all_layers)
|
204
260
|
update
|
205
261
|
loss_value
|
206
262
|
end
|
207
263
|
|
264
|
+
# Evaluate model and get accurate of test data.
|
265
|
+
# @param [Numo::SFloat] x Input test data.
|
266
|
+
# @param [Numo::SFloat] y Output test data.
|
267
|
+
# @yield [x, y] batch_proc Set proc to process per batch.
|
208
268
|
def accurate(x, y, batch_size = 100, &batch_proc)
|
209
269
|
check_xy_type(x, y)
|
210
270
|
input_data_shape_check(x, y)
|
@@ -231,22 +291,28 @@ module DNN
|
|
231
291
|
end
|
232
292
|
correct.to_f / x.shape[0]
|
233
293
|
end
|
234
|
-
|
294
|
+
|
295
|
+
# Predict data.
|
296
|
+
# @param [Numo::SFloat] x Input data.
|
235
297
|
def predict(x)
|
236
298
|
check_xy_type(x)
|
237
299
|
input_data_shape_check(x)
|
238
300
|
forward(x, false)
|
239
301
|
end
|
240
302
|
|
303
|
+
# Predict one data.
|
304
|
+
# @param [Numo::SFloat] x Input data. However, x is single data.
|
241
305
|
def predict1(x)
|
242
306
|
check_xy_type(x)
|
243
307
|
predict(Xumo::SFloat.cast([x]))[0, false]
|
244
308
|
end
|
245
309
|
|
310
|
+
# @return [DNN::Model] Copy this model.
|
246
311
|
def copy
|
247
312
|
Marshal.load(Marshal.dump(self))
|
248
313
|
end
|
249
314
|
|
315
|
+
# Get the layer that the model has.
|
250
316
|
def get_layer(*args)
|
251
317
|
if args.length == 1
|
252
318
|
index = args[0]
|
@@ -257,13 +323,17 @@ module DNN
|
|
257
323
|
end
|
258
324
|
end
|
259
325
|
|
326
|
+
# Get the all layers.
|
327
|
+
# @return [Array] all layers array.
|
260
328
|
def get_all_layers
|
261
329
|
@layers.map { |layer|
|
262
330
|
layer.is_a?(Model) ? layer.get_all_layers : layer
|
263
331
|
}.flatten
|
264
332
|
end
|
265
333
|
|
266
|
-
|
334
|
+
# TODO
|
335
|
+
# It is not good to write the Layer class name directly in the Model class. I will fix it later.
|
336
|
+
def forward(x, learning_phase)01
|
267
337
|
@layers.each do |layer|
|
268
338
|
x = if layer.is_a?(Layers::Dropout) || layer.is_a?(Layers::BatchNormalization) || layer.is_a?(Model)
|
269
339
|
layer.forward(x, learning_phase)
|
@@ -274,13 +344,9 @@ module DNN
|
|
274
344
|
x
|
275
345
|
end
|
276
346
|
|
277
|
-
def backward(dout
|
347
|
+
def backward(dout)
|
278
348
|
@layers.reverse.each do |layer|
|
279
|
-
|
280
|
-
dout = layer.backward(dout, learning_phase)
|
281
|
-
else
|
282
|
-
dout = layer.backward(dout)
|
283
|
-
end
|
349
|
+
dout = layer.backward(dout)
|
284
350
|
end
|
285
351
|
dout
|
286
352
|
end
|
@@ -364,12 +430,6 @@ module DNN
|
|
364
430
|
raise TypeError.new("y:#{y.class.name} is not an instance of #{Xumo::SFloat.name} class.")
|
365
431
|
end
|
366
432
|
end
|
367
|
-
|
368
|
-
def type_check(var_name, var, type)
|
369
|
-
unless var.is_a?(type)
|
370
|
-
raise TypeError.new("#{var_name}:#{var.class} is not an instance of #{type} class.")
|
371
|
-
end
|
372
|
-
end
|
373
433
|
end
|
374
434
|
|
375
435
|
end
|
data/lib/dnn/core/rnn_layers.rb
CHANGED
@@ -3,19 +3,20 @@ module DNN
|
|
3
3
|
|
4
4
|
# Super class of all RNN classes.
|
5
5
|
class RNN < Connection
|
6
|
-
include
|
6
|
+
include Initializers
|
7
7
|
|
8
|
+
# @return [Integer] number of nodes.
|
8
9
|
attr_reader :num_nodes
|
10
|
+
# @return [Bool] Maintain state between batches.
|
9
11
|
attr_reader :stateful
|
10
|
-
|
11
|
-
attr_reader :
|
12
|
-
attr_reader :bias
|
12
|
+
# @return [Bool] Only the last of each cell of RNN is left.
|
13
|
+
attr_reader :return_sequences
|
13
14
|
|
14
15
|
def initialize(num_nodes,
|
15
16
|
stateful: false,
|
16
17
|
return_sequences: true,
|
17
|
-
weight_initializer:
|
18
|
-
bias_initializer:
|
18
|
+
weight_initializer: RandomNormal.new,
|
19
|
+
bias_initializer: Zeros.new,
|
19
20
|
l1_lambda: 0,
|
20
21
|
l2_lambda: 0)
|
21
22
|
super(weight_initializer: weight_initializer, bias_initializer: bias_initializer,
|
@@ -25,6 +26,8 @@ module DNN
|
|
25
26
|
@return_sequences = return_sequences
|
26
27
|
@layers = []
|
27
28
|
@hidden = @params[:h] = Param.new
|
29
|
+
# TODO
|
30
|
+
# Change to a good name.
|
28
31
|
@params[:weight2] = @weight2 = Param.new
|
29
32
|
end
|
30
33
|
|
@@ -74,10 +77,7 @@ module DNN
|
|
74
77
|
super(hash)
|
75
78
|
end
|
76
79
|
|
77
|
-
|
78
|
-
@return_sequences ? [@time_length, @num_nodes] : [@num_nodes]
|
79
|
-
end
|
80
|
-
|
80
|
+
# Reset the state of RNN.
|
81
81
|
def reset_state
|
82
82
|
@hidden.data = @hidden.data.fill(0) if @hidden.data
|
83
83
|
end
|
@@ -152,6 +152,8 @@ module DNN
|
|
152
152
|
|
153
153
|
|
154
154
|
class SimpleRNN < RNN
|
155
|
+
include Activations
|
156
|
+
|
155
157
|
attr_reader :activation
|
156
158
|
|
157
159
|
def self.load_hash(hash)
|
@@ -170,8 +172,8 @@ module DNN
|
|
170
172
|
stateful: false,
|
171
173
|
return_sequences: true,
|
172
174
|
activation: Tanh.new,
|
173
|
-
weight_initializer:
|
174
|
-
bias_initializer:
|
175
|
+
weight_initializer: RandomNormal.new,
|
176
|
+
bias_initializer: Zeros.new,
|
175
177
|
l1_lambda: 0,
|
176
178
|
l2_lambda: 0)
|
177
179
|
super(num_nodes,
|
@@ -273,8 +275,8 @@ module DNN
|
|
273
275
|
def initialize(num_nodes,
|
274
276
|
stateful: false,
|
275
277
|
return_sequences: true,
|
276
|
-
weight_initializer:
|
277
|
-
bias_initializer:
|
278
|
+
weight_initializer: RandomNormal.new,
|
279
|
+
bias_initializer: Zeros.new,
|
278
280
|
l1_lambda: 0,
|
279
281
|
l2_lambda: 0)
|
280
282
|
super
|
@@ -416,8 +418,8 @@ module DNN
|
|
416
418
|
def initialize(num_nodes,
|
417
419
|
stateful: false,
|
418
420
|
return_sequences: true,
|
419
|
-
weight_initializer:
|
420
|
-
bias_initializer:
|
421
|
+
weight_initializer: RandomNormal.new,
|
422
|
+
bias_initializer: Zeros.new,
|
421
423
|
l1_lambda: 0,
|
422
424
|
l2_lambda: 0)
|
423
425
|
super
|
data/lib/dnn/core/utils.rb
CHANGED
data/lib/dnn/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby-dnn
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.9.
|
4
|
+
version: 0.9.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- unagiootoro
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-05-
|
11
|
+
date: 2019-05-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: numo-narray
|