ruby-dnn 0.15.3 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +1 -9
- data/examples/api-examples/early_stopping_example.rb +1 -1
- data/examples/api-examples/initializer_example.rb +1 -1
- data/examples/api-examples/regularizer_example.rb +1 -1
- data/examples/api-examples/save_example.rb +1 -1
- data/examples/dcgan/dcgan.rb +3 -3
- data/examples/iris_example.rb +41 -17
- data/examples/mnist_define_by_run.rb +1 -1
- data/examples/pix2pix/dcgan.rb +157 -0
- data/examples/pix2pix/imgen.rb +27 -0
- data/examples/pix2pix/train.rb +52 -0
- data/lib/dnn.rb +2 -0
- data/lib/dnn/core/layers/activations.rb +37 -19
- data/lib/dnn/core/layers/basic_layers.rb +110 -25
- data/lib/dnn/core/layers/cnn_layers.rb +19 -21
- data/lib/dnn/core/layers/embedding.rb +3 -3
- data/lib/dnn/core/layers/math_layers.rb +169 -0
- data/lib/dnn/core/layers/merge_layers.rb +29 -24
- data/lib/dnn/core/layers/normalizations.rb +4 -2
- data/lib/dnn/core/layers/rnn_layers.rb +44 -36
- data/lib/dnn/core/link.rb +7 -2
- data/lib/dnn/core/losses.rb +54 -30
- data/lib/dnn/core/models.rb +47 -47
- data/lib/dnn/core/monkey_patch.rb +75 -0
- data/lib/dnn/core/optimizers.rb +10 -6
- data/lib/dnn/core/param.rb +17 -0
- data/lib/dnn/core/regularizers.rb +35 -33
- data/lib/dnn/core/tensor.rb +40 -0
- data/lib/dnn/core/utils.rb +1 -1
- data/lib/dnn/datasets/cifar10.rb +10 -9
- data/lib/dnn/datasets/cifar100.rb +10 -9
- data/lib/dnn/datasets/downloader.rb +1 -5
- data/lib/dnn/datasets/fashion-mnist.rb +4 -12
- data/lib/dnn/datasets/iris.rb +9 -9
- data/lib/dnn/datasets/mnist.rb +4 -12
- data/lib/dnn/datasets/stl-10.rb +6 -8
- data/lib/dnn/version.rb +1 -1
- data/ruby-dnn.gemspec +1 -1
- metadata +7 -5
- data/ext/cifar_loader/cifar_loader.c +0 -77
- data/ext/cifar_loader/extconf.rb +0 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ea912bca075445de925fda876e2b003bf0f63936100a3c14adf58bf810a6a9af
|
4
|
+
data.tar.gz: 41a40ee396cbda27faa7719cfcf2bed5e1337dc2963c1315561e186e05b81d77
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 16a12d59eb61b73f76b1361ddf2e5d3e2ca8929d5195f8f8d89299f829fa4a32284d2138ca6b8c4b8c5de5249c18e4a273390e9e80298e52d0e9a8109f4533a8
|
7
|
+
data.tar.gz: 859b180d139bff4f8904c939a41482574b1dd2c972fc83cf42743e078c3e04c28a1f8d740ba20dc970923bcb9411accefa86442fb7b77661de9ae015db1fc6b0
|
data/Rakefile
CHANGED
@@ -8,23 +8,15 @@ Rake::TestTask.new(:test) do |t|
|
|
8
8
|
t.test_files = FileList["test/*_test.rb", "test/layers_test/*_test.rb"]
|
9
9
|
end
|
10
10
|
|
11
|
-
task :build_cifar_loader do
|
12
|
-
sh "cd ext/cifar_loader; ruby extconf.rb; make"
|
13
|
-
end
|
14
|
-
|
15
11
|
task :build_rb_stb_image do
|
16
12
|
sh "cd ext/rb_stb_image; ruby extconf.rb; make"
|
17
13
|
end
|
18
14
|
|
19
|
-
task :clean_cifar_loader do
|
20
|
-
sh "cd ext/cifar_loader; make clean; unlink Makefile"
|
21
|
-
end
|
22
|
-
|
23
15
|
task :clean_rb_stb_image do
|
24
16
|
sh "cd ext/rb_stb_image; make clean; unlink Makefile"
|
25
17
|
end
|
26
18
|
|
27
|
-
task :default => [:test, :
|
19
|
+
task :default => [:test, :build_rb_stb_image]
|
28
20
|
|
29
21
|
task :doc do
|
30
22
|
src_list = Dir["lib/dnn.rb"]
|
data/examples/dcgan/dcgan.rb
CHANGED
@@ -19,7 +19,7 @@ class Generator < Model
|
|
19
19
|
@bn6 = BatchNormalization.new
|
20
20
|
end
|
21
21
|
|
22
|
-
def
|
22
|
+
def forward(x)
|
23
23
|
x = InputLayer.new(20).(x)
|
24
24
|
x = @l1.(x)
|
25
25
|
x = @bn1.(x)
|
@@ -63,7 +63,7 @@ class Discriminator < Model
|
|
63
63
|
@l6 = Dense.new(1)
|
64
64
|
end
|
65
65
|
|
66
|
-
def
|
66
|
+
def forward(x, trainable = true)
|
67
67
|
@l1.trainable = trainable
|
68
68
|
@l2.trainable = trainable
|
69
69
|
@l3.trainable = trainable
|
@@ -103,7 +103,7 @@ class DCGAN < Model
|
|
103
103
|
@dis = dis
|
104
104
|
end
|
105
105
|
|
106
|
-
def
|
106
|
+
def forward(x)
|
107
107
|
x = @gen.(x)
|
108
108
|
x = @dis.(x, false)
|
109
109
|
x
|
data/examples/iris_example.rb
CHANGED
@@ -3,7 +3,6 @@ require "dnn/datasets/iris"
|
|
3
3
|
# If you use numo/linalg then please uncomment out.
|
4
4
|
# require "numo/linalg/autoloader"
|
5
5
|
|
6
|
-
include DNN::Models
|
7
6
|
include DNN::Layers
|
8
7
|
include DNN::Optimizers
|
9
8
|
include DNN::Losses
|
@@ -15,19 +14,44 @@ x_test, y_test = x[100...150, true], y[100...150]
|
|
15
14
|
y_train = DNN::Utils.to_categorical(y_train, 3, Numo::SFloat)
|
16
15
|
y_test = DNN::Utils.to_categorical(y_test, 3, Numo::SFloat)
|
17
16
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
17
|
+
epochs = 1000
|
18
|
+
batch_size = 32
|
19
|
+
|
20
|
+
opt = Adam.new
|
21
|
+
lf = SoftmaxCrossEntropy.new
|
22
|
+
|
23
|
+
train_iter = DNN::Iterator.new(x_train, y_train)
|
24
|
+
test_iter = DNN::Iterator.new(x_test, y_test, random: false)
|
25
|
+
|
26
|
+
w1 = DNN::Param.new(Numo::SFloat.new(4, 16).rand_norm)
|
27
|
+
b1 = DNN::Param.new(Numo::SFloat.zeros(16))
|
28
|
+
w2 = DNN::Param.new(Numo::SFloat.new(16, 3).rand_norm)
|
29
|
+
b2 = DNN::Param.new(Numo::SFloat.zeros(3))
|
30
|
+
|
31
|
+
net = -> x, y do
|
32
|
+
h = Dot.(x, w1) + b1
|
33
|
+
h = Sigmoid.(h)
|
34
|
+
out = Dot.(h, w2) + b2
|
35
|
+
out
|
36
|
+
end
|
37
|
+
|
38
|
+
(1..epochs).each do |epoch|
|
39
|
+
train_iter.foreach(batch_size) do |x_batch, y_batch, step|
|
40
|
+
x = DNN::Tensor.convert(x_batch)
|
41
|
+
y = DNN::Tensor.convert(y_batch)
|
42
|
+
out = net.(x, y)
|
43
|
+
loss = lf.(out, y)
|
44
|
+
loss.link.backward(nil)
|
45
|
+
puts "epoch: #{epoch}, step: #{step}, loss = #{loss.data}"
|
46
|
+
opt.update([w1, b1, w2, b2])
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
correct = 0
|
51
|
+
test_iter.foreach(batch_size) do |x_batch, y_batch, step|
|
52
|
+
x = DNN::Tensor.convert(x_batch)
|
53
|
+
y = DNN::Tensor.convert(y_batch)
|
54
|
+
out = net.(x, y)
|
55
|
+
correct += out.data.max_index(axis: 1).eq(y_batch.max_index(axis: 1)).count
|
56
|
+
end
|
57
|
+
puts "correct = #{correct}"
|
@@ -0,0 +1,157 @@
|
|
1
|
+
include DNN::Models
|
2
|
+
include DNN::Layers
|
3
|
+
|
4
|
+
class Generator < Model
|
5
|
+
def initialize(input_shape)
|
6
|
+
super()
|
7
|
+
@input_shape = input_shape
|
8
|
+
@l1 = Conv2D.new(32, 4, padding: true)
|
9
|
+
@l2 = Conv2D.new(32, 4, strides: 2, padding: true)
|
10
|
+
@l3 = Conv2D.new(64, 4, padding: true)
|
11
|
+
@l4 = Conv2D.new(64, 4, strides: 2, padding: true)
|
12
|
+
@l5 = Conv2D.new(128, 4, padding: true)
|
13
|
+
@l6 = Conv2DTranspose.new(64, 4, strides: 2, padding: true)
|
14
|
+
@l7 = Conv2D.new(64, 4, padding: true)
|
15
|
+
@l8 = Conv2DTranspose.new(32, 4, strides: 2, padding: true)
|
16
|
+
@l9 = Conv2D.new(32, 4, padding: true)
|
17
|
+
@l10 = Conv2D.new(32, 4, padding: true)
|
18
|
+
@l11 = Conv2D.new(3, 4, padding: true)
|
19
|
+
@bn1 = BatchNormalization.new
|
20
|
+
@bn2 = BatchNormalization.new
|
21
|
+
@bn3 = BatchNormalization.new
|
22
|
+
@bn4 = BatchNormalization.new
|
23
|
+
@bn5 = BatchNormalization.new
|
24
|
+
@bn6 = BatchNormalization.new
|
25
|
+
@bn7 = BatchNormalization.new
|
26
|
+
@bn8 = BatchNormalization.new
|
27
|
+
@bn9 = BatchNormalization.new
|
28
|
+
end
|
29
|
+
|
30
|
+
def forward(x)
|
31
|
+
input = InputLayer.new(@input_shape).(x)
|
32
|
+
x = @l1.(input)
|
33
|
+
x = @bn1.(x)
|
34
|
+
h1 = ReLU.(x)
|
35
|
+
|
36
|
+
x = @l2.(h1)
|
37
|
+
x = @bn2.(x)
|
38
|
+
x = ReLU.(x)
|
39
|
+
|
40
|
+
x = @l3.(x)
|
41
|
+
x = @bn3.(x)
|
42
|
+
h2 = ReLU.(x)
|
43
|
+
|
44
|
+
x = @l4.(x)
|
45
|
+
x = @bn4.(x)
|
46
|
+
x = ReLU.(x)
|
47
|
+
|
48
|
+
x = @l5.(x)
|
49
|
+
x = @bn5.(x)
|
50
|
+
x = ReLU.(x)
|
51
|
+
|
52
|
+
x = @l6.(x)
|
53
|
+
x = @bn6.(x)
|
54
|
+
x = ReLU.(x)
|
55
|
+
|
56
|
+
x = @l7.(x)
|
57
|
+
x = @bn7.(x)
|
58
|
+
x = ReLU.(x)
|
59
|
+
x = Concatenate.(x, h2, axis: 3)
|
60
|
+
|
61
|
+
x = @l8.(x)
|
62
|
+
x = @bn8.(x)
|
63
|
+
x = ReLU.(x)
|
64
|
+
|
65
|
+
x = @l9.(x)
|
66
|
+
x = @bn9.(x)
|
67
|
+
x = ReLU.(x)
|
68
|
+
x = Concatenate.(x, h1, axis: 3)
|
69
|
+
|
70
|
+
x = @l10.(x)
|
71
|
+
x = ReLU.(x)
|
72
|
+
|
73
|
+
x = @l11.(x)
|
74
|
+
x = Tanh.(x)
|
75
|
+
x
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
class Discriminator < Model
|
80
|
+
def initialize(gen_input_shape, gen_output_shape)
|
81
|
+
super()
|
82
|
+
@gen_input_shape = gen_input_shape
|
83
|
+
@gen_output_shape = gen_output_shape
|
84
|
+
@l1_1 = Conv2D.new(32, 4, padding: true)
|
85
|
+
@l1_2 = Conv2D.new(32, 4, padding: true)
|
86
|
+
@l2 = Conv2D.new(32, 4, strides: 2, padding: true)
|
87
|
+
@l3 = Conv2D.new(32, 4, padding: true)
|
88
|
+
@l4 = Conv2D.new(64, 4, strides: 2, padding: true)
|
89
|
+
@l5 = Conv2D.new(64, 4, padding: true)
|
90
|
+
@l6 = Dense.new(1024)
|
91
|
+
@l7 = Dense.new(1)
|
92
|
+
@bn1 = BatchNormalization.new
|
93
|
+
@bn2 = BatchNormalization.new
|
94
|
+
@bn3 = BatchNormalization.new
|
95
|
+
@bn4 = BatchNormalization.new
|
96
|
+
@bn5 = BatchNormalization.new
|
97
|
+
@bn6 = BatchNormalization.new
|
98
|
+
end
|
99
|
+
|
100
|
+
def forward(inputs, trainable = true)
|
101
|
+
trainable_layers.each do |layer|
|
102
|
+
layer.trainable = trainable
|
103
|
+
end
|
104
|
+
|
105
|
+
input, images = *inputs
|
106
|
+
x = InputLayer.new(@gen_input_shape).(input)
|
107
|
+
x = @l1_1.(x)
|
108
|
+
x = @bn1.(x)
|
109
|
+
x1 = LeakyReLU.(x, 0.2)
|
110
|
+
|
111
|
+
x = InputLayer.new(@gen_output_shape).(images)
|
112
|
+
x = @l1_2.(x)
|
113
|
+
x = @bn2.(x)
|
114
|
+
x2 = LeakyReLU.(x, 0.2)
|
115
|
+
|
116
|
+
x = Concatenate.(x1, x2)
|
117
|
+
x = @l2.(x)
|
118
|
+
x = @bn3.(x)
|
119
|
+
x = LeakyReLU.(x, 0.2)
|
120
|
+
|
121
|
+
x = @l3.(x)
|
122
|
+
x = @bn4.(x)
|
123
|
+
x = LeakyReLU.(x, 0.2)
|
124
|
+
|
125
|
+
x = @l4.(x)
|
126
|
+
x = @bn5.(x)
|
127
|
+
x = LeakyReLU.(x, 0.2)
|
128
|
+
|
129
|
+
x = @l5.(x)
|
130
|
+
x = @bn6.(x)
|
131
|
+
x = LeakyReLU.(x, 0.2)
|
132
|
+
|
133
|
+
x = Flatten.(x)
|
134
|
+
x = @l6.(x)
|
135
|
+
x = LeakyReLU.(x, 0.2)
|
136
|
+
|
137
|
+
x = @l7.(x)
|
138
|
+
x
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
class DCGAN < Model
|
143
|
+
attr_reader :gen
|
144
|
+
attr_reader :dis
|
145
|
+
|
146
|
+
def initialize(gen, dis)
|
147
|
+
super()
|
148
|
+
@gen = gen
|
149
|
+
@dis = dis
|
150
|
+
end
|
151
|
+
|
152
|
+
def forward(input)
|
153
|
+
x = @gen.(input)
|
154
|
+
x = @dis.([input, x], false)
|
155
|
+
x
|
156
|
+
end
|
157
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require "dnn"
|
2
|
+
require "dnn/image"
|
3
|
+
require "dnn/datasets/cifar10"
|
4
|
+
require "numo/linalg/autoloader"
|
5
|
+
require_relative "dcgan"
|
6
|
+
|
7
|
+
def load_dataset
|
8
|
+
x, y = DNN::CIFAR10.load_test
|
9
|
+
x_out = Numo::SFloat.cast(x)
|
10
|
+
x_in = x_out.mean(axis: 3, keepdims: true)
|
11
|
+
x_in = (x_in / 127.5) - 1
|
12
|
+
x_out = (x_out / 127.5) - 1
|
13
|
+
[x_in, x_out]
|
14
|
+
end
|
15
|
+
|
16
|
+
batch_size = 100
|
17
|
+
|
18
|
+
dcgan = DCGAN.load("trained/dcgan_model_epoch20.marshal")
|
19
|
+
gen = dcgan.gen
|
20
|
+
|
21
|
+
x_in, x_out = load_dataset
|
22
|
+
images = gen.predict(x_in[0...batch_size, false])
|
23
|
+
|
24
|
+
batch_size.times do |i|
|
25
|
+
img = Numo::UInt8.cast(((images[i, false] + 1) * 127.5).round)
|
26
|
+
DNN::Image.write("img/img_#{i}.jpg", img)
|
27
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
# This example converts a CIFAR10 grayscale image to a color image.
|
2
|
+
|
3
|
+
require "dnn"
|
4
|
+
require "dnn/datasets/cifar10"
|
5
|
+
require "numo/linalg/autoloader"
|
6
|
+
require_relative "dcgan"
|
7
|
+
|
8
|
+
include DNN::Optimizers
|
9
|
+
include DNN::Losses
|
10
|
+
|
11
|
+
def load_dataset
|
12
|
+
x, y = DNN::CIFAR10.load_train
|
13
|
+
x_out = Numo::SFloat.cast(x)
|
14
|
+
x_in = x_out.mean(axis: 3, keepdims: true)
|
15
|
+
x_in = (x_in / 127.5) - 1
|
16
|
+
x_out = (x_out / 127.5) - 1
|
17
|
+
[x_in, x_out]
|
18
|
+
end
|
19
|
+
|
20
|
+
epochs = 20
|
21
|
+
batch_size = 128
|
22
|
+
|
23
|
+
gen = Generator.new([32, 32, 1])
|
24
|
+
dis = Discriminator.new([32, 32, 1], [32, 32, 3])
|
25
|
+
dcgan = DCGAN.new(gen, dis)
|
26
|
+
|
27
|
+
dis.setup(Adam.new(alpha: 0.00001, beta1: 0.1), SigmoidCrossEntropy.new)
|
28
|
+
dcgan.setup(Adam.new(alpha: 0.0002, beta1: 0.5), SigmoidCrossEntropy.new)
|
29
|
+
|
30
|
+
x_in, x_out = load_dataset
|
31
|
+
|
32
|
+
iter1 = DNN::Iterator.new(x_in, x_out)
|
33
|
+
iter2 = DNN::Iterator.new(x_in, x_out)
|
34
|
+
num_batchs = x_in.shape[0] / batch_size
|
35
|
+
(1..epochs).each do |epoch|
|
36
|
+
num_batchs.times do |index|
|
37
|
+
x_in, x_out = iter1.next_batch(batch_size)
|
38
|
+
images = gen.predict(x_in)
|
39
|
+
y_real = Numo::SFloat.ones(batch_size, 1)
|
40
|
+
y_fake = Numo::SFloat.zeros(batch_size, 1)
|
41
|
+
dis_loss = dis.train_on_batch([x_in, x_out], y_real)
|
42
|
+
dis_loss += dis.train_on_batch([x_in, images], y_fake)
|
43
|
+
|
44
|
+
x_in, x_out = iter2.next_batch(batch_size)
|
45
|
+
dcgan_loss = dcgan.train_on_batch(x_in, y_real)
|
46
|
+
|
47
|
+
puts "epoch: #{epoch}, index: #{index}, dis_loss: #{dis_loss}, dcgan_loss: #{dcgan_loss}"
|
48
|
+
end
|
49
|
+
iter1.reset
|
50
|
+
iter2.reset
|
51
|
+
dcgan.save("trained/dcgan_model_epoch#{epoch}.marshal")
|
52
|
+
end
|
data/lib/dnn.rb
CHANGED
@@ -8,6 +8,7 @@ module DNN
|
|
8
8
|
end
|
9
9
|
|
10
10
|
require_relative "dnn/version"
|
11
|
+
require_relative "dnn/core/monkey_patch"
|
11
12
|
require_relative "dnn/core/error"
|
12
13
|
require_relative "dnn/core/global"
|
13
14
|
require_relative "dnn/core/tensor"
|
@@ -22,6 +23,7 @@ require_relative "dnn/core/layers/merge_layers"
|
|
22
23
|
require_relative "dnn/core/layers/cnn_layers"
|
23
24
|
require_relative "dnn/core/layers/embedding"
|
24
25
|
require_relative "dnn/core/layers/rnn_layers"
|
26
|
+
require_relative "dnn/core/layers/math_layers"
|
25
27
|
require_relative "dnn/core/optimizers"
|
26
28
|
require_relative "dnn/core/losses"
|
27
29
|
require_relative "dnn/core/initializers"
|
@@ -2,70 +2,84 @@ module DNN
|
|
2
2
|
module Layers
|
3
3
|
|
4
4
|
class Sigmoid < Layer
|
5
|
-
|
5
|
+
include LayerNode
|
6
|
+
|
7
|
+
def forward_node(x)
|
6
8
|
@y = 1 / (1 + Xumo::NMath.exp(-x))
|
7
9
|
end
|
8
10
|
|
9
|
-
def
|
11
|
+
def backward_node(dy)
|
10
12
|
dy * (1 - @y) * @y
|
11
13
|
end
|
12
14
|
end
|
13
15
|
|
14
16
|
class Tanh < Layer
|
15
|
-
|
17
|
+
include LayerNode
|
18
|
+
|
19
|
+
def forward_node(x)
|
16
20
|
@y = Xumo::NMath.tanh(x)
|
17
21
|
end
|
18
22
|
|
19
|
-
def
|
23
|
+
def backward_node(dy)
|
20
24
|
dy * (1 - @y**2)
|
21
25
|
end
|
22
26
|
end
|
23
27
|
|
24
28
|
class Softsign < Layer
|
25
|
-
|
29
|
+
include LayerNode
|
30
|
+
|
31
|
+
def forward_node(x)
|
26
32
|
@x = x
|
27
33
|
x / (1 + x.abs)
|
28
34
|
end
|
29
35
|
|
30
|
-
def
|
36
|
+
def backward_node(dy)
|
31
37
|
dy * (1 / (1 + @x.abs)**2)
|
32
38
|
end
|
33
39
|
end
|
34
40
|
|
35
41
|
class Softplus < Layer
|
36
|
-
|
42
|
+
include LayerNode
|
43
|
+
|
44
|
+
def forward_node(x)
|
37
45
|
@x = x
|
38
46
|
Xumo::NMath.log(1 + Xumo::NMath.exp(x))
|
39
47
|
end
|
40
48
|
|
41
|
-
def
|
49
|
+
def backward_node(dy)
|
42
50
|
dy * (1 / (1 + Xumo::NMath.exp(-@x)))
|
43
51
|
end
|
44
52
|
end
|
45
53
|
|
46
54
|
class Swish < Layer
|
47
|
-
|
55
|
+
include LayerNode
|
56
|
+
|
57
|
+
def forward_node(x)
|
48
58
|
@x = x
|
49
59
|
@y = x * (1 / (1 + Xumo::NMath.exp(-x)))
|
50
60
|
end
|
51
61
|
|
52
|
-
def
|
62
|
+
def backward_node(dy)
|
53
63
|
dy * (@y + (1 / (1 + Xumo::NMath.exp(-@x))) * (1 - @y))
|
54
64
|
end
|
55
65
|
end
|
56
66
|
|
57
67
|
class ReLU < Layer
|
58
|
-
|
68
|
+
include LayerNode
|
69
|
+
|
70
|
+
def forward_node(x)
|
59
71
|
@x = x
|
60
72
|
Xumo::SFloat.maximum(0, x)
|
61
73
|
end
|
62
74
|
|
63
|
-
def
|
75
|
+
def backward_node(dy)
|
64
76
|
dy * Xumo::SFloat.cast(@x > 0)
|
65
77
|
end
|
66
78
|
end
|
67
79
|
|
68
80
|
class LeakyReLU < Layer
|
81
|
+
include LayerNode
|
82
|
+
|
69
83
|
attr_reader :alpha
|
70
84
|
|
71
85
|
# @param [Float] alpha The slope when the output value is negative.
|
@@ -74,14 +88,14 @@ module DNN
|
|
74
88
|
@alpha = alpha
|
75
89
|
end
|
76
90
|
|
77
|
-
def
|
91
|
+
def forward_node(x)
|
78
92
|
@x = x
|
79
93
|
a = Xumo::SFloat.ones(x.shape)
|
80
94
|
a[x <= 0] = @alpha
|
81
95
|
x * a
|
82
96
|
end
|
83
97
|
|
84
|
-
def
|
98
|
+
def backward_node(dy)
|
85
99
|
dx = Xumo::SFloat.ones(@x.shape)
|
86
100
|
dx[@x <= 0] = @alpha
|
87
101
|
dy * dx
|
@@ -97,6 +111,8 @@ module DNN
|
|
97
111
|
end
|
98
112
|
|
99
113
|
class ELU < Layer
|
114
|
+
include LayerNode
|
115
|
+
|
100
116
|
attr_reader :alpha
|
101
117
|
|
102
118
|
# @param [Float] alpha The slope when the output value is negative.
|
@@ -105,7 +121,7 @@ module DNN
|
|
105
121
|
@alpha = alpha
|
106
122
|
end
|
107
123
|
|
108
|
-
def
|
124
|
+
def forward_node(x)
|
109
125
|
@x = x
|
110
126
|
x1 = Xumo::SFloat.zeros(x.shape)
|
111
127
|
x1[x >= 0] = 1
|
@@ -116,7 +132,7 @@ module DNN
|
|
116
132
|
x1 + x2
|
117
133
|
end
|
118
134
|
|
119
|
-
def
|
135
|
+
def backward_node(dy)
|
120
136
|
dx = Xumo::SFloat.ones(@x.shape)
|
121
137
|
dx[@x < 0] = 0
|
122
138
|
dx2 = Xumo::SFloat.zeros(@x.shape)
|
@@ -135,12 +151,14 @@ module DNN
|
|
135
151
|
end
|
136
152
|
|
137
153
|
class Mish < Layer
|
138
|
-
|
154
|
+
include LayerNode
|
155
|
+
|
156
|
+
def forward_node(x)
|
139
157
|
@x = x
|
140
|
-
x * Xumo::NMath.tanh(Softplus.new.
|
158
|
+
x * Xumo::NMath.tanh(Softplus.new.forward_node(x))
|
141
159
|
end
|
142
160
|
|
143
|
-
def
|
161
|
+
def backward_node(dy)
|
144
162
|
omega = 4 * (@x + 1) + 4 * Xumo::NMath.exp(2 * @x) + Xumo::NMath.exp(3 * @x) + Xumo::NMath.exp(@x) * (4 * @x + 6)
|
145
163
|
delta = 2 * Xumo::NMath.exp(@x) + Xumo::NMath.exp(2 * @x) + 2
|
146
164
|
dy * (Xumo::NMath.exp(@x) * omega) / delta**2
|