ruby-dnn 0.15.3 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +1 -9
  3. data/examples/api-examples/early_stopping_example.rb +1 -1
  4. data/examples/api-examples/initializer_example.rb +1 -1
  5. data/examples/api-examples/regularizer_example.rb +1 -1
  6. data/examples/api-examples/save_example.rb +1 -1
  7. data/examples/dcgan/dcgan.rb +3 -3
  8. data/examples/iris_example.rb +41 -17
  9. data/examples/mnist_define_by_run.rb +1 -1
  10. data/examples/pix2pix/dcgan.rb +157 -0
  11. data/examples/pix2pix/imgen.rb +27 -0
  12. data/examples/pix2pix/train.rb +52 -0
  13. data/lib/dnn.rb +2 -0
  14. data/lib/dnn/core/layers/activations.rb +37 -19
  15. data/lib/dnn/core/layers/basic_layers.rb +110 -25
  16. data/lib/dnn/core/layers/cnn_layers.rb +19 -21
  17. data/lib/dnn/core/layers/embedding.rb +3 -3
  18. data/lib/dnn/core/layers/math_layers.rb +169 -0
  19. data/lib/dnn/core/layers/merge_layers.rb +29 -24
  20. data/lib/dnn/core/layers/normalizations.rb +4 -2
  21. data/lib/dnn/core/layers/rnn_layers.rb +44 -36
  22. data/lib/dnn/core/link.rb +7 -2
  23. data/lib/dnn/core/losses.rb +54 -30
  24. data/lib/dnn/core/models.rb +47 -47
  25. data/lib/dnn/core/monkey_patch.rb +75 -0
  26. data/lib/dnn/core/optimizers.rb +10 -6
  27. data/lib/dnn/core/param.rb +17 -0
  28. data/lib/dnn/core/regularizers.rb +35 -33
  29. data/lib/dnn/core/tensor.rb +40 -0
  30. data/lib/dnn/core/utils.rb +1 -1
  31. data/lib/dnn/datasets/cifar10.rb +10 -9
  32. data/lib/dnn/datasets/cifar100.rb +10 -9
  33. data/lib/dnn/datasets/downloader.rb +1 -5
  34. data/lib/dnn/datasets/fashion-mnist.rb +4 -12
  35. data/lib/dnn/datasets/iris.rb +9 -9
  36. data/lib/dnn/datasets/mnist.rb +4 -12
  37. data/lib/dnn/datasets/stl-10.rb +6 -8
  38. data/lib/dnn/version.rb +1 -1
  39. data/ruby-dnn.gemspec +1 -1
  40. metadata +7 -5
  41. data/ext/cifar_loader/cifar_loader.c +0 -77
  42. data/ext/cifar_loader/extconf.rb +0 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b213164ca3e4a7d781a673c4c49ec8dfbd6713468687a98c3739c0cf5629de73
4
- data.tar.gz: 92a8133dc0ab085d387199f92c2a602bb3782b810c20a931f7ee8de6e247f2a4
3
+ metadata.gz: ea912bca075445de925fda876e2b003bf0f63936100a3c14adf58bf810a6a9af
4
+ data.tar.gz: 41a40ee396cbda27faa7719cfcf2bed5e1337dc2963c1315561e186e05b81d77
5
5
  SHA512:
6
- metadata.gz: ea6652994f71d97142fe7357947701eff5f0c26708698a670a148e995fb10816b75b62b0bd60004d9ea99b6c837e5b599e790982ad12d010394e4ec7d6962135
7
- data.tar.gz: 9d474a7bdd120f0e39fcb05e11f0f8370100dd21e79440fe0408bbccf676890726623f516ada78ae6b8829bcf0799ceeea78218f64d74a70ca7f5dd9286bdae3
6
+ metadata.gz: 16a12d59eb61b73f76b1361ddf2e5d3e2ca8929d5195f8f8d89299f829fa4a32284d2138ca6b8c4b8c5de5249c18e4a273390e9e80298e52d0e9a8109f4533a8
7
+ data.tar.gz: 859b180d139bff4f8904c939a41482574b1dd2c972fc83cf42743e078c3e04c28a1f8d740ba20dc970923bcb9411accefa86442fb7b77661de9ae015db1fc6b0
data/Rakefile CHANGED
@@ -8,23 +8,15 @@ Rake::TestTask.new(:test) do |t|
8
8
  t.test_files = FileList["test/*_test.rb", "test/layers_test/*_test.rb"]
9
9
  end
10
10
 
11
- task :build_cifar_loader do
12
- sh "cd ext/cifar_loader; ruby extconf.rb; make"
13
- end
14
-
15
11
  task :build_rb_stb_image do
16
12
  sh "cd ext/rb_stb_image; ruby extconf.rb; make"
17
13
  end
18
14
 
19
- task :clean_cifar_loader do
20
- sh "cd ext/cifar_loader; make clean; unlink Makefile"
21
- end
22
-
23
15
  task :clean_rb_stb_image do
24
16
  sh "cd ext/rb_stb_image; make clean; unlink Makefile"
25
17
  end
26
18
 
27
- task :default => [:test, :build_cifar_loader, :build_rb_stb_image]
19
+ task :default => [:test, :build_rb_stb_image]
28
20
 
29
21
  task :doc do
30
22
  src_list = Dir["lib/dnn.rb"]
@@ -34,7 +34,7 @@ class MLP < Model
34
34
  @bn2 = BatchNormalization.new
35
35
  end
36
36
 
37
- def call(x)
37
+ def forward(x)
38
38
  x = InputLayer.(x)
39
39
  x = @l1.(x)
40
40
  x = @bn1.(x)
@@ -35,7 +35,7 @@ class MLP < Model
35
35
  @bn2 = BatchNormalization.new
36
36
  end
37
37
 
38
- def call(x)
38
+ def forward(x)
39
39
  x = InputLayer.(x)
40
40
  x = @l1.(x)
41
41
  x = @bn1.(x)
@@ -36,7 +36,7 @@ class MLP < Model
36
36
  @bn2 = BatchNormalization.new
37
37
  end
38
38
 
39
- def call(x)
39
+ def forward(x)
40
40
  x = InputLayer.(x)
41
41
  x = @l1.(x)
42
42
  x = @bn1.(x)
@@ -45,7 +45,7 @@ class MLP < Model
45
45
  @bn2 = BatchNormalization.new
46
46
  end
47
47
 
48
- def call(x)
48
+ def forward(x)
49
49
  x = InputLayer.new(784).(x)
50
50
  x = @l1.(x)
51
51
  x = @bn1.(x)
@@ -19,7 +19,7 @@ class Generator < Model
19
19
  @bn6 = BatchNormalization.new
20
20
  end
21
21
 
22
- def call(x)
22
+ def forward(x)
23
23
  x = InputLayer.new(20).(x)
24
24
  x = @l1.(x)
25
25
  x = @bn1.(x)
@@ -63,7 +63,7 @@ class Discriminator < Model
63
63
  @l6 = Dense.new(1)
64
64
  end
65
65
 
66
- def call(x, trainable = true)
66
+ def forward(x, trainable = true)
67
67
  @l1.trainable = trainable
68
68
  @l2.trainable = trainable
69
69
  @l3.trainable = trainable
@@ -103,7 +103,7 @@ class DCGAN < Model
103
103
  @dis = dis
104
104
  end
105
105
 
106
- def call(x)
106
+ def forward(x)
107
107
  x = @gen.(x)
108
108
  x = @dis.(x, false)
109
109
  x
@@ -3,7 +3,6 @@ require "dnn/datasets/iris"
3
3
  # If you use numo/linalg then please uncomment out.
4
4
  # require "numo/linalg/autoloader"
5
5
 
6
- include DNN::Models
7
6
  include DNN::Layers
8
7
  include DNN::Optimizers
9
8
  include DNN::Losses
@@ -15,19 +14,44 @@ x_test, y_test = x[100...150, true], y[100...150]
15
14
  y_train = DNN::Utils.to_categorical(y_train, 3, Numo::SFloat)
16
15
  y_test = DNN::Utils.to_categorical(y_test, 3, Numo::SFloat)
17
16
 
18
- model = Sequential.new
19
-
20
- model << InputLayer.new(4)
21
-
22
- model << Dense.new(64)
23
- model << ReLU.new
24
-
25
- model << Dense.new(3)
26
-
27
- model.setup(Adam.new, SoftmaxCrossEntropy.new)
28
-
29
- model.train(x_train, y_train, 500, batch_size: 32, test: [x_test, y_test])
30
-
31
- accuracy, loss = model.evaluate(x_test, y_test)
32
- puts "accuracy: #{accuracy}"
33
- puts "loss: #{loss}"
17
+ epochs = 1000
18
+ batch_size = 32
19
+
20
+ opt = Adam.new
21
+ lf = SoftmaxCrossEntropy.new
22
+
23
+ train_iter = DNN::Iterator.new(x_train, y_train)
24
+ test_iter = DNN::Iterator.new(x_test, y_test, random: false)
25
+
26
+ w1 = DNN::Param.new(Numo::SFloat.new(4, 16).rand_norm)
27
+ b1 = DNN::Param.new(Numo::SFloat.zeros(16))
28
+ w2 = DNN::Param.new(Numo::SFloat.new(16, 3).rand_norm)
29
+ b2 = DNN::Param.new(Numo::SFloat.zeros(3))
30
+
31
+ net = -> x, y do
32
+ h = Dot.(x, w1) + b1
33
+ h = Sigmoid.(h)
34
+ out = Dot.(h, w2) + b2
35
+ out
36
+ end
37
+
38
+ (1..epochs).each do |epoch|
39
+ train_iter.foreach(batch_size) do |x_batch, y_batch, step|
40
+ x = DNN::Tensor.convert(x_batch)
41
+ y = DNN::Tensor.convert(y_batch)
42
+ out = net.(x, y)
43
+ loss = lf.(out, y)
44
+ loss.link.backward(nil)
45
+ puts "epoch: #{epoch}, step: #{step}, loss = #{loss.data}"
46
+ opt.update([w1, b1, w2, b2])
47
+ end
48
+ end
49
+
50
+ correct = 0
51
+ test_iter.foreach(batch_size) do |x_batch, y_batch, step|
52
+ x = DNN::Tensor.convert(x_batch)
53
+ y = DNN::Tensor.convert(y_batch)
54
+ out = net.(x, y)
55
+ correct += out.data.max_index(axis: 1).eq(y_batch.max_index(axis: 1)).count
56
+ end
57
+ puts "correct = #{correct}"
@@ -28,7 +28,7 @@ class MLP < Model
28
28
  @l3 = Dense.new(10)
29
29
  end
30
30
 
31
- def call(x)
31
+ def forward(x)
32
32
  x = InputLayer.new(784).(x)
33
33
  x = @l1.(x)
34
34
  x = ReLU.(x)
@@ -0,0 +1,157 @@
1
+ include DNN::Models
2
+ include DNN::Layers
3
+
4
+ class Generator < Model
5
+ def initialize(input_shape)
6
+ super()
7
+ @input_shape = input_shape
8
+ @l1 = Conv2D.new(32, 4, padding: true)
9
+ @l2 = Conv2D.new(32, 4, strides: 2, padding: true)
10
+ @l3 = Conv2D.new(64, 4, padding: true)
11
+ @l4 = Conv2D.new(64, 4, strides: 2, padding: true)
12
+ @l5 = Conv2D.new(128, 4, padding: true)
13
+ @l6 = Conv2DTranspose.new(64, 4, strides: 2, padding: true)
14
+ @l7 = Conv2D.new(64, 4, padding: true)
15
+ @l8 = Conv2DTranspose.new(32, 4, strides: 2, padding: true)
16
+ @l9 = Conv2D.new(32, 4, padding: true)
17
+ @l10 = Conv2D.new(32, 4, padding: true)
18
+ @l11 = Conv2D.new(3, 4, padding: true)
19
+ @bn1 = BatchNormalization.new
20
+ @bn2 = BatchNormalization.new
21
+ @bn3 = BatchNormalization.new
22
+ @bn4 = BatchNormalization.new
23
+ @bn5 = BatchNormalization.new
24
+ @bn6 = BatchNormalization.new
25
+ @bn7 = BatchNormalization.new
26
+ @bn8 = BatchNormalization.new
27
+ @bn9 = BatchNormalization.new
28
+ end
29
+
30
+ def forward(x)
31
+ input = InputLayer.new(@input_shape).(x)
32
+ x = @l1.(input)
33
+ x = @bn1.(x)
34
+ h1 = ReLU.(x)
35
+
36
+ x = @l2.(h1)
37
+ x = @bn2.(x)
38
+ x = ReLU.(x)
39
+
40
+ x = @l3.(x)
41
+ x = @bn3.(x)
42
+ h2 = ReLU.(x)
43
+
44
+ x = @l4.(x)
45
+ x = @bn4.(x)
46
+ x = ReLU.(x)
47
+
48
+ x = @l5.(x)
49
+ x = @bn5.(x)
50
+ x = ReLU.(x)
51
+
52
+ x = @l6.(x)
53
+ x = @bn6.(x)
54
+ x = ReLU.(x)
55
+
56
+ x = @l7.(x)
57
+ x = @bn7.(x)
58
+ x = ReLU.(x)
59
+ x = Concatenate.(x, h2, axis: 3)
60
+
61
+ x = @l8.(x)
62
+ x = @bn8.(x)
63
+ x = ReLU.(x)
64
+
65
+ x = @l9.(x)
66
+ x = @bn9.(x)
67
+ x = ReLU.(x)
68
+ x = Concatenate.(x, h1, axis: 3)
69
+
70
+ x = @l10.(x)
71
+ x = ReLU.(x)
72
+
73
+ x = @l11.(x)
74
+ x = Tanh.(x)
75
+ x
76
+ end
77
+ end
78
+
79
+ class Discriminator < Model
80
+ def initialize(gen_input_shape, gen_output_shape)
81
+ super()
82
+ @gen_input_shape = gen_input_shape
83
+ @gen_output_shape = gen_output_shape
84
+ @l1_1 = Conv2D.new(32, 4, padding: true)
85
+ @l1_2 = Conv2D.new(32, 4, padding: true)
86
+ @l2 = Conv2D.new(32, 4, strides: 2, padding: true)
87
+ @l3 = Conv2D.new(32, 4, padding: true)
88
+ @l4 = Conv2D.new(64, 4, strides: 2, padding: true)
89
+ @l5 = Conv2D.new(64, 4, padding: true)
90
+ @l6 = Dense.new(1024)
91
+ @l7 = Dense.new(1)
92
+ @bn1 = BatchNormalization.new
93
+ @bn2 = BatchNormalization.new
94
+ @bn3 = BatchNormalization.new
95
+ @bn4 = BatchNormalization.new
96
+ @bn5 = BatchNormalization.new
97
+ @bn6 = BatchNormalization.new
98
+ end
99
+
100
+ def forward(inputs, trainable = true)
101
+ trainable_layers.each do |layer|
102
+ layer.trainable = trainable
103
+ end
104
+
105
+ input, images = *inputs
106
+ x = InputLayer.new(@gen_input_shape).(input)
107
+ x = @l1_1.(x)
108
+ x = @bn1.(x)
109
+ x1 = LeakyReLU.(x, 0.2)
110
+
111
+ x = InputLayer.new(@gen_output_shape).(images)
112
+ x = @l1_2.(x)
113
+ x = @bn2.(x)
114
+ x2 = LeakyReLU.(x, 0.2)
115
+
116
+ x = Concatenate.(x1, x2)
117
+ x = @l2.(x)
118
+ x = @bn3.(x)
119
+ x = LeakyReLU.(x, 0.2)
120
+
121
+ x = @l3.(x)
122
+ x = @bn4.(x)
123
+ x = LeakyReLU.(x, 0.2)
124
+
125
+ x = @l4.(x)
126
+ x = @bn5.(x)
127
+ x = LeakyReLU.(x, 0.2)
128
+
129
+ x = @l5.(x)
130
+ x = @bn6.(x)
131
+ x = LeakyReLU.(x, 0.2)
132
+
133
+ x = Flatten.(x)
134
+ x = @l6.(x)
135
+ x = LeakyReLU.(x, 0.2)
136
+
137
+ x = @l7.(x)
138
+ x
139
+ end
140
+ end
141
+
142
+ class DCGAN < Model
143
+ attr_reader :gen
144
+ attr_reader :dis
145
+
146
+ def initialize(gen, dis)
147
+ super()
148
+ @gen = gen
149
+ @dis = dis
150
+ end
151
+
152
+ def forward(input)
153
+ x = @gen.(input)
154
+ x = @dis.([input, x], false)
155
+ x
156
+ end
157
+ end
@@ -0,0 +1,27 @@
1
+ require "dnn"
2
+ require "dnn/image"
3
+ require "dnn/datasets/cifar10"
4
+ require "numo/linalg/autoloader"
5
+ require_relative "dcgan"
6
+
7
+ def load_dataset
8
+ x, y = DNN::CIFAR10.load_test
9
+ x_out = Numo::SFloat.cast(x)
10
+ x_in = x_out.mean(axis: 3, keepdims: true)
11
+ x_in = (x_in / 127.5) - 1
12
+ x_out = (x_out / 127.5) - 1
13
+ [x_in, x_out]
14
+ end
15
+
16
+ batch_size = 100
17
+
18
+ dcgan = DCGAN.load("trained/dcgan_model_epoch20.marshal")
19
+ gen = dcgan.gen
20
+
21
+ x_in, x_out = load_dataset
22
+ images = gen.predict(x_in[0...batch_size, false])
23
+
24
+ batch_size.times do |i|
25
+ img = Numo::UInt8.cast(((images[i, false] + 1) * 127.5).round)
26
+ DNN::Image.write("img/img_#{i}.jpg", img)
27
+ end
@@ -0,0 +1,52 @@
1
+ # This example converts a CIFAR10 grayscale image to a color image.
2
+
3
+ require "dnn"
4
+ require "dnn/datasets/cifar10"
5
+ require "numo/linalg/autoloader"
6
+ require_relative "dcgan"
7
+
8
+ include DNN::Optimizers
9
+ include DNN::Losses
10
+
11
+ def load_dataset
12
+ x, y = DNN::CIFAR10.load_train
13
+ x_out = Numo::SFloat.cast(x)
14
+ x_in = x_out.mean(axis: 3, keepdims: true)
15
+ x_in = (x_in / 127.5) - 1
16
+ x_out = (x_out / 127.5) - 1
17
+ [x_in, x_out]
18
+ end
19
+
20
+ epochs = 20
21
+ batch_size = 128
22
+
23
+ gen = Generator.new([32, 32, 1])
24
+ dis = Discriminator.new([32, 32, 1], [32, 32, 3])
25
+ dcgan = DCGAN.new(gen, dis)
26
+
27
+ dis.setup(Adam.new(alpha: 0.00001, beta1: 0.1), SigmoidCrossEntropy.new)
28
+ dcgan.setup(Adam.new(alpha: 0.0002, beta1: 0.5), SigmoidCrossEntropy.new)
29
+
30
+ x_in, x_out = load_dataset
31
+
32
+ iter1 = DNN::Iterator.new(x_in, x_out)
33
+ iter2 = DNN::Iterator.new(x_in, x_out)
34
+ num_batchs = x_in.shape[0] / batch_size
35
+ (1..epochs).each do |epoch|
36
+ num_batchs.times do |index|
37
+ x_in, x_out = iter1.next_batch(batch_size)
38
+ images = gen.predict(x_in)
39
+ y_real = Numo::SFloat.ones(batch_size, 1)
40
+ y_fake = Numo::SFloat.zeros(batch_size, 1)
41
+ dis_loss = dis.train_on_batch([x_in, x_out], y_real)
42
+ dis_loss += dis.train_on_batch([x_in, images], y_fake)
43
+
44
+ x_in, x_out = iter2.next_batch(batch_size)
45
+ dcgan_loss = dcgan.train_on_batch(x_in, y_real)
46
+
47
+ puts "epoch: #{epoch}, index: #{index}, dis_loss: #{dis_loss}, dcgan_loss: #{dcgan_loss}"
48
+ end
49
+ iter1.reset
50
+ iter2.reset
51
+ dcgan.save("trained/dcgan_model_epoch#{epoch}.marshal")
52
+ end
data/lib/dnn.rb CHANGED
@@ -8,6 +8,7 @@ module DNN
8
8
  end
9
9
 
10
10
  require_relative "dnn/version"
11
+ require_relative "dnn/core/monkey_patch"
11
12
  require_relative "dnn/core/error"
12
13
  require_relative "dnn/core/global"
13
14
  require_relative "dnn/core/tensor"
@@ -22,6 +23,7 @@ require_relative "dnn/core/layers/merge_layers"
22
23
  require_relative "dnn/core/layers/cnn_layers"
23
24
  require_relative "dnn/core/layers/embedding"
24
25
  require_relative "dnn/core/layers/rnn_layers"
26
+ require_relative "dnn/core/layers/math_layers"
25
27
  require_relative "dnn/core/optimizers"
26
28
  require_relative "dnn/core/losses"
27
29
  require_relative "dnn/core/initializers"
@@ -2,70 +2,84 @@ module DNN
2
2
  module Layers
3
3
 
4
4
  class Sigmoid < Layer
5
- def forward(x)
5
+ include LayerNode
6
+
7
+ def forward_node(x)
6
8
  @y = 1 / (1 + Xumo::NMath.exp(-x))
7
9
  end
8
10
 
9
- def backward(dy)
11
+ def backward_node(dy)
10
12
  dy * (1 - @y) * @y
11
13
  end
12
14
  end
13
15
 
14
16
  class Tanh < Layer
15
- def forward(x)
17
+ include LayerNode
18
+
19
+ def forward_node(x)
16
20
  @y = Xumo::NMath.tanh(x)
17
21
  end
18
22
 
19
- def backward(dy)
23
+ def backward_node(dy)
20
24
  dy * (1 - @y**2)
21
25
  end
22
26
  end
23
27
 
24
28
  class Softsign < Layer
25
- def forward(x)
29
+ include LayerNode
30
+
31
+ def forward_node(x)
26
32
  @x = x
27
33
  x / (1 + x.abs)
28
34
  end
29
35
 
30
- def backward(dy)
36
+ def backward_node(dy)
31
37
  dy * (1 / (1 + @x.abs)**2)
32
38
  end
33
39
  end
34
40
 
35
41
  class Softplus < Layer
36
- def forward(x)
42
+ include LayerNode
43
+
44
+ def forward_node(x)
37
45
  @x = x
38
46
  Xumo::NMath.log(1 + Xumo::NMath.exp(x))
39
47
  end
40
48
 
41
- def backward(dy)
49
+ def backward_node(dy)
42
50
  dy * (1 / (1 + Xumo::NMath.exp(-@x)))
43
51
  end
44
52
  end
45
53
 
46
54
  class Swish < Layer
47
- def forward(x)
55
+ include LayerNode
56
+
57
+ def forward_node(x)
48
58
  @x = x
49
59
  @y = x * (1 / (1 + Xumo::NMath.exp(-x)))
50
60
  end
51
61
 
52
- def backward(dy)
62
+ def backward_node(dy)
53
63
  dy * (@y + (1 / (1 + Xumo::NMath.exp(-@x))) * (1 - @y))
54
64
  end
55
65
  end
56
66
 
57
67
  class ReLU < Layer
58
- def forward(x)
68
+ include LayerNode
69
+
70
+ def forward_node(x)
59
71
  @x = x
60
72
  Xumo::SFloat.maximum(0, x)
61
73
  end
62
74
 
63
- def backward(dy)
75
+ def backward_node(dy)
64
76
  dy * Xumo::SFloat.cast(@x > 0)
65
77
  end
66
78
  end
67
79
 
68
80
  class LeakyReLU < Layer
81
+ include LayerNode
82
+
69
83
  attr_reader :alpha
70
84
 
71
85
  # @param [Float] alpha The slope when the output value is negative.
@@ -74,14 +88,14 @@ module DNN
74
88
  @alpha = alpha
75
89
  end
76
90
 
77
- def forward(x)
91
+ def forward_node(x)
78
92
  @x = x
79
93
  a = Xumo::SFloat.ones(x.shape)
80
94
  a[x <= 0] = @alpha
81
95
  x * a
82
96
  end
83
97
 
84
- def backward(dy)
98
+ def backward_node(dy)
85
99
  dx = Xumo::SFloat.ones(@x.shape)
86
100
  dx[@x <= 0] = @alpha
87
101
  dy * dx
@@ -97,6 +111,8 @@ module DNN
97
111
  end
98
112
 
99
113
  class ELU < Layer
114
+ include LayerNode
115
+
100
116
  attr_reader :alpha
101
117
 
102
118
  # @param [Float] alpha The slope when the output value is negative.
@@ -105,7 +121,7 @@ module DNN
105
121
  @alpha = alpha
106
122
  end
107
123
 
108
- def forward(x)
124
+ def forward_node(x)
109
125
  @x = x
110
126
  x1 = Xumo::SFloat.zeros(x.shape)
111
127
  x1[x >= 0] = 1
@@ -116,7 +132,7 @@ module DNN
116
132
  x1 + x2
117
133
  end
118
134
 
119
- def backward(dy)
135
+ def backward_node(dy)
120
136
  dx = Xumo::SFloat.ones(@x.shape)
121
137
  dx[@x < 0] = 0
122
138
  dx2 = Xumo::SFloat.zeros(@x.shape)
@@ -135,12 +151,14 @@ module DNN
135
151
  end
136
152
 
137
153
  class Mish < Layer
138
- def forward(x)
154
+ include LayerNode
155
+
156
+ def forward_node(x)
139
157
  @x = x
140
- x * Xumo::NMath.tanh(Softplus.new.forward(x))
158
+ x * Xumo::NMath.tanh(Softplus.new.forward_node(x))
141
159
  end
142
160
 
143
- def backward(dy)
161
+ def backward_node(dy)
144
162
  omega = 4 * (@x + 1) + 4 * Xumo::NMath.exp(2 * @x) + Xumo::NMath.exp(3 * @x) + Xumo::NMath.exp(@x) * (4 * @x + 6)
145
163
  delta = 2 * Xumo::NMath.exp(@x) + Xumo::NMath.exp(2 * @x) + 2
146
164
  dy * (Xumo::NMath.exp(@x) * omega) / delta**2