rann 0.2.6 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 80b0dff761e66e192ee76aefe2cd6f91354dc061
-  data.tar.gz: a4957efc538906c8f5b4e265d9bd5bb5ae0308fc
+  metadata.gz: b604698cf3630cf7056f4b10d5b1b301292074b4
+  data.tar.gz: a341f41f53c78db0f026948b7bad72ff04574e54
 SHA512:
-  metadata.gz: 4e212b698e71a4e9804f6a3ac2b5169861cb2f35baa782cacaac798657cf741266c1b085f876f6355d0fd6bfbe3993fd8fe30d360fd6e23c156eb034e32cfa85
-  data.tar.gz: fe707c128b2d21651278fff44a63c6272f16e16262d54b17610bf8d27c020983e5a894e0a4a7ac7197956768f06712eb3bae861a1be7c36cea12ad56d8c16e50
+  metadata.gz: 3e7167b3fb0fb79d0fcd3fc7a72b59d0ab4c9d29b69e6fbc230599b11752b3b9e001c06e9bef284325974649e7f64d4f34dd05864fe1fedb9fa7786ef9d40586
+  data.tar.gz: 6816e982c0536c21a474e377c87ca6de213b62268dfda48317d2c360972e2fa2d0f32eb491c8a088cccae8ca3bb5ef367587d601db2a7eda3d45d695a6c5ea8a

data/CHANGES.md

@@ -1,3 +1,11 @@
+## 0.2.7 (December 2, 2017)
+
+- Allow different weights to the different gates in the LSTM. Previously it was
+  using one weighted connection into the LSTM and unweighted connections to the
+  gates, this has been reversed.
+
+  *Michael Campbell*
+
 - Add save and restore methods to backprop.
 
   *Michael Campbell*

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rann (0.2.6)
+    rann (0.2.7)
       parallel (~> 1.12, >= 1.12.0)
       ruby-graphviz (~> 1.2, >= 1.2.3)
 

data/lib/rann/backprop.rb

@@ -128,35 +128,34 @@ module RANN
 
         # neuron delta is summation of neuron deltas deltas for the connections
         # from this neuron
-        node_delta =
+        step_one =
           if neuron.output?
             output_index = network.output_neurons.index neuron
-            activation_derivative = ACTIVATION_DERIVATIVES[neuron.activation_function]
-            mse_delta targets[output_index], outputs[output_index], activation_derivative
+            mse_delta targets[output_index], outputs[output_index]
           else
-            sum_of_deltas =
-              network.connections_from(neuron).reduce 0.to_d do |m, c|
-                out_timestep = c.output_neuron.context? ? timestep + 1 : timestep
-                output_node_delta = node_deltas[out_timestep][c.output_neuron.id]
-
-                # connection delta is the output neuron delta multiplied by the
-                # connection's weight
-                connection_delta =
-                  if c.output_neuron.is_a? ProductNeuron
-                    intermediate = states[out_timestep][:intermediates][c.output_neuron.id]
-                    output_node_delta.mult intermediate.div(states[timestep][:values][c.input_neuron.id], 10), 10
-                  else
-                    output_node_delta.mult c.weight, 10
-                  end
-
-                m + connection_delta
-              end
-
-            ACTIVATION_DERIVATIVES[neuron.activation_function]
-              .call(states[timestep][:values][neuron.id])
-              .mult(sum_of_deltas, 10)
+            network.connections_from(neuron).reduce 0.to_d do |m, c|
+              out_timestep = c.output_neuron.context? ? timestep + 1 : timestep
+              output_node_delta = node_deltas[out_timestep][c.output_neuron.id]
+
+              # connection delta is the output neuron delta multiplied by the
+              # connection's weight
+              connection_delta =
+                if c.output_neuron.is_a? ProductNeuron
+                  intermediate = states[out_timestep][:intermediates][c.output_neuron.id]
+                  output_node_delta.mult intermediate.div(states[timestep][:values][c.input_neuron.id], 10), 10
+                else
+                  output_node_delta.mult c.weight, 10
+                end
+
+              m + connection_delta
+            end
           end
 
+        node_delta =
+          ACTIVATION_DERIVATIVES[neuron.activation_function]
+            .call(states[timestep][:values][neuron.id])
+            .mult(step_one, 10)
+
         node_deltas[timestep][neuron.id] = node_delta
 
         network.connections_to(neuron).each do |c|
@@ -224,11 +223,8 @@ module RANN
       total_squared_error
     end
 
-    def self.mse_delta target, actual, activation_derivative
-      step_one = actual - target
-      step_two = activation_derivative.call actual
-
-      step_one.mult step_two, 10
+    def self.mse_delta target, actual
+      actual - target
     end
 
     def self.bptt_connecting_to neuron, network, timestep

data/lib/rann/lstm.rb

@@ -18,7 +18,6 @@ module RANN
     end
 
     def init
-      input_bias = RANN::Neuron.new("LSTM #{name} Input Bias", 0, :bias).tap{ |n| @network.add n }
       @size.times do |j|
         input = RANN::Neuron.new("LSTM #{name} Input #{j}", 0, :standard).tap{ |n| @network.add n }
         @inputs << input
@@ -49,10 +48,10 @@ module RANN
         memory_context = RANN::Neuron.new("LSTM #{name} Mem Context #{j}", 1, :context).tap{ |n| @network.add n }
         output_context = RANN::Neuron.new("LSTM #{name} Output Context #{j}", 1, :context).tap{ |n| @network.add n }
 
-        @network.add RANN::LockedConnection.new input, f, 1
-        @network.add RANN::LockedConnection.new input, i, 1
-        @network.add RANN::LockedConnection.new input, g, 1
-        @network.add RANN::LockedConnection.new input, o, 1
+        @network.add RANN::Connection.new input, f
+        @network.add RANN::Connection.new input, i
+        @network.add RANN::Connection.new input, g
+        @network.add RANN::Connection.new input, o
         @network.add RANN::LockedConnection.new f, memory_product, 1
         @network.add RANN::LockedConnection.new i, i_g_product, 1
         @network.add RANN::LockedConnection.new g, i_g_product, 1
@@ -63,8 +62,8 @@ module RANN
         @network.add RANN::LockedConnection.new memory_tanh, memory_o_product, 1
         @network.add RANN::LockedConnection.new memory_o_product, output, 1
         @network.add RANN::LockedConnection.new memory_standard, memory_context, 1
-        @network.add RANN::Connection.new memory_context, memory_product
-        @network.add RANN::Connection.new memory_context, i
+        @network.add RANN::LockedConnection.new memory_context, memory_product, 1
+        @network.add RANN::LockedConnection.new memory_context, i, 1
         @network.add RANN::LockedConnection.new memory_o_product, output_context, 1
         @network.add RANN::Connection.new output_context, f
         @network.add RANN::Connection.new output_context, i
@@ -74,13 +73,12 @@ module RANN
         @network.add RANN::Connection.new bias_i, i
         @network.add RANN::Connection.new bias_g, g
         @network.add RANN::Connection.new bias_o, o
-        @network.add RANN::Connection.new input_bias, input
       end
     end
 
     def add_input neuron
       @inputs.each do |input|
-        @network.add RANN::Connection.new neuron, input
+        @network.add RANN::LockedConnection.new neuron, input, 1
       end
     end
   end

data/lib/rann/version.rb

@@ -1,3 +1,3 @@
 module RANN
-  VERSION = "0.2.6"
+  VERSION = "0.2.7"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rann
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.7
 platform: ruby
 authors:
 - Michael Campbell
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-30 00:00:00.000000000 Z
+date: 2017-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel