rann 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b474ce98e49bb5067d2fc8ddba98a9c6f887c618
-  data.tar.gz: feb50a3a4804494c0a4c2551e7a2b3807b89dd69
+  metadata.gz: 199b82878ecd7e186aca7df6529af6b725b049a5
+  data.tar.gz: 5a61a9fb308c3c54055a61e516aa370f152170e8
 SHA512:
-  metadata.gz: fc4f352df3b64af7d6b4c513d0aefd343ec35b4b6b599e7154cdbb7eefd06365023c2192c1736e3d7298a6895b388db3047387cfb4e52e7136f97eb09a320e5d
-  data.tar.gz: eec1f19c738594b4b29c5de6da0268d249c4eaaddec7f243919f96ee73331025e571a52d215e894255c79f2451b55de5eb43caaa53a91d898048548190ee355e
+  metadata.gz: 8d02e3a788f9be66b2912edc21b0b807a6ba16ea867ac6ecb03454988e35e11971f8e39d1938324b834a5241df1c85775240b7665fa566b109ed2578d5fb23e3
+  data.tar.gz: 07b770013594c1b6e9a42ecdd812f2b2e12b3d902d2608d23ad24b73d44e270fb5bfb84636018405d5af7ea94580f89d5d2a09dc6f00917c987c336116a08442

data/README.md

@@ -34,17 +34,16 @@ ruby examples/xor.rb
 
 ## TODO
 
-So much. So much.
+So much.
 
+- Matrix calculations (WIP)
 - Convenience methods for setting up standard network topologies, crucially,
-  layers
-- Batch normalization/drop out/early stopping
+  layers (WIP)
+- Batch normalization/drop out/early stopping (WIP, dep. on matrix)
 - Hyperparameter optimisation
 - Other adaptive learning rate algorithms (Adadelta, Adam, etc?)
 - Explore matrix operations and other ways to optimise performance of algorithms
 - RPROP?
-- Use enumerable-statistics gem?
-- Speed up by adding a reduce step to the parallel gem?
 - More examples
 - Tests
 

data/bench/xor_benchmark.rb

@@ -0,0 +1,55 @@
+require "bundler/setup"
+require "rann"
+
+xor_inputs = [[0,0],[0,1],[1,0],[1,1]]
+xor_targets = [[0],[1],[1],[0]]
+
+time = Time.now.to_i
+results =
+  Array.new(100) do |j|
+    # inputs
+    inputs = Array.new(2){ |i| RANN::Neuron.new "input #{i}", 0, :input }
+
+    # hidden layer
+    hiddens = Array.new(3){ |i| RANN::Neuron.new "hidden #{i}", 3 }
+    bias = RANN::Neuron.new "bias", 0, :bias
+
+    # output layer
+    output = RANN::Neuron.new "output", 3, :output, :sig
+
+    # connect it all
+    connections = []
+    hiddens.each do |h|
+      inputs.each do |i|
+        connections.push RANN::Connection.new i, h
+      end
+      connections.push RANN::Connection.new bias, h
+      connections.push RANN::Connection.new h, output
+    end
+
+    network = RANN::Network.new connections
+    backprop = RANN::Backprop.new network
+
+    i = 0
+    loop do
+      i += 1
+      sample_index = (rand * xor_inputs.size).to_i
+
+      avg_error =
+        backprop.run_batch(
+          [xor_inputs[sample_index].map(&:to_d)],
+          [xor_targets[sample_index].map(&:to_d)],
+          processes: 0,
+          checking: false
+        )
+
+      break if avg_error < 0.0001
+    end
+
+    puts j
+    i
+  end
+
+taken = Time.now.to_i - time
+puts results.reduce(:+).fdiv(results.size).round(2)
+puts "in #{taken}s"

data/lib/rann/backprop.rb

@@ -1,6 +1,10 @@
+require "bigdecimal"
+require "bigdecimal/util"
 require "parallel"
 require "rann/gradient_checker"
 require "rann/util/array_ext"
+require "rann/optimisers/adagrad"
+require "rann/optimisers/rmsprop"
 
 module RANN
   class Backprop
@@ -14,27 +18,16 @@ module RANN
       step:   ->(_){ 0.to_d },
     }
 
-    DECAY = BigDecimal.new('0.9')
-    MASTER_STEP_SIZE = BigDecimal.new('0.01')
-    FUDGE_FACTOR = BigDecimal.new('0.00000001')
-    LEARNING_RATE = BigDecimal.new('0.01')
-    FRICTION = BigDecimal.new('0.8')
-    NUM_ITERATIONS_BEFORE_LR_ANNEALING = BigDecimal.new('10')
-
-    attr_accessor :network, :lr, :velocities
-
-    def initialize network, restore = {}
-      @network             = network
-      @connections_hash    = network.connections.each.with_object({}){ |c, h| h[c.id] = c }
-      @lr                  = LEARNING_RATE
-      @friction            = FRICTION
-      @velocities          = Hash.new(BigDecimal.new('0'))
-      @historical_gradient = (restore[:historical_gradient] || {}).tap{ |h| h.default = 0.to_d }
-      @historical_update   = Hash.new(MASTER_STEP_SIZE)
-      @batch_count         = BigDecimal.new('0')
+    attr_accessor :network
+
+    def initialize network, opts = {}, restore = {}
+      @network          = network
+      @connections_hash = network.connections.each.with_object({}){ |c, h| h[c.id] = c }
+      @optimiser        = RANN::Optimisers.const_get(opts[:optimiser] || 'RMSProp').new opts, restore
+      @batch_count      = 0.to_d
     end
 
-    def run_batch(inputs, targets, opts = {})
+    def run_batch inputs, targets, opts = {}
       @batch_count += 1
 
       batch_size      = inputs.size
@@ -43,37 +36,47 @@ module RANN
 
       # force longer bits of work per iteration, to maximise CPU usage
       # less marshalling data etc, more work.
-      grouped_inputs  = in_groups inputs, [1, opts[:processes]].max * 10, false
-      grouped_results =
-        Parallel.map_with_index grouped_inputs, in_processes: opts[:processes] do |inputs, i|
-          group_avg_gradients = Hash.new{ |h, k| h[k] = 0.to_d }
-          group_avg_error     = 0.to_d
+      grouped_inputs  = in_groups(inputs, [1, opts[:processes]].max * 10, false).reject &:empty?
+      reduce_proc =
+        lambda do |_, _, result|
+          group_avg_gradients, group_avg_error = result
 
-          inputs.each do |input|
-            gradients, error = Backprop.run_single(network, input, targets[i])
+          avg_gradients.merge!(group_avg_gradients){ |_, o, n| o + n }
+          avg_batch_error += group_avg_error
+        end
 
-            gradients.each do |cid, g|
-              group_avg_gradients[cid] += g.div batch_size, 10
-            end
-            group_avg_error += error.div batch_size, 10
-          end
+      Parallel.each_with_index(
+        grouped_inputs,
+        in_processes: opts[:processes],
+        finish: reduce_proc
+      ) do |inputs, i|
+        group_avg_gradients = Hash.new{ |h, k| h[k] = 0.to_d }
+        group_avg_error = 0.to_d
 
-          group_avg_gradients.default_proc = nil
-          [group_avg_gradients, group_avg_error]
+        inputs.each_with_index do |input, j|
+          gradients, error = Backprop.run_single network, input, targets[i + j]
+
+          gradients.each do |cid, g|
+            group_avg_gradients[cid] += g.div batch_size, 10
+          end
+          group_avg_error += error.div batch_size, 10
         end
 
-      grouped_results.each do |group_avg_gradients, group_avg_error|
-        avg_gradients.merge!(group_avg_gradients){ |_, o, n| o + n }
-        avg_batch_error += group_avg_error
+        group_avg_gradients.default_proc = nil
+        [group_avg_gradients, group_avg_error]
       end
 
       if opts[:checking]
         # check assumes batchsize 1 for now
         sorted_gradients = avg_gradients.values_at *network.connections.map(&:id)
-        if GradientChecker.check network, inputs.first, targets.first, sorted_gradients
+        invalid = GradientChecker.check network, inputs.first, targets.first, sorted_gradients
+        if invalid.empty?
           puts "gradient valid"
         else
-          puts "gradient INVALID"
+          puts "gradients INVALID for connections:"
+          invalid.each do |i|
+            puts "#{network.connections[i].input_neuron.name} -> #{network.connections[i].output_neuron.name}"
+          end
         end
       end
 
@@ -81,7 +84,7 @@ module RANN
         con = @connections_hash[con_id]
         next if con.locked?
 
-        update = adagrad gradient, con.id
+        update = @optimiser.update gradient, con.id
 
         con.weight += update
       end
@@ -107,60 +110,68 @@ module RANN
       error = mse targets, outputs
 
       # backward pass with unravelling for recurrent networks
-      deltas = Hash.new{ |h, k| h[k] = Hash.new(0.to_d) }
-
-      # outputs first
-      network.output_neurons.each.with_index do |o, i|
-        activation_derivative = ACTIVATION_DERIVATIVES[o.activation_function]
-
-        deltas[0][o.id] = mse_delta(targets[i], outputs[i], activation_derivative)
-      end
-
-      # remove this push mechanism, shouldn't be necessary and uses extra memory.
-      incoming_deltas = Hash.new{ |h, k| h[k] = Hash.new{ |h, k| h[k] = [] } }
-      # each timestep backwards through time
-      (inputs.size - 1).downto 0 do |t|
-        network.output_neurons.each do |o|
-          traverse from: o, network: network, timestep: t, deltas: deltas do |other, con|
-            if other.context?
-              this_t = t - 1
-              other  = o
-            else
-              this_t = t
-            end
-
-            incoming_deltas[this_t][other.id] <<
-              deltas[t][o.id].mult(con.weight, 10)
-
-            if incoming_deltas[this_t][other.id].size == network.connections_from(other).size
-              sum_of_deltas = incoming_deltas[this_t][other.id].reduce(:+)
+      node_deltas = Hash.new{ |h, k| h[k] = Hash.new(0.to_d) }
+      gradients = Hash.new(0)
+
+      initial_timestep = inputs.size - 1
+      neuron_stack = network.output_neurons.map{ |n| [n, initial_timestep] }
+
+      while current = neuron_stack.shift
+        neuron, timestep = current
+        next if node_deltas[timestep].key? neuron
+
+        from_here = bptt_connecting_to neuron, network, timestep
+        neuron_stack.push *from_here
+
+        # neuron delta is summation of neuron deltas deltas for the connections
+        # from this neuron
+        node_delta =
+          if neuron.output?
+            output_index = network.output_neurons.index neuron
+            activation_derivative = ACTIVATION_DERIVATIVES[neuron.activation_function]
+            mse_delta targets[output_index], outputs[output_index], activation_derivative
+          else
+            sum_of_deltas =
+              network.connections_from(neuron).reduce 0.to_d do |m, c|
+                out_timestep = c.output_neuron.context? ? timestep + 1 : timestep
+                output_node_delta = node_deltas[out_timestep][c.output_neuron.id]
+
+                # connection delta is the output neuron delta multiplied by the
+                # connection's weight
+                connection_delta =
+                  if c.output_neuron.is_a? ProductNeuron
+                    intermediate = states[out_timestep][:intermediates][c.output_neuron.id]
+                    output_node_delta.mult intermediate.div(states[timestep][:values][c.input_neuron.id], 10), 10
+                  else
+                    output_node_delta.mult c.weight, 10
+                  end
+
+                m + connection_delta
+              end
 
-              deltas[this_t][other.id] =
-                ACTIVATION_DERIVATIVES[other.activation_function]
-                  .call(states[this_t][other.id])
-                  .mult(sum_of_deltas, 10)
-            end
+            ACTIVATION_DERIVATIVES[neuron.activation_function]
+              .call(states[timestep][:values][neuron.id])
+              .mult(sum_of_deltas, 10)
           end
-        end
-      end
 
-      gradients = {}
+        node_deltas[timestep][neuron.id] = node_delta
 
-      network.connections.each_with_index do |con, i|
-        gradients[con.id] = 0.to_d
-        next if con.output_neuron.context?
+        network.connections_to(neuron).each do |c|
+          in_timestep = neuron.context? ? timestep - 1 : timestep
 
-        (inputs.size - 1).downto 0 do |t|
-          if nd = deltas[t][con.output_neuron.id]
-            gradient =
-              if con.input_neuron.context?
-                t == 0 ? 0.to_d : nd.mult(states[t - 1][con.input_neuron.id], 10)
-              else
-                nd.mult states[t][con.input_neuron.id], 10
-              end
+          # connection gradient is the output neuron delta multipled by the
+          # connection's input neuron value.
+          gradient =
+            if c.output_neuron.is_a? ProductNeuron
+              intermediate = states[timestep][:intermediates][c.output_neuron.id]
+              node_delta.mult intermediate.div(c.weight, 10), 10
+            elsif c.input_neuron.context? && timestep == 0
+              0.to_d
+            else
+              node_delta.mult states[in_timestep][:values][c.input_neuron.id], 10
+            end
 
-            gradients[con.id] += gradient
-          end
+          gradients[c.id] += gradient
         end
       end
 
@@ -200,22 +211,16 @@ module RANN
       step_one.mult step_two, 10
     end
 
-    def self.traverse from:, network:, timestep:, deltas:, &block
-      # halt traversal if reached next timestep.
-      return if from.context?
-
-      bptt_connecting_to(from, network, timestep, deltas).each do |n, c|
-        yield n, c
+    def self.bptt_connecting_to neuron, network, timestep
+      # halt traversal if we're at a context and we're at the base timestep
+      return [] if neuron.context? && timestep == 0
 
-        traverse from: n, network: network, timestep: timestep, deltas: deltas, &block
-      end
-    end
-
-    def self.bptt_connecting_to neuron, network, timestep, deltas
       network.connections_to(neuron).each.with_object [] do |c, a|
-        unless c.input_neuron.input? || deltas[timestep].key?(c.input_neuron.id)
-          a << [c.input_neuron, c]
-        end
+        # don't enqueue connections from inputs
+        next if c.input_neuron.input?
+
+        timestep -= timestep if neuron.context?
+        a << [c.input_neuron, timestep]
       end
     end
   end

data/lib/rann/gradient_checker.rb

@@ -24,7 +24,9 @@ module RANN
         gradapprox[i] = (error_thetaplus - error_thetaminus).div(EPSILON.mult(2, 10), 10)
       end
 
-      gradapprox.each.with_index.all?{ |ga, i| in_epsilon? ga, dvec[i] }
+      gradapprox.each.with_index.with_object [] do |(ga, i), res|
+        res << i unless in_epsilon? ga, dvec[i]
+      end
     end
 
     def self.error outputs, targets

data/lib/rann/lstm.rb

@@ -8,44 +8,46 @@ module RANN
   class LSTM
     attr_reader :network, :inputs, :outputs, :name
 
-    def initialize name
+    def initialize name, size
       @name    = name
       @network = RANN::Network.new
       @inputs  = []
       @outputs = []
+      @size    = size
+      init
     end
 
     def init
-      @inputs.each.with_index do |input, i|
-        f = RANN::Neuron.new("LSTM #{name} F #{i}", 3, :standard, :sig).tap{ |n| @network.add n }
-        i = RANN::Neuron.new("LSTM #{name} I #{i}", 4, :standard, :sig).tap{ |n| @network.add n }
-        g = RANN::Neuron.new("LSTM #{name} G #{i}", 3, :standard, :tanh).tap{ |n| @network.add n }
-        o = RANN::Neuron.new("LSTM #{name} O #{i}", 3, :standard, :sig).tap{ |n| @network.add n }
-        bias_f = RANN::Neuron.new("LSTM #{name} Bias F #{i}", 0, :bias).tap do |n|
+      input_bias = RANN::Neuron.new("LSTM #{name} Input Bias", 0, :bias).tap{ |n| @network.add n }
+      @size.times do |j|
+        input = RANN::Neuron.new("LSTM #{name} Input #{j}", 0, :standard).tap{ |n| @network.add n }
+        @inputs << input
+
+        f = RANN::Neuron.new("LSTM #{name} F #{j}", 3, :standard, :sig).tap{ |n| @network.add n }
+        i = RANN::Neuron.new("LSTM #{name} I #{j}", 4, :standard, :sig).tap{ |n| @network.add n }
+        g = RANN::Neuron.new("LSTM #{name} G #{j}", 3, :standard, :tanh).tap{ |n| @network.add n }
+        o = RANN::Neuron.new("LSTM #{name} O #{j}", 3, :standard, :sig).tap{ |n| @network.add n }
+        bias_f = RANN::Neuron.new("LSTM #{name} Bias F #{j}", 0, :bias).tap do |n|
           @network.add n
-          n.value = 1.to_d
         end
-        bias_i = RANN::Neuron.new("LSTM #{name} Bias I #{i}", 0, :bias).tap do |n|
+        bias_i = RANN::Neuron.new("LSTM #{name} Bias I #{j}", 0, :bias).tap do |n|
           @network.add n
-          n.value = 1.to_d
         end
-        bias_g = RANN::Neuron.new("LSTM #{name} Bias G #{i}", 0, :bias).tap do |n|
+        bias_g = RANN::Neuron.new("LSTM #{name} Bias G #{j}", 0, :bias).tap do |n|
           @network.add n
-          n.value = 1.to_d
         end
-        bias_o = RANN::Neuron.new("LSTM #{name} Bias O #{i}", 0, :bias).tap do |n|
+        bias_o = RANN::Neuron.new("LSTM #{name} Bias O #{j}", 0, :bias).tap do |n|
           @network.add n
-          n.value = 1.to_d
         end
-        memory_product = RANN::ProductNeuron.new("LSTM #{name} Mem Product #{i}", 2, :standard, :linear).tap{ |n| @network.add n }
-        i_g_product = RANN::ProductNeuron.new("LSTM #{name} Hidden 2/3 Product #{i}", 2, :standard, :linear).tap{ |n| @network.add n }
-        memory_standard = RANN::Neuron.new("LSTM #{name} Mem Standard #{i}", 2, :standard, :linear).tap{ |n| @network.add n }
-        memory_tanh = RANN::Neuron.new("LSTM #{name} Mem Tanh #{i}", 1, :standard, :tanh).tap{ |n| @network.add n }
-        memory_o_product = RANN::ProductNeuron.new("LSTM #{name} Mem/Hidden 4 Product #{i}", 2, :standard, :linear).tap{ |n| @network.add n }
-        output = RANN::Neuron.new("LSTM #{name} Output #{i}", 1, :standard, :linear).tap{ |n| @network.add n }
+        memory_product = RANN::ProductNeuron.new("LSTM #{name} Mem Product #{j}", 2, :standard, :linear).tap{ |n| @network.add n }
+        i_g_product = RANN::ProductNeuron.new("LSTM #{name} Hidden 2/3 Product #{j}", 2, :standard, :linear).tap{ |n| @network.add n }
+        memory_standard = RANN::Neuron.new("LSTM #{name} Mem Standard #{j}", 2, :standard, :linear).tap{ |n| @network.add n }
+        memory_tanh = RANN::Neuron.new("LSTM #{name} Mem Tanh #{j}", 1, :standard, :tanh).tap{ |n| @network.add n }
+        memory_o_product = RANN::ProductNeuron.new("LSTM #{name} Mem/Hidden 4 Product #{j}", 2, :standard, :linear).tap{ |n| @network.add n }
+        output = RANN::Neuron.new("LSTM #{name} Output #{j}", 1, :standard, :linear).tap{ |n| @network.add n }
         @outputs << output
-        memory_context = RANN::Neuron.new("LSTM #{name} Mem Context #{i}", 1, :context).tap{ |n| @network.add n }
-        output_context = RANN::Neuron.new("LSTM #{name} Output Context #{i}", 1, :context).tap{ |n| @network.add n }
+        memory_context = RANN::Neuron.new("LSTM #{name} Mem Context #{j}", 1, :context).tap{ |n| @network.add n }
+        output_context = RANN::Neuron.new("LSTM #{name} Output Context #{j}", 1, :context).tap{ |n| @network.add n }
 
         @network.add RANN::LockedConnection.new input, f, 1
         @network.add RANN::LockedConnection.new input, i, 1
@@ -72,15 +74,14 @@ module RANN
         @network.add RANN::Connection.new bias_i, i
         @network.add RANN::Connection.new bias_g, g
         @network.add RANN::Connection.new bias_o, o
+        @network.add RANN::Connection.new input_bias, input
       end
     end
 
     def add_input neuron
-      input = RANN::Neuron.new "LSTM #{name} Input #{neuron.name}", 0, :standard, :linear
-      @network.add input
-      @inputs << input
-      connection = RANN::Connection.new neuron, input
-      @network.add connection
+      @inputs.each do |input|
+        @network.add RANN::Connection.new neuron, input
+      end
     end
   end
 end

data/lib/rann/network.rb

@@ -97,7 +97,10 @@ module RANN
     end
 
     def state
-      neurons.each.with_object({}){ |n, s| s[n.id] = n.value }
+      {
+        values: neurons.each.with_object({}){ |n, s| s[n.id] = n.value },
+        intermediates: neurons.select{ |n| n.is_a? ProductNeuron }.each.with_object({}){ |n, s| s[n.id] = n.intermediate }
+      }
     end
 
     def connections_to neuron

data/lib/rann/neuron.rb

@@ -5,7 +5,7 @@ require "bigdecimal/util"
 module RANN
   class Neuron
     ACTIVATION_FUNCTIONS = {
-      sig:    ->(v){ 1.to_d.div(1 + (Math::E ** -v), 10) },
+      sig:    ->(v){ 1.to_d.div(1 + Math::E.to_d.power(-v, 10), 10) },
       tanh:   ->(v){ Math.tanh(v).to_d(10) },
       relu:   ->(v){ [0.to_d, v].max },
       linear: ->(v){ v },

data/lib/rann/optimisers/adagrad.rb

@@ -0,0 +1,21 @@
+require "bigdecimal"
+require "bigdecimal/util"
+
+# refactor to matrix stuff blah blah
+module RANN
+  module Optimisers
+    class AdaGrad
+      def initialize opts = {}, restore = {}
+        @fudge_factor        = opts[:fudge_factor] || 0.00000001.to_d
+        @learning_rate       = opts[:learning_rate] || 0.1.to_d
+        @historical_gradient = (restore[:historical_gradient] || {}).tap{ |h| h.default = 0.to_d }
+      end
+
+      def update grad, cid
+        @historical_gradient[cid] = @historical_gradient[cid] + grad.power(2, 10)
+
+        grad.mult(- @learning_rate.div(@fudge_factor + @historical_gradient[cid].sqrt(10), 10), 10)
+      end
+    end
+  end
+end

data/lib/rann/optimisers/rmsprop.rb

@@ -0,0 +1,22 @@
+require "bigdecimal"
+require "bigdecimal/util"
+
+# refactor to matrix stuff blah blah
+module RANN
+  module Optimisers
+    class RMSProp
+      def initialize opts = {}, restore = {}
+        @decay               = opts[:decay] || 0.9.to_d
+        @fudge_factor        = opts[:fudge_factor] || 0.00000001.to_d
+        @learning_rate       = opts[:learning_rate] || 0.01.to_d
+        @historical_gradient = (restore[:historical_gradient] || {}).tap{ |h| h.default = 0.to_d }
+      end
+
+      def update grad, cid
+        @historical_gradient[cid] = @decay.mult(@historical_gradient[cid], 10) + (1 - @decay).mult(grad.power(2, 10), 10)
+
+        grad.mult(- @learning_rate.div(@fudge_factor + @historical_gradient[cid].sqrt(10), 10), 10)
+      end
+    end
+  end
+end

data/lib/rann/product_neuron.rb

@@ -2,9 +2,11 @@ require "rann/neuron"
 
 module RANN
   class ProductNeuron < Neuron
+    attr_accessor :intermediate
+
     def set_value!
-      intermediate = incoming.reduce{ |i, m| m.mult i, 10 }
-      self.value   = ACTIVATION_FUNCTIONS[activation_function].call intermediate
+      @intermediate = incoming.reduce{ |i, m| m.mult(i, 10) }
+      self.value    = ACTIVATION_FUNCTIONS[activation_function].call @intermediate
     end
   end
 end

data/lib/rann/version.rb

@@ -1,3 +1,3 @@
 module RANN
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rann
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Michael Campbell
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-08 00:00:00.000000000 Z
+date: 2017-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel
@@ -95,6 +95,7 @@ files:
 - LICENCE
 - README.md
 - Rakefile
+- bench/xor_benchmark.rb
 - bin/console
 - bin/setup
 - examples/xor.rb
@@ -106,6 +107,8 @@ files:
 - lib/rann/lstm.rb
 - lib/rann/network.rb
 - lib/rann/neuron.rb
+- lib/rann/optimisers/adagrad.rb
+- lib/rann/optimisers/rmsprop.rb
 - lib/rann/product_neuron.rb
 - lib/rann/util/array_ext.rb
 - lib/rann/version.rb