ruby_brain 0.1.0

data/examples/mnist2.rb

@@ -0,0 +1,82 @@
+require 'ruby_brain'
+require 'ruby_brain/dataset/mnist/data'
+
+NUM_TRAIN_DATA = 5000
+NUM_TEST_DATA = 500
+
+dataset = RubyBrain::DataSet::Mnist::data
+
+training_input = dataset[:input][0..(NUM_TRAIN_DATA-1)]
+training_supervisor = dataset[:output][0..(NUM_TRAIN_DATA-1)]
+
+# test_input = dataset[:input][NUM_TRAIN_DATA..(NUM_TRAIN_DATA+NUM_TEST_DATA-1)]
+# test_supervisor = dataset[:output][NUM_TRAIN_DATA..(NUM_TRAIN_DATA+NUM_TEST_DATA-1)]
+test_input = dataset[:input][NUM_TRAIN_DATA..-1]
+test_supervisor = dataset[:output][NUM_TRAIN_DATA..-1]
+
+network = RubyBrain::Network.new([dataset[:input].first.size, 50, dataset[:output].first.size])
+# network.learning_rate = 0.7
+network.init_network
+network.load_weights_from_yaml_file(File.dirname(__FILE__) + '/../best_weights_1469044985.yml')
+
+### You can initializes weights by loading weights from file if you want.
+# network.load_weights_from_yaml_file('path/to/weights.yml.file')
+
+# network.learn(training_input, training_supervisor, max_training_count=100, tolerance=0.0004, monitoring_channels=[:best_params_training])
+
+### You can save weights into a yml file if you want.
+# network.dump_weights_to_yaml('path/to/weights.yml.file')
+
+
+class Array
+  def argmax
+    max_i = 0
+    max_val = self[max_i]
+    self.each_with_index do |v, i|
+      if v > max_val
+        max_val = v
+        max_i = i
+      end
+    end
+    return max_i
+  end
+end
+
+results = []
+test_input.each_with_index do |input, i|
+  ### You can see test input, label and predicated lable in standard out if you uncomment in this block 
+
+  input.each_with_index do |e, j|
+    print(e > 0.3 ? 'x' : ' ')
+    puts if (j % 28) == 0
+  end
+  puts
+  supervisor_label = test_supervisor[i].argmax
+  predicated_label = network.get_forward_outputs(test_input[i]).argmax
+  puts "test_supervisor: #{supervisor_label}"
+  puts "predicate: #{predicated_label}"
+  results << (supervisor_label == predicated_label)
+  puts "------------------------------------------------------------"
+end
+
+puts "accuracy: #{results.count(true).to_f/results.size}"
+
+
+
+### you can do above procedure simply by using Trainer
+
+# training_option = {
+#   learning_rate: 0.5,
+#   max_training_count: 50,
+#   tolerance: 0.0004,
+#   # initial_weights_file: 'weights_3_30_10_1429166740.yml',
+#   # initial_weights_file: 'best_weights_1429544001.yml',
+#   monitoring_channels: [:best_params_training]
+# }
+
+# RubyBrain::Trainer.normal_learning([dataset[:input].first.size, 50, dataset[:output].first.size],
+#                                    training_input, training_supervisor,
+#                                    training_option)
+
+
+

data/lib/ruby_brain.rb

@@ -0,0 +1,19 @@
+require "ruby_brain/version"
+require 'forwardable'
+require 'yaml'
+require 'pp'
+
+module RubyBrain
+  require "ruby_brain/nodes"
+  require "ruby_brain/layer"
+  require "ruby_brain/weights"
+  require "ruby_brain/network"
+  require "ruby_brain/trainer"
+  require 'ruby_brain/exception'
+  require 'ruby_brain/training_data_manipulator'
+  
+  module Nodes end
+  module Trainer end
+  module Exception end
+  module DataSet end
+end

data/lib/ruby_brain/dataset/mnist/data.rb

@@ -0,0 +1,62 @@
+module RubyBrain::DataSet::Mnist
+
+  require 'mnist'
+  require 'open-uri'
+  
+  def download_file(target_url, dest_path)
+    File.open(dest_path, "wb") do |saved_file|
+      open(target_url, "rb") do |read_file|
+        saved_file.write(read_file.read)
+      end
+    end
+  end
+
+  def data
+    train_images_path = Dir.pwd + '/train-images-idx3-ubyte.gz'
+    train_labels_path = Dir.pwd + '/train-labels-idx1-ubyte.gz'
+    
+    unless File.exist?(train_images_path)
+      puts 'downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz ...'
+      download_file('http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz', train_images_path) 
+    end
+    
+    unless File.exist?(train_labels_path)
+      puts 'downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz'
+      download_file('http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz', train_labels_path) 
+    end
+
+    train_images = Mnist.load_images(train_images_path)
+    train_labels = Mnist.load_labels(train_labels_path)
+
+    input_training_set = train_images[2].map do |image|
+      image.unpack('C*').map {|e| e / 255.0}
+    end
+    
+    output_training_set = train_labels.map do |label|
+      one_hot_vector = Array.new(10, 0)
+      one_hot_vector[label] = 1
+      one_hot_vector
+    end
+
+    # puts train_images[0].class
+    # puts train_images[1].class
+    # puts train_images[2].size
+    # puts train_images[2][0].size
+    # puts train_images[2][59999][783].class
+    # puts train_images[2][59999].class
+    # puts "------------------------------"
+
+    # 10.times do |j|
+    #   train_images[2][j].unpack('C*').each_with_index do |e, i|
+    #     print(e > 50 ? 'x' : ' ')
+    #     puts if (i % 28) == 0
+    #   end
+    #   puts
+    #   puts train_labels[j]
+    # end
+    
+    {input: input_training_set, output: output_training_set}
+  end
+  
+  module_function :data, :download_file
+end

data/lib/ruby_brain/dataset/mnist/test_mnist.rb

@@ -0,0 +1,28 @@
+require 'mnist'
+
+
+train_images = Mnist.load_images('./train-images-idx3-ubyte.gz')
+train_labels = Mnist.load_labels('./train-labels-idx1-ubyte.gz')
+
+puts train_images[0].class
+puts train_images[1].class
+puts train_images[2].size
+puts train_images[2][0].size
+puts train_images[2][59999][783].class
+puts train_images[2][59999].class
+puts "------------------------------"
+
+10.times do |j|
+  train_images[2][j].unpack('C*').each_with_index do |e, i|
+    print(e > 50 ? 'x' : ' ')
+    puts if (i % 28) == 0
+  end
+  puts
+  puts train_labels[j]
+end
+
+
+
+
+
+

data/lib/ruby_brain/exception.rb

@@ -0,0 +1,17 @@
+
+
+module RubyBrain
+  module Exception
+
+    class RubyBrainError < StandardError
+    end
+
+
+    class DataDimensionError < RubyBrainError
+    end
+
+    class TrainingDataError < DataDimensionError
+    end
+
+  end
+end

data/lib/ruby_brain/layer.rb

@@ -0,0 +1,37 @@
+module RubyBrain
+  class Layer
+    attr_accessor :input_weights, :output_weights
+    attr_reader :next_node_order_index, :nodes
+
+    def initialize
+      @nodes = []
+      @next_node_order_index = 0
+    end
+
+    def append(node)
+      node.order_index = @next_node_order_index
+      node.left_side_weights = @input_weights
+      node.right_side_weights = @output_weights
+      @nodes << node
+      @next_node_order_index += 1
+    end
+
+    def num_nodes
+      @nodes.size
+    end
+
+    def each_node
+      @nodes.each do |node|
+        yield node
+      end
+    end
+
+    def forward_outputs(inputs=[])
+      @nodes.map { |node| node.output_of_forward_calc(inputs) }
+    end
+
+    def backward_outputs(inputs)
+      @nodes.map { |node| node.output_of_backward_calc(inputs) }.compact
+    end
+  end
+end

data/lib/ruby_brain/network.rb

@@ -0,0 +1,252 @@
+module RubyBrain
+  class Network
+    extend Forwardable
+    def_delegators :@weights_set, :overwrite_weights, :get_weights_as_array
+
+    attr_accessor :learning_rate
+
+    def initialize(num_units_list)
+      @layers = []
+      @num_units_list = num_units_list
+      @weights_set = WeightContainer.new(@num_units_list)
+    end
+
+    def load_weights_from(weights_set_source)
+      @weights_set.load_from(weights_set_source)
+      init_network
+    end
+
+    # def overwrite_weights(weights_set_source)
+    #   @weights_set.overwrite_weights(weights_set_source)
+    # end
+
+    def init_network
+      @layers = []
+      layer = Layer.new
+      (@num_units_list[0] + 1).times do
+        layer.append Nodes::ConstNode.new
+        layer.output_weights = @weights_set.weights_of_order(0)
+      end
+      @layers << layer
+
+      @num_units_list[1..-2].each_with_index do |num_units, i|
+        layer = Layer.new
+        layer.input_weights = @weights_set.weights_of_order(i)
+        layer.output_weights = @weights_set.weights_of_order(i+1)
+        (num_units).times do
+          layer.append Nodes::Neuron.new
+        end
+        layer.append Nodes::ConstNode.new
+        @layers << layer
+      end
+
+      layer = Layer.new
+      layer.input_weights = @weights_set.weights_of_order(@num_units_list.size - 2)
+      @num_units_list[-1].times do
+        layer.append Nodes::Neuron.new
+      end
+      @layers << layer
+    end
+
+    # def get_weights_as_array
+    #   @weights_set.get_weights_as_array
+    # end
+
+
+    def get_forward_outputs(inputs)
+      inputs.each_with_index do |input, i|
+        @layers.first.nodes[i].value = input
+      end
+
+      a_layer_outputs = nil
+      a_layer_inputs = @layers.first.forward_outputs
+      @layers.each do |layer|
+        a_layer_outputs = layer.forward_outputs(a_layer_inputs)
+        a_layer_inputs = a_layer_outputs
+      end
+      a_layer_outputs
+    end
+
+    def run_backpropagate(backward_inputs)
+      a_layer_outputs = nil
+      a_layer_inputs = backward_inputs
+      @layers.reverse[0..-2].each do |layer|
+        a_layer_outputs = layer.backward_outputs(a_layer_inputs)
+        a_layer_inputs = a_layer_outputs
+      end
+      a_layer_outputs
+    end
+
+    def update_weights
+      @weights_set.each_weights_with_index do |weights, i|
+        weights.each_with_index do |wl, j|
+          wl.each_with_index do |w, k|
+            wl[k] = w - (@learning_rate * @layers[i].nodes[j].this_output * @layers[i+1].nodes[k].this_backward_output)
+          end
+        end
+      end
+    end
+
+    def update_weights_of_layer(layer_index)
+      layer_index = @weights_set.num_sets + layer_index if layer_index < 0
+      @weights_set.each_weights_with_index do |weights, i|
+        next if i != layer_index
+        weights.each_with_index do |wl, j|
+          wl.each_with_index do |w, k|
+            wl[k] = w - (@learning_rate * @layers[i].nodes[j].this_output * @layers[i+1].nodes[k].this_backward_output)
+          end
+        end
+      end
+    end
+
+    # def calculate_rms_error(training_inputs_set, training_outputs_set)
+    #   accumulated_errors = 0.0
+    #   training_inputs_set.zip(training_outputs_set).each do |t_input, t_output|
+    #     forward_outputs = get_forward_outputs(t_input)
+    #     total_error_of_output_nodes = 0.0
+    #     forward_outputs.zip(t_output).each do |o, t|
+    #       total_error_of_output_nodes += (o - t)**2 / 2.0
+    #     end
+    #     accumulated_errors += total_error_of_output_nodes / forward_outputs.size
+    #   end
+    #   Math.sqrt(2.0 * accumulated_errors / training_inputs_set.size)
+    # end
+
+    def learn(inputs_set, outputs_set, max_training_count=50, tolerance=0.0, monitoring_channels=[])
+      raise RubyBrain::Exception::TrainingDataError if inputs_set.size != outputs_set.size
+      #      raise "inputs_set and outputs_set has different size!!!!" if inputs_set.size != outputs_set.size
+
+      best_error = 9999999999999
+      best_weights_array = []
+      max_training_count.times do |i_training|
+        accumulated_errors = 0.0 # for rms
+        inputs_set.zip(outputs_set).each do |t_input, t_output|
+          forward_outputs = get_forward_outputs(t_input)
+          # for rms start
+          total_error_of_output_nodes = forward_outputs.zip(t_output).reduce(0.0) do |a, output_pair|
+            a + ((output_pair[0] - output_pair[1])**2 / 2.0)
+          end
+          # end
+          accumulated_errors += total_error_of_output_nodes / forward_outputs.size
+          # accumulated_errors += forward_outputs.zip(t_output).reduce(0.0) { |a, output_pair| a + ((output_pair[0] - output_pair[1])**2 / 2.0) } / forward_outputs.size
+          # for rms end
+          backward_inputs = forward_outputs.zip(t_output).map { |o, t| o - t }
+          run_backpropagate(backward_inputs)
+          update_weights
+        end
+
+        rms_error = Math.sqrt(2.0 * accumulated_errors / inputs_set.size) # for rms
+        # rms_error = calculate_rms_error(inputs_set, outputs_set)
+        puts "--> #{rms_error} (#{i_training}/#{max_training_count})"
+        
+        if rms_error < best_error
+          puts "update best!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+          best_error = rms_error
+          best_weights_array = @weights_set.get_weights_as_array
+        end
+        puts "best: #{best_error}"
+
+
+        break if rms_error <= tolerance
+      end
+      if monitoring_channels.include? :best_params_training
+        File.open "best_weights_#{Time.now.to_i}.yml", 'w+' do |f|
+          YAML.dump(best_weights_array, f)
+        end
+      end
+    end
+
+
+    def learn2(inputs_set, outputs_set, max_training_count=50, tolerance=0.0, monitoring_channels=[])
+      # looks like works well for networks which has many layers... [1, 10, 10, 10, 1], [1, 100, 100, 100, 1]
+      # looks like NOT works well for networks which has many units in a layer... [1, 100, 1]
+      raise RubyBrain::Exception::TrainingDataError if inputs_set.size != outputs_set.size
+      # raise "inputs_set and outputs_set has different size!!!!" if inputs_set.size != outputs_set.size
+      initial_learning_rate = @learning_rate
+
+      rms_error = Float::INFINITY
+      max_training_count.times do |i_training|
+        accumulated_errors = 0.0 # for rms
+        inputs_set.zip(outputs_set).each do |t_input, t_output|
+          forward_outputs = get_forward_outputs(t_input)
+          # for rms start
+          total_error_of_output_nodes = forward_outputs.zip(t_output).reduce(0.0) do |a, output_pair|
+            a + ((output_pair[0] - output_pair[1])**2 / 2.0)
+          end
+          # end
+          error_of_this_training_data = total_error_of_output_nodes / forward_outputs.size
+          accumulated_errors += error_of_this_training_data
+          # accumulated_errors += forward_outputs.zip(t_output).reduce(0.0) { |a, output_pair| a + ((output_pair[0] - output_pair[1])**2 / 2.0) } / forward_outputs.size
+          # for rms end
+          # if error_of_this_training_data > rms_error**2/2.0
+          #   @learning_rate *= 10.0
+          # end
+          backward_inputs = forward_outputs.zip(t_output).map { |o, t| o - t }
+          run_backpropagate(backward_inputs)
+          update_weights
+          # @learning_rate = initial_learning_rate
+        end
+
+        rms_error = Math.sqrt(2.0 * accumulated_errors / inputs_set.size) # for rms
+
+        # rms_error = calculate_rms_error(inputs_set, outputs_set)
+        puts "--> #{rms_error} (#{i_training}/#{max_training_count})"
+        break if rms_error <= tolerance
+      end
+    end
+
+
+    def learn_only_specified_layer(layer_index, inputs_set, outputs_set, max_training_count=50, tolerance=0.0)
+      # looks like works well for networks which has many layers... [1, 10, 10, 10, 1], [1, 100, 100, 100, 1]
+      # looks like NOT works well for networks which has many units in a layer... [1, 100, 1]
+      raise "inputs_set and outputs_set has different size!!!!" if inputs_set.size != outputs_set.size
+      initial_learning_rate = @learning_rate
+
+      rms_error = Float::INFINITY
+      max_training_count.times do |i_training|
+        accumulated_errors = 0.0 # for rms
+        inputs_set.zip(outputs_set).each do |t_input, t_output|
+          forward_outputs = get_forward_outputs(t_input)
+          # for rms start
+          total_error_of_output_nodes = forward_outputs.zip(t_output).reduce(0.0) do |a, output_pair|
+            a + ((output_pair[0] - output_pair[1])**2 / 2.0)
+          end
+          # end
+          error_of_this_training_data = total_error_of_output_nodes / forward_outputs.size
+          accumulated_errors += error_of_this_training_data
+          # accumulated_errors += forward_outputs.zip(t_output).reduce(0.0) { |a, output_pair| a + ((output_pair[0] - output_pair[1])**2 / 2.0) } / forward_outputs.size
+          # for rms end
+          if error_of_this_training_data > rms_error**2/2.0
+            @learning_rate *= 10.0
+          end
+          backward_inputs = forward_outputs.zip(t_output).map { |o, t| o - t }
+          run_backpropagate(backward_inputs)
+          update_weights_of_layer(layer_index)
+          @learning_rate = initial_learning_rate
+        end
+
+        rms_error = Math.sqrt(2.0 * accumulated_errors / inputs_set.size) # for rms
+
+        # rms_error = calculate_rms_error(inputs_set, outputs_set)
+        puts "--> #{rms_error} (#{i_training}/#{max_training_count})"
+        break if rms_error <= tolerance
+      end
+    end
+
+
+    def dump_weights
+      @weights_set.each_weights do |weights|
+        pp weights
+      end
+    end
+
+    def dump_weights_to_yaml(file_name=nil)
+      @weights_set.dump_to_yaml(file_name)
+    end
+
+    def load_weights_from_yaml_file(yaml_file)
+      @weights_set.load_from_yaml_file(yaml_file)
+    end
+
+  end
+end