neural_network_rb 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0b0d6401b734f3aaf9ceb8fd0f500e5e8b587c90
-  data.tar.gz: c050c4adb730e487cbe4602d405d5e609e648808
+  metadata.gz: 181ecc0a7feea433a3e23e7de5aa9a34538d4550
+  data.tar.gz: dee6381c4cfa993f16f15d4759d3e2dd27a51533
 SHA512:
-  metadata.gz: 2966e79612f5a59b6b78961669ace3ea6181e3709da18f94fdbbdd0501f21d8ad98acb2dfacbee6e7db2d40032ec7824f1bd812339943ea385bb6ca5e25d06dc
-  data.tar.gz: 6f8b34eac283b5710018e16fbce57938ca135ea86b13b2d9f776f38f0945b9706c8c19c7a1308109da9f9ad665849a5ea643216d9790781fd5df058d67a1b357
+  metadata.gz: d212a1cb5bbf93ece105db806dfb3ee28e848b3ed3f8aa7a8e4dfb46a9b9fd214c2d964e55e332c3872056322638ae2f83026f97cf8fff630b0a92912fd05170
+  data.tar.gz: 0b981b368bb4f3cc8d6fef832cdb8b73c97208ead0d8e9a9ae832d447d2072f35d2bda849057ab0905b41c1dc69d8c2b89dcb7b8f3a87a4624402f4fa51d1ade

data/.gitignore

@@ -10,4 +10,5 @@
 # rspec failure tracking
 .rspec_status
 *.gz
-*.png
+*.png
+*.gem

data/Gemfile.lock

@@ -3,6 +3,7 @@ PATH
   specs:
     neural_network_rb (0.1.0)
       chunky_png (~> 1.3.10)
+      numo-linalg (~> 0.1.1)
       numo-narray (~> 0.9.1.1)
 
 GEM
@@ -36,6 +37,8 @@ GEM
     notiffany (0.1.1)
       nenv (~> 0.1)
       shellany (~> 0.0)
+    numo-linalg (0.1.1)
+      numo-narray (>= 0.9.0.7)
     numo-narray (0.9.1.1)
     pry (0.11.3)
       coderay (~> 1.1.0)

data/lib/neural_network_rb.rb

@@ -1,9 +1,16 @@
 require 'numo/narray'
+require 'numo/linalg/autoloader'
+require 'neural_network_rb/embeddings/one_hot'
+require 'neural_network_rb/activations/dot'
+require 'neural_network_rb/activations/sigmoid'
+require 'neural_network_rb/activations/relu'
+require 'neural_network_rb/loss/softmax_cross_entropy'
+require 'neural_network_rb/loss/cross_entropy_fetch'
 require 'neural_network_rb/narray'
 require 'neural_network_rb/mnist_image'
 require 'neural_network_rb/mnist'
 require "neural_network_rb/version"
-require 'neural_network_rb/softmax'
-require 'neural_network_rb/sigmoid'
+require 'neural_network_rb/activations'
 require 'neural_network_rb/error'
 require 'neural_network_rb/neural_network'
+require 'neural_network_rb/builder'

data/lib/neural_network_rb/activations.rb

@@ -0,0 +1,26 @@
+module NeuralNetworkRb
+
+  E = Math.exp(1)
+
+  class << self 
+    def sigmoid(array)
+      max_array = array.max(1)
+      array.map_with_index {|a, i| a - max_array[i]}
+           .map_with_index {|x, i| 1/(1 + E**(-x))}
+    end
+
+    def sigmoid_prime(x)
+      s = sigmoid(x)
+      s * (1-s)
+    end
+
+    def softmax(array)
+      max_array = array.max(1)
+      exp_array = array.map_with_index {|a, i| a - max_array[i]}.map {|x| Math.exp(x)}
+      sum_array = exp_array.sum(1)
+      exp_array.map_with_index {|a, i| a/sum_array[i]}
+    end
+  
+  end
+
+end

data/lib/neural_network_rb/activations/activations.rb

@@ -0,0 +1,5 @@
+module NeuralNetworkRb
+  module Activations
+    
+  end
+end

data/lib/neural_network_rb/activations/dot.rb

@@ -0,0 +1,56 @@
+require_relative 'activations'
+
+module NeuralNetworkRb
+  module Activations
+    class Dot
+
+      attr_reader :width, :height
+      attr_reader :weight, :bias
+
+      def initialize(next_layer, width: width, learning_rate: learning_rate)
+        @width = width
+        @learning_rate = learning_rate
+        @next_layer = next_layer
+      end
+
+      def train(input, target)
+        output = calc(input)
+        next_grad = @next_layer.train(output, target)
+
+        # learn by adjusting the weight and bias
+        @weight -= input.transpose.dot(next_grad) * @learning_rate
+        @bias   -= next_grad.sum(axis=0) * @learning_rate
+        
+        next_grad.dot(@weight.transpose)
+      end
+
+      def predict(input)
+        @next_layer.predict(calc(input))
+      end
+
+      def calc(input)
+        if @height.nil?
+          @height = input.shape[1]
+          @weight = init_weight(@width, @height)
+          @bias   = init_bias(@width)
+        end
+        Numo::Linalg.matmul(input, @weight) + @bias
+      end
+      
+      private 
+
+        def init_weight(width, height)
+           balanced_distr = Numo::DFloat.new( height, width).rand - 
+                            Numo::DFloat.ones(height, width)/2
+           balanced_distr * 0.01 / Math.sqrt(width)
+        end
+
+        def init_bias(width)
+          balanced_distr = Numo::DFloat.new(width).rand - 
+                           Numo::DFloat.ones(width)/2
+          balanced_distr * 0.01 / Math.sqrt(width)
+       end
+
+    end
+  end
+end

data/lib/neural_network_rb/activations/relu.rb

@@ -0,0 +1,29 @@
+require_relative 'activations'
+
+module NeuralNetworkRb
+  module Activations
+    class ReLU
+
+      def initialize(next_layer, options = {})
+        @next_layer = next_layer
+      end
+
+      def train(input, target)
+        output = calc(input)
+        grad(output) * @next_layer.train(output, target)
+      end
+
+      def predict(input)
+        @next_layer.predict(calc(input))          
+      end
+
+      def calc(x)
+        x * (x > 0)
+      end
+  
+      def grad(output)
+        1.0 * (output > 0)
+      end
+    end
+  end
+end

data/lib/neural_network_rb/activations/sigmoid.rb

@@ -0,0 +1,39 @@
+require_relative 'activations'
+
+module NeuralNetworkRb
+  module Activations
+    class Sigmoid
+
+      def initialize(next_layer, options = {})
+        @next_layer = next_layer
+      end
+
+      def train(input, target)
+        output = calc(input)
+        grad(output) * @next_layer.train(output, target)
+      end
+
+      def predict(input)
+        @next_layer.predict(calc(input))          
+      end
+
+      def call(input, target)
+        output = calc(input)
+        if target
+          grad(output) * @next_layer.call(output, target)
+        else
+          @next_layer.call(output, target)          
+        end
+      end
+
+      def calc(x)
+        1/(1+ Numo::NMath.exp(-(x - x.max(axis: 1, keepdims: true))))
+      end
+  
+      def grad(output)
+        # s = calc(x)
+        output * (1 - output)
+      end
+    end
+  end
+end

data/lib/neural_network_rb/builder.rb

@@ -0,0 +1,35 @@
+module NeuralNetworkRb
+  class NeuralNetwork
+    class Builder
+
+      attr_reader :layers, :named_layers
+
+      def initialize(random_seed = 1234, &block)
+        @layers = []
+        @named_layers = {}
+        @app = nil
+        Numo::NArray.srand(random_seed)
+        instance_eval(&block) if block_given?
+      end
+
+      def use(clazz, options = {}, &block)
+        @layers.unshift [clazz, options, block]
+      end
+
+      def to_network
+        layer = nil
+        @layers.each do |clazz, options, block|
+          layer_name = options.delete(:name)
+          layer = clazz.new(layer, options, &block)
+          @named_layers[layer_name] = layer if layer_name
+        end
+        @app = layer
+      end
+
+      def train(input, target)
+
+      end
+    end
+    
+  end
+end

data/lib/neural_network_rb/embeddings/one_hot.rb

@@ -0,0 +1,13 @@
+module NeuralNetworkRb
+  module Embeddings
+    class << self
+      def one_hot(labels, class_count)
+        Numo::Int8.zeros(labels.shape[0], class_count).tap do |matrix|
+          labels.each_with_index do |v, i|
+            matrix[i, v] = 1
+          end          
+        end
+      end
+    end
+  end
+end

data/lib/neural_network_rb/error.rb

@@ -8,8 +8,21 @@ module NeuralNetworkRb
       (v1 - v2).abs.sum
     end
 
+    def plain_diff(v1, v2)
+      v1 - v2
+    end
+
     def cross_entropy(values, labels)
       - (labels * values.map {|x| Math.log(x)}).sum/values.ndim
     end
+
+    def accuracy(values, labels)
+      size = values.shape[0]
+      hits = 0
+      size.times.each do |i|
+        hits += 1 if labels[i] == values[i, true].max_index
+      end
+      hits.to_f/size
+    end
   end
 end

data/lib/neural_network_rb/loss/cross_entropy_fetch.rb

@@ -0,0 +1,33 @@
+module NeuralNetworkRb
+  module Loss
+    class CrossEntropyFetch
+
+      attr_reader :epoch
+
+      def initialize(next_layer, options = {}, &block)
+        @next_layer = next_layer
+        @every = options[:every]
+        @block = block if block_given?
+        @epoch = 0
+      end
+
+      def train(input, target)
+        error = cross_entropy(input, target)
+        @block.call(@epoch, error) if @block && (@epoch % @every == 0)
+        @epoch +=1
+        @next_layer.nil? ? 1 : @next_layer.train(input, target)
+      end
+
+      def predict(input)
+        @next_layer.nil? ? input : @next_layer.predict(input)
+      end
+
+      private
+
+        def cross_entropy(input, target)
+          - (target * input.map {|x| Math.log(x)}).sum/input.ndim
+        end
+
+    end
+  end
+end

data/lib/neural_network_rb/loss/softmax_cross_entropy.rb

@@ -0,0 +1,36 @@
+module NeuralNetworkRb
+  module Loss
+    class SoftmaxCrossEntropy
+
+      def initialize(next_layer, options = {})
+        @next_layer = next_layer
+      end
+
+      def train(input, target)
+        output = softmax(input)
+        grad(output, target) * @next_layer.train(output, target)
+      end
+
+      def predict(input)
+        @next_layer.predict(softmax(input))  
+      end
+
+      def call(input, target)
+        error = cross_entropy(softmax(input), target)
+        grad(input, target) if target
+      end
+
+      private 
+        
+        def grad(input, target)
+          input - target        
+        end
+      
+        def softmax(x)
+          e_x = Numo::NMath.exp(x - x.max(axis: 1, keepdims: true))
+          e_x / e_x.sum(axis: 1, keepdims: true)
+        end
+
+    end
+  end
+end

data/lib/neural_network_rb/mnist.rb

@@ -7,44 +7,93 @@ module NeuralNetworkRb
 
     N_FEATURES = 28 * 28
     N_CLASSES = 10
-    FILE_NAMES = %w(train-images-idx3-ubyte.gz train-labels-idx1-ubyte.gz t10k-images-idx3-ubyte.gz t10k-labels-idx1-ubyte.gz)
+    TRAIN_FILE_NAMES = %w(train-images-idx3-ubyte.gz train-labels-idx1-ubyte.gz)
+    TEST_FILE_NAMES = %w(t10k-images-idx3-ubyte.gz t10k-labels-idx1-ubyte.gz)
     ROOT_URL = 'http://yann.lecun.com/exdb/mnist/'
 
     class << self
-      def download!
-        FILE_NAMES.each do |name|
-          url = "#{ROOT_URL}#{name}"
-          fetch_file(name, url)
+
+      def training_set
+        download(TRAIN_FILE_NAMES)        
+      end
+
+      def test_set
+        download(TEST_FILE_NAMES)        
+      end
+
+      def display_image(narray)
+        narray.shape[0].times do |r|
+          puts narray[r*28..((r+1)*28-1)].to_a.inspect
         end
-        self.new
       end
 
-      private
+      def embed_labels(labels, algorithm, class_count)
+        NeuralNetworkRb::Embeddings.send(algorithm, labels, class_count)
+      end
+  
+      private 
+        def download(files)
+          files.each do |name|
+            if !File.exists?(name) 
+              url = "#{ROOT_URL}#{name}"
+              fetch_file(name, url)
+            end
+          end
+          self.new(*files)
+        end
+
         def fetch_file(filePath, url)
           File.open(filePath, "w") do |output|
             IO.copy_stream(open(url), output)
           end
         end
+
     end
 
-    def training_labels
-      labels(FILE_NAMES[1])
+    attr_accessor :labels, :data, :validation_data, :validation_labels
+
+    def initialize(data_file = nil, label_file = nil)
+      @labels = get_labels(label_file) if label_file
+      @data   = get_images(data_file)  if data_file
     end
 
-    def test_labels
-      labels(FILE_NAMES[3])
+    def clone
+      self.class.new.tap do |m|
+        m.data = self.data.copy
+        m.labels = self.labels.copy
+        m.validation_data = self.validation_data.copy if self.validation_data
+        m.validation_labels = self.validation_labels.copy if self.validation_labels
+      end
+    end
+    
+    def shuffle!(seed)
+      @data, @labels = NeuralNetworkRb.shuffle(@data, @labels, seed)
+      self
     end
 
-    def training_images
-      images(FILE_NAMES[0])
+    def partition!(train_ratio)
+      @data, @validation_data = NeuralNetworkRb.split(@data, train_ratio)
+      @labels, @validation_labels = NeuralNetworkRb.split(@labels, train_ratio)
+      self
     end
 
-    def test_images
-      images(FILE_NAMES[2])
+
+    def batches(batches_count)
+      total_size = self.data.shape[0]
+      batch_size = (total_size.to_f/batches_count).ceil
+      Array.new(batches_count).tap do |result|
+        batches_count.times do |i|
+          range = batch_size*i..batch_size*(i+1)-1
+          batch_data   = NeuralNetworkRb.rows(@data,   range) # self.data[batch_size*i..batch_size*(i+1)-1, true]
+          batch_labels = NeuralNetworkRb.rows(@labels, range) # self.labels[batch_size*i..batch_size*(i+1)-1]
+          result[i] = [batch_data, batch_labels]
+        end
+      end
     end
 
     private 
-      def images(file_name)
+
+      def get_images(file_name)
         images = []
         Zlib::GzipReader.open(file_name) do |f|
           _, n_images = f.read(8).unpack('N2')
@@ -57,7 +106,7 @@ module NeuralNetworkRb
         Numo::Int16.cast(images)  
       end
 
-      def labels(file_name)
+      def get_labels(file_name)
         labels = nil
         Zlib::GzipReader.open(file_name) do |f|
           _, @n_labels = f.read(8).unpack('N2')

data/lib/neural_network_rb/narray.rb

@@ -2,24 +2,21 @@ module NeuralNetworkRb
 
   class << self
 
-    def shuffle(data, target)
+    def shuffle(data, target, seed)
       sample_size = data.shape[0]
-      new_order = Numo::DFloat[*(0..sample_size-1).to_a.shuffle]
-      [data[*subset(data, new_order)], target[*subset(target, new_order)]]
+      new_order = Numo::DFloat[*(0..sample_size-1).to_a.shuffle(random: Random.new(seed))]
+      [rows(data, new_order), rows(target, new_order)]
     end
 
     def split(data, ratio)
       sample_size = (data.shape[0] * ratio).to_i
-      [data[*subset(data, 0..sample_size-1)], data[*subset(data, sample_size..-1)]]
+      [rows(data, 0..sample_size-1), rows(data, sample_size..-1)]
+    end
+
+    def rows(data, obj)
+      data[obj, *Array.new(data.ndim-1, true)]
     end
-    
-    private
 
-      def subset(data, obj)
-        Array.new(data.ndim, true).tap do |x|
-          x[0] = obj
-        end
-      end
   end
 
 end

data/lib/neural_network_rb/neural_network.rb

@@ -5,41 +5,57 @@ module NeuralNetworkRb
     attr_reader :target, :output, :epoch
     
     def input=(input)
-      @input = input
+      @input = input/255.0
       input_width = input.shape[1]
-      @w_hidden = Numo::DFloat.new(input_width,  @neurons_count).rand
+      @w1 = (Numo::DFloat.new(input_width,  @neurons_count).rand - 
+             Numo::DFloat.ones(input_width,  @neurons_count)/2) * 0.01 / Math.sqrt(input_width)
+      @b1 = Numo::DFloat.zeros(@neurons_count)
     end
 
     def target=(target)
       @target = target
       output_width = target.shape[1]
-      @w_output = Numo::DFloat.new(@neurons_count, output_width).rand
+      @w2 = (Numo::DFloat.new(@neurons_count, output_width).rand -
+             Numo::DFloat.ones(@neurons_count, output_width)/2) * 0.01 / Math.sqrt(@neurons_count)
+      @b2 = Numo::DFloat.zeros(output_width)
     end
 
-    def initialize(neurons_count, learning_rate)
+    def initialize(neurons_count, learning_rate, random_seed)
       @neurons_count = neurons_count
       @learning_rate = learning_rate
       @epoch = 0
+      Numo::NArray.srand(random_seed)
     end
 
-    def train()
-      # forward
-      @hidden = NeuralNetworkRb.sigmoid(@input.dot(@w_hidden))
-      @output = @hidden.dot(@w_output)
+    def fit()
+      forward
+      backprop
+      @epoch += 1 
+      yield self if block_given?
+    end
 
-      # calculate error
-      error = @target - @output
+    def forward
+      # forward
+      @hidden = NeuralNetworkRb.sigmoid(
+                  Numo::Linalg.matmul(@input, @w1) + 
+                  @b1)
+      @output = NeuralNetworkRb.softmax(
+                  Numo::Linalg.matmul(@hidden, @w2) + 
+                  @b2)
+    end
 
+    def backprop
       # backward 
-      dZ = error * @learning_rate
-
-      @w_output += @hidden.transpose.dot(dZ)
-      dH = dZ.dot(@w_output.transpose) * NeuralNetworkRb.sigmoid_prime(@hidden)
-      @w_hidden += @input.transpose.dot(dH)
-
-      @epoch += 1 
-      yield self if block_given?
+      dZ   = NeuralNetworkRb.plain_diff(@target,  @output) 
+      @w2 += Numo::Linalg.matmul(@hidden.transpose, dZ) * @learning_rate
+      dH   = Numo::Linalg.matmul(dZ, @w2.transpose) * NeuralNetworkRb.sigmoid_prime(@hidden)
+      @w1 += Numo::Linalg.matmul(@input.transpose, dH) * @learning_rate
     end
 
+    def predict(input)
+      hidden = NeuralNetworkRb.sigmoid(Numo::Linalg.matmul(input/255.0, @w1))
+      output = NeuralNetworkRb.softmax(Numo::Linalg.matmul(hidden, @w2))
+      output
+    end
   end
 end

data/lib/neural_network_rb/version.rb

@@ -1,3 +1,3 @@
 module NeuralNetworkRb
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/neural_network_rb.gemspec

@@ -29,6 +29,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "numo-narray", "~> 0.9.1.1"
+  spec.add_dependency "numo-linalg", "~> 0.1.1"
   spec.add_dependency "chunky_png", "~> 1.3.10"
 
   spec.add_development_dependency "bundler", "~> 1.16"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: neural_network_rb
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Jack Xu
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-08 00:00:00.000000000 Z
+date: 2018-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.1.1
+- !ruby/object:Gem::Dependency
+  name: numo-linalg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.1
 - !ruby/object:Gem::Dependency
   name: chunky_png
   requirement: !ruby/object:Gem::Requirement
@@ -96,13 +110,20 @@ files:
 - README.md
 - Rakefile
 - lib/neural_network_rb.rb
+- lib/neural_network_rb/activations.rb
+- lib/neural_network_rb/activations/activations.rb
+- lib/neural_network_rb/activations/dot.rb
+- lib/neural_network_rb/activations/relu.rb
+- lib/neural_network_rb/activations/sigmoid.rb
+- lib/neural_network_rb/builder.rb
+- lib/neural_network_rb/embeddings/one_hot.rb
 - lib/neural_network_rb/error.rb
+- lib/neural_network_rb/loss/cross_entropy_fetch.rb
+- lib/neural_network_rb/loss/softmax_cross_entropy.rb
 - lib/neural_network_rb/mnist.rb
 - lib/neural_network_rb/mnist_image.rb
 - lib/neural_network_rb/narray.rb
 - lib/neural_network_rb/neural_network.rb
-- lib/neural_network_rb/sigmoid.rb
-- lib/neural_network_rb/softmax.rb
 - lib/neural_network_rb/version.rb
 - neural_network_rb.gemspec
 homepage: https://github.com/jackxxu/neural_network_rb

data/lib/neural_network_rb/sigmoid.rb

@@ -1,16 +0,0 @@
-module NeuralNetworkRb
-
-  E = Math.exp(1)
-
-  class << self 
-    def sigmoid(x)
-      1/(1 + E**(-x))
-    end
-
-    def sigmoid_prime(x)
-      s = sigmoid(x)
-      s * (1-s)
-    end
-  end
-
-end

data/lib/neural_network_rb/softmax.rb

@@ -1,8 +0,0 @@
-module NeuralNetworkRb
-
-  def self.softmax(array)
-    exp_array = array.map {|x| Math.exp(x)}
-    exp_array = exp_array/exp_array.sum
-  end
-
-end