sabina 0.1.0

data/lib/sabina/auto_encoder.rb

@@ -0,0 +1,46 @@
+module Sabina
+  class AutoEncoder < MultilayerPerceptron
+
+    def self.load_csv(file_name)
+      table = CSV.table(file_name)
+      table.map do |data|
+        x = (data.size-1).times.map { |d| data["x#{d}".to_sym] }
+        { :x => x, :d => x }
+      end
+    end
+
+    # Check if `@layers` is valid.
+    def check_layers
+      super
+
+      if layers.size != 3
+        raise "The number of layers must be three."
+      end
+
+      if layers.first.size != layers.last.size
+        raise "The number of units of the input layer must be equal to that of the output layer."
+      end
+    end
+
+    # Error function (a example of squared error)
+    def error(test_data)
+      d = Matrix.columns( test_data.map { |data| data[:d] } )
+      y = propagate_forward(d)
+      (d - y).to_a.flatten.inject(0.0) { |sum, v| sum + v**2 }
+    end
+
+    def next_input_data(input_data)
+      x = Matrix.columns( input_data.map { |data| data[:x] } )
+      propagate_forward(x)
+      @Z[1].t.to_a.map do |z|
+        { :x => z, :d => z }
+      end
+    end
+
+    # Update the weights of this auto-encoder.
+    def update
+      super
+      @layers[2].W = Marshal.load(Marshal.dump(@layers[1].W.t))
+    end
+  end
+end

data/lib/sabina/configuration.rb

@@ -0,0 +1,41 @@
+module Sabina
+  module Configuration
+
+    OPTIONS_KEYS = [
+      :layers,
+      :mini_batch_size,
+      :training_data,
+      :learning_rate,
+    ].freeze
+
+    DEFAULTS = {
+      :layers => [
+        Sabina::Layer::MPInputLayer.new(1),
+        Sabina::Layer::MPHiddenLayer.new(1),
+        Sabina::Layer::MPOutputLayer.new(1)
+      ],
+      :mini_batch_size => 10,
+      :training_data => [{:x => [0], :d => [0]}],
+      :learning_rate => 0.01,
+    }
+
+    attr_accessor *OPTIONS_KEYS
+
+    # This method is used for setting configuration options.
+    def configure
+      yield self
+    end
+
+    # Create a hash of options.
+    def options
+      Hash[*OPTIONS_KEYS.map{ |key| [key, send(key)] }.flatten(1)]
+    end
+
+    # Reset all options to their default values.
+    def reset
+      DEFAULTS.each do |option, default|
+        self.send("#{option}=".to_sym, default)
+      end
+    end
+  end
+end

data/lib/sabina/layer/ae_hidden_layer.rb

@@ -0,0 +1,10 @@
+module Sabina::Layer
+  class AEHiddenLayer < BaseLayer
+
+    def initialize(size)
+      super
+      @f = ->(x){ x }
+      @f_ = ->(_){ 1.0 }
+    end
+  end
+end

data/lib/sabina/layer/ae_input_layer.rb

@@ -0,0 +1,4 @@
+module Sabina::Layer
+  class AEInputLayer < BaseLayer
+  end
+end

data/lib/sabina/layer/ae_output_layer.rb

@@ -0,0 +1,10 @@
+module Sabina::Layer
+  class AEOutputLayer < BaseLayer
+
+    def initialize(size)
+      super
+      @f = ->(x){ x }
+      @f_ = ->(_){ 1.0 }
+    end
+  end
+end

data/lib/sabina/layer/base_layer.rb

@@ -0,0 +1,33 @@
+module Sabina::Layer
+  class BaseLayer
+    attr_reader :size
+    attr_accessor :b, :W, :I, :J
+
+    def initialize(size)
+      @size = size
+      @f = ->(x){ 1.0 / (1.0 + Math.exp(-x)) }
+      @f_ = ->(x){ @f[x]*(1.0 - @f[x]) }
+    end
+
+    # Initialize the weights of this layer.
+    def init_weight
+      # (J, 1)
+      @b = Matrix.columns([Array.new(@size) { 0.0 }])
+
+      # (J, I)
+      @W = Array.new(@J) do
+        Array.new(@I) { Sabina::Util.box_muller }
+      end.tap { |ary| break Matrix[*ary] }
+    end
+
+    # An activation function
+    def activate(u_ary)
+      u_ary.map { |u| @f[u] }
+    end
+
+    # Differentiation of activation function
+    def activate_(u_ary)
+      u_ary.map { |u| @f_[u] }
+    end
+  end
+end

data/lib/sabina/layer/mp_hidden_layer.rb

@@ -0,0 +1,4 @@
+module Sabina::Layer
+  class MPHiddenLayer < BaseLayer
+  end
+end

data/lib/sabina/layer/mp_input_layer.rb

@@ -0,0 +1,4 @@
+module Sabina::Layer
+  class MPInputLayer < BaseLayer
+  end
+end

data/lib/sabina/layer/mp_output_layer.rb

@@ -0,0 +1,12 @@
+module Sabina::Layer
+  class MPOutputLayer < BaseLayer
+
+    # softmax function
+    def activate(u_ary)
+      sum = u_ary.inject(0.0) { |s, u| s + Math.exp(u) }
+      u_ary.map do |u|
+        Math.exp(u) / sum
+      end
+    end
+  end
+end

data/lib/sabina/layer.rb

@@ -0,0 +1,11 @@
+module Sabina
+  module Layer
+    autoload :MPInputLayer, "sabina/layer/mp_input_layer"
+    autoload :MPHiddenLayer, "sabina/layer/mp_hidden_layer"
+    autoload :MPOutputLayer, "sabina/layer/mp_output_layer"
+
+    autoload :AEInputLayer, "sabina/layer/ae_input_layer"
+    autoload :AEHiddenLayer, "sabina/layer/ae_hidden_layer"
+    autoload :AEOutputLayer, "sabina/layer/ae_output_layer"
+  end
+end

data/lib/sabina/multilayer_perceptron.rb

@@ -0,0 +1,135 @@
+module Sabina
+  class MultilayerPerceptron
+    attr_accessor *Configuration::OPTIONS_KEYS
+    LAMBDA = 0.001
+    MU = 0.5
+
+    def self.load_csv(file_name)
+      table = CSV.table(file_name)
+      k = table[:label].max + 1
+      table.map do |data|
+        x = (data.size-1).times.map { |d| data["x#{d}".to_sym] }
+        d = Array.new(k) { |i| i == data[:label] ? 1 : 0 }
+        { :x => x, :d => d }
+      end
+    end
+
+    def initialize(options = {})
+      load_config(options)
+      check_layers
+      @L = @layers.size - 1
+      @K = @layers.last.size
+      @delta_W_prev = []
+      @delta_b_prev = []
+
+      @layers[0].J = @layers[0].size
+      (1..@L).each do |j|
+        @layers[j].I = @layers[j-1].size
+        @layers[j].J = @layers[j].size
+        @layers[j].init_weight
+        @delta_W_prev[j] = Marshal.load(Marshal.dump(@layers[j].W * 0.0))
+        @delta_b_prev[j] = Marshal.load(Marshal.dump(@layers[j].b * 0.0))
+      end
+
+      @mini_batches =
+        @training_data.shuffle
+        .each_slice(@mini_batch_size).to_a
+    end
+
+    # Load the configuration.
+    def load_config(options = {})
+      merged_options = Sabina.options.merge(options)
+      Configuration::OPTIONS_KEYS.each do |key|
+        send("#{key}=".to_sym, merged_options[key])
+      end
+    end
+
+    # Check if `@layers` is valid.
+    def check_layers
+      if layers.size < 3
+        raise "The number of layers size must be more than three."
+      end
+    end
+
+    # Error function (a example of cross entropy)
+    def error(test_data)
+      x = Matrix.columns( test_data.map { |data| data[:x] } )
+      y = propagate_forward(x)
+      test_data.each_with_index.inject(0.0) do |mn, (data, n)|
+        mn + data[:d].each_with_index.inject(0.0) do |mk, (d, k)|
+          mk - d * Math.log(y[k, n])
+        end
+      end
+    end
+
+    # A learning step consists of 
+    # a forward propagation, a backward propagation
+    # and updating the weights of this multi-layer perceptron.
+    def learn
+      @mini_batches.each do |mini_batch|
+        @X = Matrix.columns( mini_batch.map { |data| data[:x] } ) # (Dim, N)
+        @D = Matrix.columns( mini_batch.map { |data| data[:d] } ) # (Dim, N)
+
+        propagate_forward(@X)
+        propagate_backward
+        update
+      end
+    end
+
+    # Input values are propagated forward.
+    def propagate_forward(x_mat)
+      @X = x_mat
+      @N = @X.column_size
+      @Z, @U = [], []
+
+      # l = 0
+      @Z[0] = @X
+
+      # l = 1..L
+      ones = Matrix[Array.new(@N) { 1.0 }]
+      (1..@L).each do |l|
+        @U[l] = @layers[l].W*@Z[l-1] + @layers[l].b*ones
+        @Z[l] = Matrix.columns( @U[l].t.to_a.map { |u| @layers[l].activate(u) } )
+      end
+
+      # Oputput (K, N)
+      @Y = @Z[@L]
+    end
+
+    # Training errors are propagated backwards.
+    def propagate_backward
+      @Delta = []
+
+      # l = L
+      @Delta[@L] = @Y - @D
+
+      # l = (L-1)..1
+      [*(1...@L)].reverse.each do |l|
+        f_u = Matrix.columns( @U[l].t.to_a.map { |u| @layers[l].activate_(u) } )
+        w_d = @layers[l+1].W.t * @Delta[l+1]
+        @Delta[l] = @layers[l].J.times.map do |j|
+          @N.times.map do |n|
+            f_u[j, n] * w_d[j, n]
+          end
+        end.tap { |ary| break Matrix[*ary] }
+      end
+    end
+
+    # Update the weights of this multi-layer perceptron.
+    def update
+      ones = Matrix.columns( [Array.new(@N) { 1.0 }] )
+      (1..@L).each do |l|
+        delta_W = ( MU*@delta_W_prev[l] ) -
+          @learning_rate*( (@Delta[l] * @Z[l-1].t / @N) + LAMBDA*@layers[l].W )
+        delta_b = ( MU*@delta_b_prev[l] ) -
+          @learning_rate*( @Delta[l] * ones )
+
+        @delta_W_prev[l] = Marshal.load(Marshal.dump(delta_W))
+        @delta_b_prev[l] = Marshal.load(Marshal.dump(delta_b))
+
+        @layers[l].W = @layers[l].W + delta_W
+        @layers[l].b = @layers[l].b + delta_b
+      end
+    end
+  end
+end

data/lib/sabina/sparse_auto_encoder.rb

@@ -0,0 +1,49 @@
+module Sabina
+  class SparseAutoEncoder < AutoEncoder
+    BETA = 0.1
+
+    def initialize(options = {})
+      super
+
+      @RHO_const = Matrix[Array.new(@layers[1].J) { 0.05 }]
+      @RHO_prev = Matrix[Array.new(@layers[1].J) { 1.0 }]
+    end
+
+    # Errors are propagated backwards.
+    def propagate_backward
+      calc_rho
+      @Delta = []
+
+      # l = L
+      @Delta[@L] = @Y - @D
+
+      # l = 1
+      l = 1
+      f_u = Matrix.columns( @U[l].t.to_a.map { |u| @layers[l].activate_(u) } )
+      w_d = @layers[l+1].W.t * @Delta[l+1]
+      sps = @layers[l].J.times.map do |j|
+        ((1.0 - @RHO_const[0, j]) / (1.0 - @RHO[0, j])) -
+        (@RHO_const[0, j] / @RHO[0, j])
+      end.tap do |ary|
+        ary = ary.map { |v| v > 1e10 ? 1e10 : v }
+        break Matrix.columns(Array.new(@N) { ary })
+      end
+      w_d_s = w_d + BETA*sps
+
+      @Delta[l] = @layers[l].J.times.map do |j|
+        @N.times.map do |n|
+          f_u[j, n] * w_d_s[j, n]
+        end
+      end.tap { |ary| break Matrix[*ary] }
+    end
+
+    # Calculate average activities
+    def calc_rho
+      @RHO = @Z[1].to_a.map do |z_ary|
+        z_ary.inject(0.0, :+)
+      end.tap { |ary| break Matrix[ary] / @N }
+
+      @RHO = 0.9*@RHO_prev + 0.1*@RHO
+    end
+  end
+end

data/lib/sabina/util.rb

@@ -0,0 +1,11 @@
+module Sabina
+  module Util
+    def box_muller(s = 1.0)
+      r_1 = rand
+      r_2 = rand
+      s * Math.sqrt(-2*Math.log(r_1)) * Math.cos(2*Math::PI*r_2)
+    end
+
+    module_function :box_muller
+  end
+end

data/lib/sabina/version.rb

@@ -0,0 +1,3 @@
+module Sabina
+  VERSION = "0.1.0"
+end

data/lib/sabina.rb

@@ -0,0 +1,17 @@
+require 'matrix'
+require 'csv'
+
+require "sabina/version"
+require "sabina/util"
+require "sabina/layer"
+require "sabina/layer/base_layer"
+require "sabina/configuration"
+require "sabina/multilayer_perceptron"
+require "sabina/auto_encoder"
+require "sabina/sparse_auto_encoder"
+
+module Sabina
+  extend Configuration
+end
+
+Sabina.reset

data/sabina.gemspec

@@ -0,0 +1,37 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sabina/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "sabina"
+  spec.version       = Sabina::VERSION
+  spec.authors       = ["seinosuke"]
+  spec.email         = ["seinosuke.3606@gmail.com"]
+
+  spec.summary       = %q{This gem is a machine learning library.}
+  spec.homepage      = "https://github.com/seinosuke/sabina"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
+  # delete this section to allow pushing this gem to any host.
+  # if spec.respond_to?(:metadata)
+  #   spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
+  # else
+  #   raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  # end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+
+  spec.description   = <<-END
+Sabina is a machine learning library.
+This gem provides tools for Multi-Layer Perceptrons and Auto-Encoders.
+END
+end

metadata

@@ -0,0 +1,124 @@
+--- !ruby/object:Gem::Specification
+name: sabina
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- seinosuke
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-04-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: |
+  Sabina is a machine learning library.
+  This gem provides tools for Multi-Layer Perceptrons and Auto-Encoders.
+email:
+- seinosuke.3606@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- examples/example_ae_01/ae_learning_process.gif
+- examples/example_ae_01/ae_original_data.png
+- examples/example_ae_01/main.rb
+- examples/example_ae_01/training_data.csv
+- examples/example_mp_01/main.rb
+- examples/example_mp_01/mp_learning_process.gif
+- examples/example_mp_01/mp_training_data.png
+- examples/example_mp_01/training_data.csv
+- examples/example_mp_02/main.rb
+- examples/example_mp_02/mp_learning_process.gif
+- examples/example_mp_02/mp_training_data.png
+- examples/example_mp_02/training_data.csv
+- lib/sabina.rb
+- lib/sabina/auto_encoder.rb
+- lib/sabina/configuration.rb
+- lib/sabina/layer.rb
+- lib/sabina/layer/ae_hidden_layer.rb
+- lib/sabina/layer/ae_input_layer.rb
+- lib/sabina/layer/ae_output_layer.rb
+- lib/sabina/layer/base_layer.rb
+- lib/sabina/layer/mp_hidden_layer.rb
+- lib/sabina/layer/mp_input_layer.rb
+- lib/sabina/layer/mp_output_layer.rb
+- lib/sabina/multilayer_perceptron.rb
+- lib/sabina/sparse_auto_encoder.rb
+- lib/sabina/util.rb
+- lib/sabina/version.rb
+- sabina.gemspec
+homepage: https://github.com/seinosuke/sabina
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: This gem is a machine learning library.
+test_files: []