ml 0.3.0 → 0.4.0

data/README.md

@@ -8,6 +8,9 @@ Machine learning library for ruby
 * Adaptive Perceptron (Adaline) Learning Algorithm
 * Pocket Learning Algorithm
 * Decision Stump Learning Algorithm
+* Linear Regression Algorithm
+* Logistic Regression Algorithm
+* Cyclic Coordinate Descent 
 
 # Tools
 

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.4.0

data/lib/data/generator.rb

@@ -9,8 +9,8 @@ module ML
       # @param [Integer] dim dimension
       # @param [Numeric] scale the magnitude of the vector
       # @param [Numeric] noise the percentage of noise
-      # @param [Symbol] model the noise model, #:random# for flipping 
-      #   all the element in a probability, while #:flip# only flips a
+      # @param [Symbol] model the noise model, :random for flipping 
+      #   all the element in a probability, while :flip only flips a
       #   portion of elements randomly
       def initialize dim, scale = 1, noise = 0, model = :random
         @dim = dim
@@ -85,10 +85,14 @@ module ML
       # @param [Integer] x_range x range
       # @param [Integer] y_range y range
       # @param [Numeric] noise the percentage of noise
-      def initialize x_range = 100, y_range = 100, noise = 0
+      # @param [Symbol] model the noise model, :random for flipping 
+      #   all the element in a probability, while :flip only flips a
+      #   portion of elements randomly
+      def initialize x_range = 100, y_range = 100, noise = 0, model = :random
         @x_range = x_range
         @y_range = y_range
         @noise = noise
+        @model = model
       end
 
       # Generate two groups of points on 2d plain

data/lib/method/adaptive_perceptron.rb

@@ -8,18 +8,18 @@ module ML
       #
       # @param [Integer] dim the number of dimension
       # @param [Float] the eta parameter
-      def initialize dim, eta
+      def initialize dim, eta = 0.1
         super(dim)
         @eta = eta
       end
 
     protected
       def wrongly_classify x, y
-        classify(x) * y <= 1
+        classify_inner(x) * y <= 1
       end
 
       def update_vector x, y
-        @w = @w + @eta * (y - classify(x))* x
+        self.current_vector += @eta * (y - classify_inner(x))* x
       end
     end
   end

data/lib/method/cyclic_descent.rb

@@ -0,0 +1,67 @@
+require 'matrix'
+
+module ML
+  module Learner
+    # Implementation of cyclic coordinate descent learner
+    class CyclicDescentLearner
+      include Toolbox
+      include LinearToolbox
+
+      # Initialize a learner
+      #
+      # @param [Integer] dim dimension
+      def initialize dim, model = :basis
+        @dim = dim
+        @model = model
+      end
+
+      # Train with a supervised data
+      #
+      # @param [Hash] data supervised input data (mapping from array to integer)
+      # @param [Integer] iteration the desired iteration number
+      def train! data, iteration = 1000
+        self.current_vector = Matrix.column_vector(Array.new(@dim + 1, 0))
+        iteration.times do |i|
+          v = calc_v i
+          eta = calc_eta data, v
+          self.current_vector += eta * v
+        end
+      end
+
+    private
+
+      def calc_eta data, v
+        v_t = v.transpose
+        w_t = self.current_vector.transpose
+        train = {}
+
+        for xn, yn in data
+          x_n = Matrix.column_vector(xn)
+          dot = (v_t * x_n)[0,0] * yn
+          thr = (w_t * x_n)[0,0] * (-yn) / dot
+          
+          next if dot == 0
+          if dot > 0
+            train[[thr]] = 1
+          else
+            train[[thr]] = -1
+          end
+        end
+
+        learner = DecisionStumpLearner.new(1)
+        learner.train! train
+        learner.hypothesis[2]
+      end
+
+      def calc_v iteration
+        v = Array.new(@dim + 1, 0)
+        if @model == :basis
+          v[iteration % @dim] = 1
+        else
+          v[iteration % @dim] = Util.normal_distribution 0,1
+        end
+        Matrix.column_vector(v)
+      end
+    end
+  end
+end

data/lib/method/decision_stump.rb

@@ -2,6 +2,7 @@ module ML
   module Learner
     # Implementation of decision stump learning
     class DecisionStumpLearner
+      include Toolbox
       # Initialize a decision stump learner
       #
       # @param [Integer] dim dimension
@@ -14,15 +15,12 @@ module ML
       # Train with a supervised data
       #
       # @param [Hash] data supervised input data (mapping from array to integer)
-      # @return [Hash] {error} error of the training data
       def train! data
         for i in 0...@dim
           hypo, error = search data, i
           update_hypo hypo, error
           @error_vector[i] = error
         end
-
-        {:error => @min_error}
       end
 
       # Predict certain data
@@ -67,8 +65,9 @@ module ML
       def search data, dim
         pool = data.to_a.sort_by {|line| line[0][dim]}
         max_diff, index = 0, nil
-        pcount, ncount = 0, 0
 
+        # in order search
+        pcount, ncount = 0, 0
         pool.each_with_index do |dat, i|
           if dat[1] == 1
             pcount += 1
@@ -82,10 +81,33 @@ module ML
           end
         end
 
-        thres = if index == pool.size - 1
-                  pool[-1][0][dim] + 0.01
+        # reverse search
+        pcount, ncount = 0, 0
+        pool.reverse.each_with_index do |dat, i|
+          if dat[1] == 1
+            pcount += 1
+          else
+            ncount += 1
+          end
+
+          if (ncount - pcount).abs > max_diff.abs
+            max_diff = ncount - pcount
+            index = pool.size - i - 1
+          end
+        end
+
+        thres = if max_diff > 0
+                  if index == pool.size - 1
+                    pool[-1][0][dim] + 0.01
+                  else
+                    (pool[index][0][dim] + pool[index+1][0][dim]) / 2.0
+                  end
                 else
-                  (pool[index][0][dim] + pool[index+1][0][dim]) / 2.0
+                  if index == 0
+                    pool[0][0][dim] - 0.01
+                  else
+                    (pool[index][0][dim] + pool[index-1][0][dim]) / 2.0
+                  end
                 end
         hypo = if max_diff > 0
                  [-1, dim, thres]
@@ -93,10 +115,10 @@ module ML
                  [1, dim, thres]
                end
 
-        [hypo, classify_error(pool, hypo)]
+        [hypo, hypo_error(pool, hypo)]
       end
 
-      def classify_error data, hypo
+      def hypo_error data, hypo
         error = 0
         for dat, result in data
           error += 1 unless classify(dat, hypo) == result

data/lib/method/linear_regression.rb

@@ -0,0 +1,34 @@
+require 'matrix'
+
+module ML
+  module Learner
+    # Implementation of linear regression
+    class LinearRegressionLearner
+      include Toolbox
+      include LinearToolbox
+
+      # Intialize linear regression
+      #
+      # @param [Integer] dim the input dimension
+      def initialize dim
+        @dim = dim
+      end
+
+      # Train with supervised data
+      #
+      # @param [Hash] data supervised input data (mapping from array to integer)
+      def train! data
+        x = Matrix.rows(data.keys)
+        ary_y = []
+        for k in data.keys
+          ary_y << data[k]
+        end
+        y = Matrix.column_vector(ary_y)
+
+        x_t = x.transpose
+        x_dag = (x_t * x).inverse * x_t
+        self.current_vector = x_dag * y
+      end
+    end
+  end
+end

data/lib/method/logistic_regression.rb

@@ -0,0 +1,50 @@
+module ML
+  module Learner
+    # Implementing logistic regression
+    class LogisticRegressionLearner
+      include Toolbox
+      include LinearToolbox
+
+      # Intialize logistic regression
+      #
+      # @param [Integer] dim the input dimension
+      # @param [Numeric] eta the eta parameter
+      # @param [Symbol] model the learning model, :variate for variating
+      #   learning rate and :fixed for fixed learning rate
+      def initialize dim, eta = 0.01, model = :variate
+        @dim = dim
+        @eta = eta
+        @model = model
+      end
+
+      # Train with supervised data
+      #
+      # @param [Hash] data supervised input data (mapping from array to integer)
+      # @param [Integer] iteration the number of the iterations
+      def train! data, iteration = 1000
+        self.current_vector = Matrix.column_vector(Array.new(@dim + 1, 0))
+
+        iteration.times do
+          if @model == :variate
+            n = (rand * data.size).to_i
+            key = data.keys[n]
+            self.current_vector -= gradiant(key, data[key]).map {|e| e * @eta }
+          else
+            sum = Matrix.column_vector(Array.new(@dim + 1, 0))
+            for key, value in data
+              sum += gradiant key, value
+            end
+            self.current_vector -= sum.map {|e| e * @eta / data.size }
+          end
+        end
+      end
+
+    protected
+      def gradiant x, y
+        exp = Math.exp(-y * 2 * (self.current_vector.transpose * Matrix.column_vector(x))[0,0])
+        coef = exp * (-2 * y) / (1 + exp)
+        Matrix.column_vector(x).map {|e| e * coef}
+      end
+    end
+  end
+end

data/lib/method/perceptron.rb

@@ -4,12 +4,15 @@ module ML
   module Learner
     # Implementation of Perceptron Learning Algorithm
     class PerceptronLearner
+      include Toolbox
+      include LinearToolbox
+
       # Initialize a perceptron learner
       #
       # @param [Integer] dim the number of dimension
       def initialize dim
         @dim = dim
-        @w = Matrix.column_vector(Array.new(dim + 1, 0))
+        self.current_vector = Matrix.column_vector(Array.new(dim + 1, 0))
       end
 
       # Train with supervised data
@@ -41,54 +44,15 @@ module ML
 
           break unless misclassified
         end
-
-        # check out errors
-        error = if update >= threshold
-                  classify_error pool
-                else
-                  0
-                end
-
-        {:error => error, :update_count => update}
-      end
-
-      # The final coefficient of the line
-      #
-      # @return [Array] [a,b,c] for ax+by+c=0
-      def line
-        @w.column(0).to_a
-      end
-
-      # Predict certain data
-      #
-      # @param [Array] data data in question
-      # @return [Integer] prediction
-      def predict data
-        classify(Matrix.column_vector(data + [1.0])) <=> 0
       end
 
     protected
-      def classify data
-        (@w.transpose * data)[0,0]
-      end
-
       def wrongly_classify x, y
-        classify(x) * y <= 0
+        classify_inner(x) * y <= 0
       end
 
       def update_vector x, y
-        @w = @w + y * x
-      end
-
-      def classify_error supervised_data
-        error = 0
-
-        for data, result in supervised_data
-          classified_result = (classify(Matrix.column_vector(data)) <=> 0)
-          error += 1 unless result == classified_result
-        end
-
-        error
+        self.current_vector += y * x
       end
     end
   end

data/lib/method/pocket.rb

@@ -6,7 +6,7 @@ module ML
       #
       # @param [Hash] data supervised input data (mapping from array to integer)
       # @param [Integer] iteration the number of the iterations
-      def train! data, iteration
+      def train! data, iteration = 1000
         pool = data.to_a
         best_error, pocket = 1.0/0, nil
 
@@ -15,7 +15,7 @@ module ML
           error = classify_error pool
           if error < best_error
             error = best_error
-            pocket = @w.dup
+            pocket = current_vector.dup
           end
           break if best_error == 0
 

data/lib/method/toolbox.rb

@@ -0,0 +1,73 @@
+require 'matrix'
+
+module ML
+  module Learner
+
+    # General toolbox for learning methods
+    module Toolbox
+      # Predict a single data with current prediction
+      #
+      # @param [Array] data input vector array
+      # @return [Integer] classified data
+      def predict data
+        raise "Cannot predict"
+      end
+
+      # Calculating model error 
+      #
+      # @param [Hash] data
+      #   supervised input data (mapping from array to integer)
+      def classify_error supervised_data
+        error = 0
+
+        for data, result in supervised_data
+          classified_result = predict(data)
+          error += 1 unless result == classified_result
+        end
+
+        error.to_f / supervised_data.size
+      end
+    end
+
+    # Learner toolbox for linear model
+    #
+    # The prediction is a vector of dimension d+1 where d is the
+    # dimension of the input data
+    module LinearToolbox
+      # Current prediction vector
+      attr_accessor :current_vector
+
+      # Predict certain data
+      #
+      # @param [Array] data data in question
+      # @return [Integer] prediction
+      def predict data
+        classify_bool Matrix.column_vector(data)
+      end
+
+      # The final coefficient of the line
+      #
+      # @return [Array] [a,b,c] for ax+by+c=0 for 2-d case
+      def line
+        current_vector.column(0).to_a
+      end
+
+    protected
+      # Classify with single data with 0/1 error
+      #
+      # @param [Matrix] data input column vector
+      # @return [Integer] classified data
+      def classify_bool data
+        classify_inner(data) <=> 0
+      end
+
+      # Classify with single data with inner product
+      #
+      # @param [Matrix] data input column vector
+      # @return [Integer] classified data
+      def classify_inner data
+        (current_vector.transpose * data)[0,0]
+      end
+    end
+  end
+end

data/lib/ml.rb

@@ -1,14 +1,21 @@
 require 'rubygems'
 require 'bundler/setup'
 
+require 'util/util'
+
 require 'data/plotter'
 require 'data/generator'
 require 'data/parser'
 
+require 'method/toolbox'
+
 require 'method/perceptron'
 require 'method/adaptive_perceptron'
 require 'method/pocket'
 require 'method/decision_stump'
+require 'method/linear_regression'
+require 'method/logistic_regression'
+require 'method/cyclic_descent'
 
 # Top namespace for machine learning algorithms
 module ML

data/lib/util/util.rb

@@ -0,0 +1,20 @@
+module ML
+  module Util
+    module_function
+
+    def normal_distribution mean, variance
+      if @y
+        y = @y
+        @y = nil
+        return y
+      end
+
+      theta = 2 * Math::PI * rand
+      rho = Math.sqrt(-2 * Math.log(1 - rand))
+      scale = variance * rho
+      x = mean + scale * Math.cos(theta)
+      @y = mean + scale * Math.sin(theta)
+      x
+    end
+  end
+end

data/ml.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "ml"
-  s.version = "0.3.0"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Andrew Liu"]
-  s.date = "2011-11-17"
+  s.date = "2011-12-13"
   s.description = "Machine learning library in Ruby"
   s.email = "andrewliu33@gmail.com"
   s.extra_rdoc_files = [
@@ -27,10 +27,15 @@ Gem::Specification.new do |s|
     "lib/data/parser.rb",
     "lib/data/plotter.rb",
     "lib/method/adaptive_perceptron.rb",
+    "lib/method/cyclic_descent.rb",
     "lib/method/decision_stump.rb",
+    "lib/method/linear_regression.rb",
+    "lib/method/logistic_regression.rb",
     "lib/method/perceptron.rb",
     "lib/method/pocket.rb",
+    "lib/method/toolbox.rb",
     "lib/ml.rb",
+    "lib/util/util.rb",
     "ml.gemspec",
     "spec/data_spec.rb",
     "spec/learner_spec.rb",

data/spec/learner_spec.rb

@@ -1,111 +1,32 @@
 require 'spec_helper'
 
 describe "Learner" do
-  describe "Perceptron Learner" do
-    it "should run perceptron learning in 2d" do
-      learner = ML::Learner::PerceptronLearner.new(2)
+  [ML::Learner::PerceptronLearner, ML::Learner::DecisionStumpLearner,
+   ML::Learner::AdaptivePerceptronLearner, ML::Learner::PocketLearner,
+   ML::Learner::LinearRegressionLearner, ML::Learner::CyclicDescentLearner,
+   ML::Learner::LogisticRegressionLearner].each do |method|
+    describe method.to_s do
+      it "should run #{method.to_s} in 2d" do
+        learner = method.new(2)
 
-      generator = ML::Data::Generator2D.new
-      data = generator.points_2d(10)
+        generator = ML::Data::Generator2D.new
+        data = generator.points_2d(100)
 
-      response = learner.train! data
-      error = response[:error]
-      update_count = response[:update_count]
+        learner.train! data
 
-      line = learner.line
-      line.should.kind_of?(Array).should == true
-      line.size.should == 3
+        learner.classify_error(data).should < 0.5
+      end
 
-      update_count.should > 0
-    end
-
-    it "should run perceptron learning in hyperspace" do
-      learner = ML::Learner::PerceptronLearner.new(4)
-
-      generator = ML::Data::Generator.new(4)
-      data = generator.points(10, ML::Data::Generator.generate_vector(4))
-
-      response = learner.train! data
-      error = response[:error]
-      update_count = response[:update_count]
-
-      line = learner.line
-      line.should.kind_of?(Array).should == true
-      line.size.should == 5
-
-      update_count.should > 0
-    end
-  end
-
-  describe "Adpative Perceptron Learner" do
-    it "should run adaptive perceptron learning in hyperspace" do
-      learner = ML::Learner::AdaptivePerceptronLearner.new(4, 0.1)
-
-      generator = ML::Data::Generator.new(4, 100)
-      data = generator.points(10, ML::Data::Generator.generate_vector(4))
-
-      response = learner.train! data, 1000
-      error = response[:error]
-      update_count = response[:update_count]
-
-      line = learner.line
-      line.should.kind_of?(Array).should == true
-      line.size.should == 5
-
-      update_count.should > 0
-      update_count.should < 1000
-    end
-  end
-
-  describe "Pocket Learner" do
-    it "should run pocket perceptron learning in hyperspace" do
-      learner = ML::Learner::PocketLearner.new(4)
-
-      generator = ML::Data::Generator.new(4)
-      data = generator.points(10, ML::Data::Generator.generate_vector(4))
-
-      learner.train! data, 1000
-
-      line = learner.line
-      line.should.kind_of?(Array).should == true
-      line.size.should == 5
-    end
-
-    it "should run pocket perceptron learning in noisy data" do
-      learner = ML::Learner::PocketLearner.new(4)
-
-      generator = ML::Data::Generator.new(4, 1, 0.1)
-      data = generator.points(10, ML::Data::Generator.generate_vector(4))
-
-      learner.train! data, 1000
-
-      line = learner.line
-      line.should.kind_of?(Array).should == true
-      line.size.should == 5
-    end
-  end
-
-  describe "Decision Stump Learner" do
-    it "should run decision stump learning in hyperspace" do
-      learner = ML::Learner::DecisionStumpLearner.new(4)
-
-      generator = ML::Data::Generator.new(4)
-      data = generator.points(10, ML::Data::Generator.generate_vector(4))
-
-      learner.train! data
-      vector = learner.error_vector
-      vector.size.should == 4
-    end
+      it "should run #{method.to_s} in hyperspace" do
+        learner = method.new(4)
 
-    it "should run decision stump learning in noisy data" do
-      learner = ML::Learner::DecisionStumpLearner.new(4)
+        generator = ML::Data::Generator.new(4)
+        data = generator.points(100, ML::Data::Generator.generate_vector(4))
 
-      generator = ML::Data::Generator.new(4, 1, 0.1)
-      data = generator.points(10, ML::Data::Generator.generate_vector(4))
+        learner.train! data
 
-      learner.train! data
-      vector = learner.error_vector
-      vector.size.should == 4
+        learner.classify_error(data).should < 0.5
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ml
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-11-17 00:00:00.000000000Z
+date: 2011-12-13 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyvis
-  requirement: &2159119320 !ruby/object:Gem::Requirement
+  requirement: &2153078140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2159119320
+  version_requirements: *2153078140
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &2159118600 !ruby/object:Gem::Requirement
+  requirement: &2153076840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2159118600
+  version_requirements: *2153076840
 - !ruby/object:Gem::Dependency
   name: bacon
-  requirement: &2159117940 !ruby/object:Gem::Requirement
+  requirement: &2153075100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2159117940
+  version_requirements: *2153075100
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &2159117460 !ruby/object:Gem::Requirement
+  requirement: &2153060800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 0.6.0
   type: :development
   prerelease: false
-  version_requirements: *2159117460
+  version_requirements: *2153060800
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &2159116740 !ruby/object:Gem::Requirement
+  requirement: &2153059640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *2159116740
+  version_requirements: *2153059640
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &2159116240 !ruby/object:Gem::Requirement
+  requirement: &2153058400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +76,10 @@ dependencies:
         version: 1.6.4
   type: :development
   prerelease: false
-  version_requirements: *2159116240
+  version_requirements: *2153058400
 - !ruby/object:Gem::Dependency
   name: rcov
-  requirement: &2159109840 !ruby/object:Gem::Requirement
+  requirement: &2153056540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,7 +87,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2159109840
+  version_requirements: *2153056540
 description: Machine learning library in Ruby
 email: andrewliu33@gmail.com
 executables: []
@@ -106,10 +106,15 @@ files:
 - lib/data/parser.rb
 - lib/data/plotter.rb
 - lib/method/adaptive_perceptron.rb
+- lib/method/cyclic_descent.rb
 - lib/method/decision_stump.rb
+- lib/method/linear_regression.rb
+- lib/method/logistic_regression.rb
 - lib/method/perceptron.rb
 - lib/method/pocket.rb
+- lib/method/toolbox.rb
 - lib/ml.rb
+- lib/util/util.rb
 - ml.gemspec
 - spec/data_spec.rb
 - spec/learner_spec.rb
@@ -129,7 +134,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2827153144592279610
+      hash: -1343753565689468056
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: