ml 0.2.0 → 0.3.0

data/README.md

@@ -5,6 +5,9 @@ Machine learning library for ruby
 # Algorithm Implemented
 
 * Perceptron Learning Algorithm
+* Adaptive Perceptron (Adaline) Learning Algorithm
+* Pocket Learning Algorithm
+* Decision Stump Learning Algorithm
 
 # Tools
 

data/VERSION

@@ -1 +1 @@
-0.2.0
+0.3.0

data/lib/data/generator.rb

@@ -2,68 +2,28 @@ require 'matrix'
 
 module ML
   module Data
-    # Generating sample points on 2D plane
-    class Generator2D
-      # Generate point from line
-      #
-      # @param [Array] coef [a,b,c] for ax+by+c=0
-      # @param [Number] x x value
-      # @return [Array] point
-      def self.point_from_line coef, x
-        [x, (-coef[2]-(coef[0] * x))/coef[1]]
-      end
-
-      # Initialize a generator
-      #
-      # @param [Integer] x_range x range
-      # @param [Integer] y_range y range
-      def initialize x_range = 100, y_range = 100
-        @x_range = x_range
-        @y_range = y_range
-      end
-
-      # Generate two groups of points on 2d plain
-      #
-      # @param [Integer] points the number of points of each set
-      # @param [Array] coef [a,b,c] for ax+by+c=0
-      # @return [Hash] key: points, value: supervised value
-      def points_2d points, coef = [-1.0, 1.0, 0.0]
-        result = {}
-        # for each group
-        [1, -1].each do |grp|
-          points.times do
-            while true
-              point = generate_point
-              prod = Matrix.column_vector(point).transpose * Matrix.column_vector(coef)
-              if (prod[0,0] <=> 0) == grp
-                result[point] = grp
-                break
-              end
-            end
-          end
-        end
-        result
-      end
-
-    private
-      def generate_point
-        [@x_range * rand, @y_range * rand, 1.0]
-      end
-    end
-
     # General generator for n-dimentional space
     class Generator
       # Initial generator
       #
       # @param [Integer] dim dimension
-      def initialize dim
+      # @param [Numeric] scale the magnitude of the vector
+      # @param [Numeric] noise the percentage of noise
+      # @param [Symbol] model the noise model, #:random# for flipping 
+      #   all the element in a probability, while #:flip# only flips a
+      #   portion of elements randomly
+      def initialize dim, scale = 1, noise = 0, model = :random
         @dim = dim
+        @scale = scale
+        @noise = noise
+        @model = model
       end
 
       # Generate two groups of points
       #
       # @param [Integer] points the number of points of each set
-      # @param [Array] coef array of the size of dimension to specify the hyper plane
+      # @param [Array] coef array of the size of dimension to specify the
+      #   hyper plane
       # @return [Hash] key: points, value: supervised value
       def points points, coef
         result = {}
@@ -71,27 +31,79 @@ module ML
         [1, -1].each do |grp|
           points.times do
             while true
-              point = Generator.generate_vector(@dim, 100)
+              point = generate_vector
               prod = Matrix.column_vector(point).transpose * Matrix.column_vector(coef)
               if (prod[0,0] <=> 0) == grp
                 result[point] = grp
+                result[point] *= -1 if @model == :random and rand < @noise
                 break
               end
             end
           end
         end
+
+        if @model == :flip and @noise > 0
+          flipping = (points * @noise * 2).to_i
+          order = (0...(points * 2)).to_a.shuffle
+          for i in 0...flipping
+            result[result.keys[order[i]]] *= -1
+          end
+        end
+
         result
       end
 
       # Generating a random vector
       #
       # @param [Integer] dim the dimension of the vector
-      # @param [Integer] scale the scale of each component
+      # @param [Integer] scale the scale of each component (default [-1,1])
       # @return [Array] random vector
       def self.generate_vector dim, scale = 1
-        result = Array.new(dim) { (rand - 0.5) * scale } 
+        result = Array.new(dim) { (rand - 0.5) * 2 * scale } 
         result << 1.0
       end
+
+    protected
+      def generate_vector
+        Generator.generate_vector @dim, @scale
+      end
+    end
+
+    # Generating sample points on 2D plane
+    class Generator2D < Generator
+      # Generate point from line
+      #
+      # @param [Array] coef [a,b,c] for ax+by+c=0
+      # @param [Number] x x value
+      # @return [Array] point
+      def self.point_from_line coef, x
+        [x, (-coef[2]-(coef[0] * x))/coef[1]]
+      end
+
+      # Initialize a generator
+      #
+      # @param [Integer] x_range x range
+      # @param [Integer] y_range y range
+      # @param [Numeric] noise the percentage of noise
+      def initialize x_range = 100, y_range = 100, noise = 0
+        @x_range = x_range
+        @y_range = y_range
+        @noise = noise
+      end
+
+      # Generate two groups of points on 2d plain
+      #
+      # @param [Integer] points the number of points of each set
+      # @param [Array] coef [a,b,c] for ax+by+c=0
+      # @return [Hash] key: points, value: supervised value
+      def points_2d points, coef = [-1.0, 1.0, 0.0]
+        points(points, coef)
+      end
+
+    protected
+      def generate_vector
+        [@x_range * rand, @y_range * rand, 1.0]
+      end
     end
   end
 end

data/lib/method/decision_stump.rb

@@ -0,0 +1,108 @@
+module ML
+  module Learner
+    # Implementation of decision stump learning
+    class DecisionStumpLearner
+      # Initialize a decision stump learner
+      #
+      # @param [Integer] dim dimension
+      def initialize dim
+        @dim = dim
+        @min_error = 1.0/0
+        @error_vector = []
+      end
+
+      # Train with a supervised data
+      #
+      # @param [Hash] data supervised input data (mapping from array to integer)
+      # @return [Hash] {error} error of the training data
+      def train! data
+        for i in 0...@dim
+          hypo, error = search data, i
+          update_hypo hypo, error
+          @error_vector[i] = error
+        end
+
+        {:error => @min_error}
+      end
+
+      # Predict certain data
+      #
+      # @param [Array] data data in question
+      # @return [Integer] prediction
+      def predict data
+        classify data, @best_hypo
+      end
+
+      # Error vector of each dimension
+      #
+      # @return [Array] the error vector
+      def error_vector
+        @error_vector
+      end
+
+      # Get the hypothesis vector
+      #
+      # Format of hypothesis vector
+      # h_{s,i,t}(x) = s sign((x)_i - t)
+      # 
+      # @return [Array] [s, i, t] vector
+      def hypothesis
+        @best_hypo
+      end
+
+    private
+      def classify data, hypo
+        val = data[hypo[1]] - hypo[2]
+        sign = (val > 0) ? 1 : -1
+        hypo[0] * sign
+      end
+
+      def update_hypo hypo, error
+        if error < @min_error
+          @best_hypo = hypo
+          @min_error = error
+        end
+      end
+
+      def search data, dim
+        pool = data.to_a.sort_by {|line| line[0][dim]}
+        max_diff, index = 0, nil
+        pcount, ncount = 0, 0
+
+        pool.each_with_index do |dat, i|
+          if dat[1] == 1
+            pcount += 1
+          else
+            ncount += 1
+          end
+
+          if (pcount - ncount).abs > max_diff.abs
+            max_diff = pcount - ncount
+            index = i
+          end
+        end
+
+        thres = if index == pool.size - 1
+                  pool[-1][0][dim] + 0.01
+                else
+                  (pool[index][0][dim] + pool[index+1][0][dim]) / 2.0
+                end
+        hypo = if max_diff > 0
+                 [-1, dim, thres]
+               else
+                 [1, dim, thres]
+               end
+
+        [hypo, classify_error(pool, hypo)]
+      end
+
+      def classify_error data, hypo
+        error = 0
+        for dat, result in data
+          error += 1 unless classify(dat, hypo) == result
+        end
+        error
+      end
+    end
+  end
+end

data/lib/method/perceptron.rb

@@ -7,7 +7,7 @@ module ML
       # Initialize a perceptron learner
       #
       # @param [Integer] dim the number of dimension
-      def initialize dim, thres = 1.0/0
+      def initialize dim
         @dim = dim
         @w = Matrix.column_vector(Array.new(dim + 1, 0))
       end
@@ -16,16 +16,15 @@ module ML
       #
       # @param [Hash] data supervised input data (mapping from array to integer)
       # @param [Numeric] threshold the upper bound of the traning iteration
-      # @return [Array] error_and_update [error, update] error in traning and update numbers used
+      # @return [Hash] {error, update_count} error in traning and update numbers used
       def train! data, threshold = 1.0/0
         pool = data.to_a
         update = 0
-        error = 0
 
         while true
           break if update >= threshold
           misclassified = false
-          order = (1...(pool.size)).to_a.shuffle
+          order = (0...(pool.size)).to_a.shuffle
 
           for i in order
             dat, result = pool[i]
@@ -44,14 +43,13 @@ module ML
         end
 
         # check out errors
-        if update >= threshold
-          for dat, result in pool
-            classified_result = (classify(Matrix.column_vector(dat)) <=> 0)
-            error += 1 unless result == classified_result
-          end
-        end
+        error = if update >= threshold
+                  classify_error pool
+                else
+                  0
+                end
 
-        [error, update]
+        {:error => error, :update_count => update}
       end
 
       # The final coefficient of the line
@@ -81,6 +79,17 @@ module ML
       def update_vector x, y
         @w = @w + y * x
       end
+
+      def classify_error supervised_data
+        error = 0
+
+        for data, result in supervised_data
+          classified_result = (classify(Matrix.column_vector(data)) <=> 0)
+          error += 1 unless result == classified_result
+        end
+
+        error
+      end
     end
   end
 end

data/lib/method/pocket.rb

@@ -0,0 +1,38 @@
+module ML
+  module Learner
+    # Implementation of pocket learning algorithm
+    class PocketLearner < PerceptronLearner
+      # Train with supervised data
+      #
+      # @param [Hash] data supervised input data (mapping from array to integer)
+      # @param [Integer] iteration the number of the iterations
+      def train! data, iteration
+        pool = data.to_a
+        best_error, pocket = 1.0/0, nil
+
+        iteration.times do
+          # update pocket
+          error = classify_error pool
+          if error < best_error
+            error = best_error
+            pocket = @w.dup
+          end
+          break if best_error == 0
+
+          # the random order
+          order = (1...(pool.size)).to_a.shuffle
+
+          for i in order
+            dat, result = pool[i]
+            aug_data = Matrix.column_vector(dat)
+
+            if wrongly_classify aug_data, result
+              update_vector aug_data, result
+              break
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ml.rb

@@ -7,6 +7,8 @@ require 'data/parser'
 
 require 'method/perceptron'
 require 'method/adaptive_perceptron'
+require 'method/pocket'
+require 'method/decision_stump'
 
 # Top namespace for machine learning algorithms
 module ML
@@ -19,4 +21,4 @@ module ML
   end
 end
 
-MachingLearning = ML
+MachineLearning = ML

data/ml.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "ml"
-  s.version = "0.2.0"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Andrew Liu"]
-  s.date = "2011-10-17"
+  s.date = "2011-11-17"
   s.description = "Machine learning library in Ruby"
   s.email = "andrewliu33@gmail.com"
   s.extra_rdoc_files = [
@@ -27,7 +27,9 @@ Gem::Specification.new do |s|
     "lib/data/parser.rb",
     "lib/data/plotter.rb",
     "lib/method/adaptive_perceptron.rb",
+    "lib/method/decision_stump.rb",
     "lib/method/perceptron.rb",
+    "lib/method/pocket.rb",
     "lib/ml.rb",
     "ml.gemspec",
     "spec/data_spec.rb",

data/spec/learner_spec.rb

@@ -8,7 +8,9 @@ describe "Learner" do
       generator = ML::Data::Generator2D.new
       data = generator.points_2d(10)
 
-      error, update_count = learner.train! data
+      response = learner.train! data
+      error = response[:error]
+      update_count = response[:update_count]
 
       line = learner.line
       line.should.kind_of?(Array).should == true
@@ -23,7 +25,9 @@ describe "Learner" do
       generator = ML::Data::Generator.new(4)
       data = generator.points(10, ML::Data::Generator.generate_vector(4))
 
-      error, update_count = learner.train! data
+      response = learner.train! data
+      error = response[:error]
+      update_count = response[:update_count]
 
       line = learner.line
       line.should.kind_of?(Array).should == true
@@ -37,10 +41,12 @@ describe "Learner" do
     it "should run adaptive perceptron learning in hyperspace" do
       learner = ML::Learner::AdaptivePerceptronLearner.new(4, 0.1)
 
-      generator = ML::Data::Generator.new(4)
+      generator = ML::Data::Generator.new(4, 100)
       data = generator.points(10, ML::Data::Generator.generate_vector(4))
 
-      error, update_count = learner.train! data, 1000
+      response = learner.train! data, 1000
+      error = response[:error]
+      update_count = response[:update_count]
 
       line = learner.line
       line.should.kind_of?(Array).should == true
@@ -50,4 +56,56 @@ describe "Learner" do
       update_count.should < 1000
     end
   end
+
+  describe "Pocket Learner" do
+    it "should run pocket perceptron learning in hyperspace" do
+      learner = ML::Learner::PocketLearner.new(4)
+
+      generator = ML::Data::Generator.new(4)
+      data = generator.points(10, ML::Data::Generator.generate_vector(4))
+
+      learner.train! data, 1000
+
+      line = learner.line
+      line.should.kind_of?(Array).should == true
+      line.size.should == 5
+    end
+
+    it "should run pocket perceptron learning in noisy data" do
+      learner = ML::Learner::PocketLearner.new(4)
+
+      generator = ML::Data::Generator.new(4, 1, 0.1)
+      data = generator.points(10, ML::Data::Generator.generate_vector(4))
+
+      learner.train! data, 1000
+
+      line = learner.line
+      line.should.kind_of?(Array).should == true
+      line.size.should == 5
+    end
+  end
+
+  describe "Decision Stump Learner" do
+    it "should run decision stump learning in hyperspace" do
+      learner = ML::Learner::DecisionStumpLearner.new(4)
+
+      generator = ML::Data::Generator.new(4)
+      data = generator.points(10, ML::Data::Generator.generate_vector(4))
+
+      learner.train! data
+      vector = learner.error_vector
+      vector.size.should == 4
+    end
+
+    it "should run decision stump learning in noisy data" do
+      learner = ML::Learner::DecisionStumpLearner.new(4)
+
+      generator = ML::Data::Generator.new(4, 1, 0.1)
+      data = generator.points(10, ML::Data::Generator.generate_vector(4))
+
+      learner.train! data
+      vector = learner.error_vector
+      vector.size.should == 4
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ml
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-10-17 00:00:00.000000000Z
+date: 2011-11-17 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyvis
-  requirement: &2157025440 !ruby/object:Gem::Requirement
+  requirement: &2159119320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2157025440
+  version_requirements: *2159119320
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &2157024740 !ruby/object:Gem::Requirement
+  requirement: &2159118600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2157024740
+  version_requirements: *2159118600
 - !ruby/object:Gem::Dependency
   name: bacon
-  requirement: &2157023900 !ruby/object:Gem::Requirement
+  requirement: &2159117940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2157023900
+  version_requirements: *2159117940
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &2157022940 !ruby/object:Gem::Requirement
+  requirement: &2159117460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 0.6.0
   type: :development
   prerelease: false
-  version_requirements: *2157022940
+  version_requirements: *2159117460
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &2157022300 !ruby/object:Gem::Requirement
+  requirement: &2159116740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *2157022300
+  version_requirements: *2159116740
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &2157021360 !ruby/object:Gem::Requirement
+  requirement: &2159116240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +76,10 @@ dependencies:
         version: 1.6.4
   type: :development
   prerelease: false
-  version_requirements: *2157021360
+  version_requirements: *2159116240
 - !ruby/object:Gem::Dependency
   name: rcov
-  requirement: &2157020280 !ruby/object:Gem::Requirement
+  requirement: &2159109840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,7 +87,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2157020280
+  version_requirements: *2159109840
 description: Machine learning library in Ruby
 email: andrewliu33@gmail.com
 executables: []
@@ -106,7 +106,9 @@ files:
 - lib/data/parser.rb
 - lib/data/plotter.rb
 - lib/method/adaptive_perceptron.rb
+- lib/method/decision_stump.rb
 - lib/method/perceptron.rb
+- lib/method/pocket.rb
 - lib/ml.rb
 - ml.gemspec
 - spec/data_spec.rb
@@ -127,7 +129,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -383772979455407848
+      hash: 2827153144592279610
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: