fselector 0.1.2 → 0.2.0

data/lib/fselector/algo_base/base_ReliefF.rb

@@ -0,0 +1,157 @@
+#
+# FSelector: a Ruby gem for feature selection and ranking
+#
+module FSelector
+#
+# base class for extended Relief algorithm (ReliefF), see specialized versions for
+# discrete feature (ReliefF_d) and continuous feature (ReliefF_c), respectively
+#
+# @note applicable to multi-class problem with missing data
+#
+# ref: [Estimating Attributes: Analysis and Extensions of RELIEF](http://www.springerlink.com/content/fp23jh2h0426ww45/)
+#
+  class BaseReliefF < Base
+    #
+    # new()
+    #
+    # @param [Integer] m number of samples to be used 
+    #   for estimating feature contribution. max can be
+    #   the number of training samples
+    # @param [Integer] k number of k-nearest neighbor
+    # @param [Hash] data existing data structure
+    #
+    def initialize(m=nil, k=nil, data=nil)
+      super(data)
+      @m = (m || 30) # default 30
+      @k = (k || 10) # default 10
+    end
+    
+    private
+    
+    # calculate contribution of each feature (f) across all classes
+    def calc_contribution(f)      
+      score = 0.0
+      
+      ## use all samples if @m not provided
+      #@m = get_sample_size if not @m
+      
+      @m.times do
+        # pick a sample at random
+        rs, rk = pick_a_sample_at_random
+        
+        # find k nearest neighbor for each class
+        nbrs = find_k_nearest_nb(rs, rk)
+        
+        # calc contribution from neighbors
+        score += calc_score(f, rs, rk, nbrs)
+      end
+      
+      s = score / @m
+      
+      set_feature_score(f, :BEST, s)
+    end # calc_contribution
+    
+    
+    # pick a sample at random
+    def pick_a_sample_at_random
+      rk = get_classes[rand(get_classes.size)]
+      rks = get_data[rk]
+      
+      [ rks[rand(rks.size)], rk ]
+    end # pick_a_sample_at_random
+    
+    # # find k nearest neighbors of sample (rs) for each class 
+    def find_k_nearest_nb(rs, rk)
+      nbrs = {}
+      
+      each_class do |k|
+        res = []
+        
+        get_data[k].each do |s|
+          next if s.object_id == rs.object_id # exclude self
+          
+          d = diff_sample(rs, s, rk, k)
+          res << [d, s]
+        end
+        
+        nbrs[k] = (res.sort { |x, y| x[0] <=> y[0] }[0...@k]).collect { |z| z[1] }
+      end
+      
+      nbrs
+    end # find_k_nearest_nb
+    
+    
+    # difference between two samples
+    def diff_sample(s1, s2, k1, k2)
+      d = 0.0
+            
+      each_feature do |f|
+        d += diff_feature(f, s1, s2, k1, k2)**2
+      end
+      
+      d
+    end # diff_sample
+    
+    
+    # difference beween the feature (f) of two samples
+    def diff_feature(f, s1, s2, k1, k2)
+      abort "[#{__FILE__}@#{__LINE__}]: "+
+              "derived ReliefF algo must implement its own diff_feature()"
+    end # diff_feature
+    
+    
+    # calc probability of missing value (mv)
+    def calc_p(f, mv, k) 
+      # cache
+      if not @f2mvp
+        @f2mvp = {}
+        
+        each_feature do |f|
+          @f2mvp[f] = {}
+          
+          each_class do |k|
+            @f2mvp[f][k] = {}
+            
+            fvs = get_feature_values(f).uniq
+            fvs.each do |v|
+              n = 0.0
+              
+              get_data[k].each do |s|
+                n += 1 if s.has_key?(f) and s[f] == v
+              end
+              
+              @f2mvp[f][k][v] = n/get_data[k].size
+            end
+          end
+        end
+      end
+      
+      @f2mvp[f][k][mv]
+    end
+    
+    
+    # calc feature (f) contribution from neighbors
+    def calc_score(f, rs, rk, nbrs)
+      score = 0.0
+      
+      nbrs.each do |k, nbs|
+        if k == rk # near hit
+          nbs.each do |s|
+            score -= (diff_feature(f, rs, s, rk, k)**2/nbs.size)
+          end
+        else # near_miss
+          nbs.each do |s|
+            score += (get_data[k].size/get_sample_size.to_f *
+                     diff_feature(f, rs, s, rk, k)**2/nbs.size)
+          end
+        end
+      end
+      
+      score
+    end
+    
+    
+  end # class
+  
+  
+end # module

data/lib/fselector/{base_continuous.rb → algo_base/base_continuous.rb}

@@ -1,5 +1,5 @@
-require File.expand_path(File.dirname(__FILE__) + '/algo_continuous/normalizer.rb')
-require File.expand_path(File.dirname(__FILE__) + '/algo_continuous/discretizer.rb')
+require File.expand_path(File.dirname(__FILE__) + '/../algo_continuous/normalizer.rb')
+require File.expand_path(File.dirname(__FILE__) + '/../algo_continuous/discretizer.rb')
 #
 # FSelector: a Ruby gem for feature selection and ranking
 #

data/lib/fselector/algo_base/base_discrete.rb

@@ -0,0 +1,190 @@
+#
+# FSelector: a Ruby gem for feature selection and ranking
+#
+module FSelector
+#
+#  base ranking alogrithm for handling discrete feature
+#
+#     2 x 2 contingency table
+#     
+#           c   c'
+#         ---------
+#      f  | A | B | A+B
+#         |---|---| 
+#      f' | C | D | C+D
+#         ---------
+#          A+C B+D  N = A+B+C+D
+#     
+#      P(f)     = (A+B)/N
+#      P(f')    = (C+D)/N
+#      P(c)     = (A+C)/N
+#      P(c')    = (B+D)/N
+#      P(f,c)   = A/N
+#      P(f,c')  = B/N
+#      P(f',c)  = C/N
+#      P(f',c') = D/N
+#      P(f|c)   = A/(A+C)
+#      P(f|c')  = B/(B+D)
+#      P(f'|c)  = C/(A+C)
+#      P(f'|c') = D/(B+D)
+#
+  class BaseDiscrete < Base
+    # initialize from an existing data structure
+    def initialize(data=nil)
+      super(data)
+    end
+
+    private
+        
+    # count of sample (i.e. 'A') that
+    # contains feature (f) and belongs to class (k)
+    def get_A(f, k)
+      @A ||= calc_A
+      a = @A[k][f]
+      
+      # add 0.5 to avoid any ZERO in denominator or numerator
+      a+=0.5 if a.zero?
+      
+      a 
+    end # get_A
+
+    
+    # pre-compute 'A'
+    def calc_A
+      results = {}
+      
+      each_class do |k1|
+        results[k1] = {}
+        
+        each_feature do |f|
+          count = 0.0
+          
+          each_sample do |k2, s|
+            if k2 == k1
+              count += 1 if s.has_key? f
+            end
+          end
+          
+          results[k1][f] = count
+        end
+      end
+      
+      results
+    end # calc_A
+    
+  
+    # count of sample (i.e. 'B') that
+    # contains feature (f) but does not belong to class (k)
+    def get_B(f, k)
+      @B ||= calc_B
+      b = @B[k][f]
+          
+      # add 0.5 to avoid any ZERO in denominator or numerator
+      b+=0.5 if b.zero?
+      
+      b
+    end # get_B
+    
+    
+    # pre-compute 'B'
+    def calc_B
+      results = {}
+      
+      each_class do |k1|
+        results[k1] = {}
+        
+        each_feature do |f|
+          count = 0.0
+          
+          each_sample do |k2, s|
+            if k2 != k1
+              count += 1 if s.has_key? f
+            end
+          end
+          
+          results[k1][f] = count
+        end
+      end
+      
+      results
+    end # calc_B
+    
+    
+    # count of sample (i.e. 'C') that
+    # does not contain feature (f) but belongs to class (k)
+    def get_C(f, k)
+      @C ||= calc_C
+      c = @C[k][f]
+      
+      # add 0.5 to avoid any ZERO in denominator or numerator
+      c+=0.5 if c.zero?
+      
+      c
+    end # get_C
+    
+  
+    # pre-compute 'C'
+    def calc_C
+      results = {}
+      
+      each_class do |k1|
+        results[k1] = {}
+        
+        each_feature do |f|
+          count = 0.0
+          
+          each_sample do |k2, s|
+            if k2 == k1
+              count += 1 if not s.has_key? f
+            end
+          end
+          
+          results[k1][f] = count
+        end
+      end
+      
+      results
+    end # calc_C
+    
+    
+    # count of sample (i.e. 'D') that
+    # does not contain feature (f) and does not belong to class (c)
+    def get_D(f, k)
+      @D ||= calc_D
+      d = @D[k][f]
+      
+      # add 0.5 to avoid any ZERO in denominator or numerator
+      d+=0.5 if d.zero?
+      
+      d
+    end # get_D
+
+    
+    # pre-compute 'D'
+    def calc_D
+      results = {}
+      
+      each_class do |k1|
+        results[k1] = {}
+        
+        each_feature do |f|
+          count = 0.0
+          
+          each_sample do |k2, s|
+            if k2 != k1
+              count += 1 if not s.has_key? f
+            end
+          end
+          
+          results[k1][f] = count
+        end
+      end
+      
+      results
+    end # calc_D
+    
+    
+  end # class
+
+
+end # module

data/lib/fselector/algo_continuous/CFS_c.rb

@@ -0,0 +1,47 @@
+#
+# FSelector: a Ruby gem for feature selection and ranking
+#
+module FSelector
+#
+# Correlation-based Feature Selection (CFS) algorithm for continuous feature (CFS_c)
+#
+# ref: [Feature Selection for Discrete and Numeric Class Machine Learning](http://www.cs.waikato.ac.nz/ml/publications/1999/99MH-Feature-Select.pdf)
+#
+  class CFS_c < BaseCFS
+    
+    private
+        
+    # calc the feature-class correlation of two vectors
+    def do_rcf(cv, fv)
+      # weighted pearson's correlation as cv (class label) contains discrete data
+      r = 0.0
+      
+      cv.uniq.each do |k|
+        v = []
+        p = cv.count(k)/cv.size.to_f
+        
+        cv.each do |c|
+          if c == k
+            v << 1
+          else
+            v << 0
+          end
+        end
+        
+        r += p*v.pearson_r(fv)
+      end
+      
+      r
+    end # do_rcf
+    
+    
+    # calc the feature-feature correlation of two vectors
+    def do_rff(fv, sv)
+      fv.pearson_r(sv) # use pearson's correlation coefficient
+    end # do_rff
+    
+    
+  end # class
+  
+  
+end # module

data/lib/fselector/algo_continuous/ReliefF_c.rb

@@ -9,90 +9,12 @@ module FSelector
 #
 # ref: [Estimating Attributes: Analysis and Extensions of RELIEF](http://www.springerlink.com/content/fp23jh2h0426ww45/)
 #
-  class ReliefF_c < BaseContinuous
-    #
-    # new()
-    #
-    # @param [Integer] m number of samples to be used 
-    #   for estimating feature contribution. max can be
-    #   the number of training samples
-    # @param [Integer] k number of k-nearest neighbor
-    # @param [Hash] data existing data structure
-    #
-    def initialize(m=nil, k=10, data=nil)
-      super(data)
-      @m = m # use all samples
-      @k = (k || 10)  # default 10
-    end
-    
-    private
-    
-    # calculate contribution of each feature (f) across all classes
-    def calc_contribution(f)
-      score = 0.0
-      
-      # use all samples if @m not provided
-      @m = get_sample_size if not @m
-      
-      @m.times do
-        # pick a sample at random
-        rs, rk = pick_a_sample_at_random
-        
-        # find k nearest neighbor for each class
-        nbrs = find_k_nearest_nb(rs, rk)
-        
-        # calc contribution from neighbors
-        score += calc_score(f, rs, rk, nbrs)
-      end
-      
-      s = score / @m
-      
-      set_feature_score(f, :BEST, s)
-    end # calc_contribution
-    
-    
-    # pick a sample at random
-    def pick_a_sample_at_random
-      rk = get_classes[rand(get_classes.size)]
-      rks = get_data[rk]
-      
-      [ rks[rand(rks.size)], rk ]
-    end # pick_a_sample_at_random
-    
-    # # find k nearest neighbors of sample (rs) for each class 
-    def find_k_nearest_nb(rs, rk)
-      nbrs = {}
-      
-      each_class do |k|
-        res = []
-        
-        get_data[k].each do |s|
-          next if s == rs # exclude self
-          
-          d = diff_sample(rs, s, rk, k)
-          res << [d, s]
-        end
-        
-        nbrs[k] = (res.sort { |x, y| x[0] <=> y[0] }[0...@k]).collect { |z| z[1] }
-      end
-      
-      nbrs
-    end # find_k_nearest_nb
-    
-    
-    # difference between two samples
-    def diff_sample(s1, s2, k1, k2)
-      d = 0.0
-            
-      each_feature do |f|
-        d += diff_feature(f, s1, s2, k1, k2)**2
-      end
-      
-      d
-    end # diff_sample
+  class ReliefF_c < BaseReliefF
     
+    private    
     
     # difference beween the feature (f) of two samples
+    # specialized version for continuous feature
     def diff_feature(f, s1, s2, k1, k2)
       d = 0.0
       
@@ -117,36 +39,6 @@ module FSelector
     end # diff_feature
     
     
-    # calc probability of missing value (mv)
-    def calc_p(f, mv, k) 
-      # cache
-      if not @f2mvp
-        @f2mvp = {}
-        
-        each_feature do |f|
-          @f2mvp[f] = {}
-          
-          each_class do |k|
-            @f2mvp[f][k] = {}
-            
-            fvs = get_feature_values(f).uniq
-            fvs.each do |v|
-              n = 0.0
-              
-              get_data[k].each do |s|
-                n += 1 if s.has_key?(f) and s[f] == v
-              end
-              
-              @f2mvp[f][k][v] = n/get_data[k].size
-            end
-          end
-        end
-      end
-      
-      @f2mvp[f][k][mv]
-    end
-    
-    
     # get normalization unit for each feature
     def get_normalization_unit(fi)
       return @f2nu[fi] if @f2nu
@@ -162,28 +54,7 @@ module FSelector
     end # get_normalization_unit
     
     
-    # calc feature (f) contribution from neighbors
-    def calc_score(f, rs, rk, nbrs)
-      score = 0.0
-      
-      nbrs.each do |k, nbs|
-        if k == rk # near hit
-          nbs.each do |s|
-            score -= (diff_feature(f, rs, s, rk, k)**2/nbs.size)
-          end
-        else # near_miss
-          nbs.each do |s|
-            score += (get_data[k].size/get_sample_size.to_f *
-                     diff_feature(f, rs, s, rk, k)**2/nbs.size)
-          end
-        end
-      end
-      
-      score
-    end
-    
-    
   end # class
   
   
-end # module
+end # module