fselector 0.1.0

data/lib/fselector/algo_continuous/Relief_c.rb

@@ -0,0 +1,150 @@
+#
+# FSelector: a Ruby gem for feature selection and ranking
+#
+module FSelector
+#
+# Relief algorithm for continuous feature (Relief_c)
+#
+# @note Relief applicable only to two-class problem without missing data
+#
+# ref: [The Feature Selection Problem: Traditional Methods 
+#       and a New Algorithm][url]
+# [url]: http://www.aaai.org/Papers/AAAI/1992/AAAI92-020.pdf
+#
+  class Relief_c < BaseContinuous
+    #
+    # new()
+    #
+    # @param [Integer] m number of samples to be used 
+    #   for estimating feature contribution. max can be
+    #   the number of training samples
+    # @param [Hash] data existing data structure
+    #
+    def initialize(m=nil, data=nil)
+      super(data)
+      @m = m # default use all samples
+    end
+    
+    private
+    
+    # calculate contribution of each feature (f) across all classes
+    def calc_contribution(f)
+      if not get_classes.size == 2
+        abort "[#{__FILE__}@#{__LINE__}]: "+
+        "Relief applicable only to two-class problems without missing data"
+      end
+      
+      # use all samples if @m not provided
+      @m = get_sample_size if not @m
+      
+      k1, k2 = get_classes
+      score = 0.0
+      
+      @m.times do
+        # pick a sample at random
+        rs, rk = pick_a_sample_at_random
+        
+        # find the nearest neighbor for each class
+        nbrs = find_nearest_nb(rs, rk)
+        
+        # calc contribution from neighbors
+        score += calc_score(f, rs, rk, nbrs)
+      end
+      
+      s = score / @m
+      
+      set_feature_score(f, :BEST, s)
+    end # calc_contribution
+    
+    
+    # pick a sample at random
+    def pick_a_sample_at_random
+      rk = get_classes[rand(get_classes.size)]
+      rks = get_data[rk]
+      
+      [ rks[rand(rks.size)], rk ]
+    end # pick_a_sample_at_random
+    
+    
+    # find nearest neighbor sample for given sample (rs) within class (k)
+    def find_nearest_nb(rs, rk)
+      nbrs = {}
+      
+      each_class do |k|
+        nb, dmin = nil, 999
+        get_data[k].each do |s|
+          next if s == rs # exclude self
+          d = diff_sample(rs, s)
+          if d < dmin
+            dmin = d
+            nb = s
+          end
+        end
+        
+        nbrs[k] = nb
+      end
+      
+      nbrs
+    end # find_nearest_nb
+    
+    
+    # difference between two samples
+    def diff_sample(s1, s2)
+      d = 0.0
+            
+      each_feature do |f|
+        d += diff_feature(f, s1, s2)**2
+      end
+      
+      d
+    end # diff_sample
+    
+    
+    # difference beween the feature (f) of two samples
+    def diff_feature(f, s1, s2)
+      if not s1.has_key?(f) or not s2.has_key?(f)
+        abort "[#{__FILE__}@#{__LINE__}]: "+
+              "Relief does not allow missing values"
+      end
+      
+      nu = get_normalization_unit(f)
+      
+      (nu.zero?) ? 0.0 : (s1[f]-s2[f])/nu
+    end # diff_feature
+    
+    
+    # get normalization unit for each feature
+    def get_normalization_unit(fi)
+      return @f2nu[fi] if @f2nu
+      
+      @f2nu = {}
+      
+      each_feature do |f|
+        fvs = get_feature_values(f)
+        @f2nu[f] = (fvs.max-fvs.min).to_f
+      end
+      
+      @f2nu[fi]
+    end # get_normalization_unit
+    
+    
+    # calc feature (f) contribution from neighbors
+    def calc_score(f, rs, rk, nbrs)
+      score = 0.0
+      
+      nbrs.each do |k, s|
+        if k == rk # near hit
+          score -= diff_feature(f, rs, s)**2
+        else # near_miss
+          score += diff_feature(f, rs, s)**2
+        end
+      end
+      
+      score
+    end # calc_score
+    
+    
+  end # class
+  
+  
+end # module

data/lib/fselector/algo_continuous/TScore.rb

@@ -0,0 +1,52 @@
+#
+# FSelector: a Ruby gem for feature selection and ranking
+#
+module FSelector
+#
+# t-score (TS) based on Student's t-test for continous feature
+# 
+#                            |u1 - u2|
+#     TS(f) = --------------------------------------------
+#              sqrt((n1*sigma1^2 + n_2*sigma2^2)/(n1+n2))
+#
+# @note TS applicable only to two-class problems
+#
+# ref: [Filter versus wrapper gene selection approaches][url]
+# [url]: http://www.sciencedirect.com/science/article/pii/S0933365704000193
+#
+  class TScore < BaseContinuous
+        
+    private    
+    
+    # calculate contribution of each feature (f) across all classes
+    def calc_contribution(f)
+      if not get_classes.size == 2
+        abort "[#{__FILE__}@#{__LINE__}]: "+
+              "suitable only for two-class problem with continuous feature"
+      end
+      
+      # collect data for class 1 and 2, respectively
+      s1, s2 = [], []      
+      k1, k2 = get_classes
+           
+      each_sample do |k, ss|
+        s1 << ss[f] if k == k1 and ss.has_key? f
+        s2 << ss[f] if k == k2 and ss.has_key? f
+      end
+      
+      # calc
+      n1, n2 = s1.size, s2.size
+      s = (s1.ave-s2.ave).abs / Math.sqrt( (n1*s1.var+n2*s2.var) / (n1+n2) )
+      
+      set_feature_score(f, :BEST, s)
+    end # calc_contribution
+    
+    
+  end # class
+  
+  
+  # shortcut so that you can use FSelector::TS instead of FSelector::TScore
+  TS = TScore
+  
+  
+end # module

data/lib/fselector/algo_continuous/discretizer.rb

@@ -0,0 +1,219 @@
+#
+# discretilize continous feature
+#
+module Discretilizer
+  # discretize by equal-width intervals
+  #
+  # @param [Integer] n_interval
+  #        desired number of intervals
+  # @note data structure will be altered
+  def discretize_equal_width!(n_interval)
+    n_interval = 1 if n_interval < 1 # at least one interval
+    
+    # first determine min and max for each feature
+    f2min_max = {}
+    each_feature do |f|
+      fvs = get_feature_values(f)
+      f2min_max[f] = [fvs.min, fvs.max]
+    end
+    
+    # then discretize
+    each_sample do |k, s|
+      s.keys.each do |f|
+        min_v, max_v = f2min_max[f]
+        if min_v == max_v
+          wn = 0
+        else
+          wn = ((s[f]-min_v)*n_interval.to_f / (max_v-min_v)).to_i
+        end
+        
+        s[f] = (wn<n_interval) ? wn : n_interval-1
+      end
+    end
+    
+  end # discretize_equal_width!
+  
+  
+  # discretize by equal-frequency intervals
+  #
+  # @param [Integer] n_interval
+  #        desired number of intervals
+  # @note data structure will be altered
+  def discretize_equal_frequency!(n_interval)
+    n_interval = 1 if n_interval < 1 # at least one interval
+    
+    # first determine the boundaries
+    f2bs = Hash.new { |h,k| h[k] = [] }
+    each_feature do |f|
+      fvs = get_feature_values(f).sort
+      # number of samples in each interval
+      ns = (fvs.size.to_f/n_interval).round
+      fvs.each_with_index do |v, i|
+        if (i+1)%ns == 0 and (i+1)<fvs.size
+          f2bs[f] << (v+fvs[i+1])/2.0
+        end
+      end
+      f2bs[f] << fvs.max+1.0 # add the rightmost boundary
+    end
+    
+    # then discretize
+    each_sample do |k, s|
+      s.keys.each do |f|
+        s[f] = get_index(s[f], f2bs[f])
+      end
+    end
+    
+  end # discretize_equal_frequency!
+  
+  
+  #
+  # discretize by ChiMerge algorithm
+  #
+  # @param [Float] chisq chi-squared value
+  # @note data structure will be altered
+  #
+  # ref: [ChiMerge: Discretization of Numberic Attributes][url]
+  # [url]: http://sci2s.ugr.es/keel/pdf/algorithm/congreso/1992-Kerber-ChimErge-AAAI92.pdf
+  #
+  # chi-squared values and associated p values can be looked up at
+  # [Wikipedia](http://en.wikipedia.org/wiki/Chi-squared_distribution) <br>
+  # degrees of freedom: one less than number of classes
+  #
+  #     chi-squared values vs p values
+  #     degree_of_freedom  p<0.10  p<0.05  p<0.01  p<0.001
+  #             1          2.71    3.84    6.64    10.83
+  #             2          4.60    5.99    9.21    13.82
+  #             3          6.35    7.82    11.34   16.27
+  #    
+  def discretize_chimerge!(chisq)
+    # chisq = 4.60 # for iris::Sepal.Length
+    # for intialization
+    hzero = {}
+    each_class do |k|
+      hzero[k] = 0.0
+    end
+    
+    # determine the final boundaries for each feature
+    f2bs = {}
+    each_feature do |f|
+      #f = "Sepal.Length"
+      # 1a. initialize boundaries
+      bs, cs, qs = [], [], []
+      fvs = get_feature_values(f).sort.uniq
+      fvs.each_with_index do |v, i|
+        if i+1 < fvs.size
+          bs << (v+fvs[i+1])/2.0
+          cs << hzero.dup
+          qs << 0.0
+        end
+      end
+      bs << fvs.max+1.0 # add the rightmost boundary
+      cs << hzero.dup
+      
+      # 1b. initialize counts for each interval
+      each_sample do |k, s|
+        next if not s.has_key? f
+        bs.each_with_index do |b, i|
+          if s[f] < b
+            cs[i][k] += 1.0
+            break
+          end
+        end
+      end
+      
+      # 1c. initialize chi-squared values between two adjacent intervals
+      cs.each_with_index do |c, i|
+        if i+1 < cs.size
+          qs[i] = calc_chisq(c, cs[i+1])
+        end
+      end
+      
+      # 2. iteratively merge intervals
+      until qs.empty? or qs.min > chisq
+        qs.each_with_index do |q, i|
+          if q == qs.min
+            #pp "i: #{i}"
+            #pp bs.join(',')
+            #pp qs.join(',')
+            
+            # update cs for merged two intervals
+            cm = {}
+            each_class do |k|
+              cm[k] = cs[i][k]+cs[i+1][k]
+            end
+            
+            # update qs if necessary
+            # before merged intervals
+            if i-1 >= 0
+              qs[i-1] = calc_chisq(cs[i-1], cm)
+            end
+            # after merged intervals
+            if i+1 < qs.size
+              qs[i+1] = calc_chisq(cm, cs[i+2])
+            end
+            
+            # merge
+            bs = bs[0...i] + bs[i+1...bs.size]
+            cs = cs[0...i] + [cm] + cs[i+2...cs.size]
+            qs = qs[0...i] + qs[i+1...qs.size]
+            
+            #pp bs.join(',')
+            #pp qs.join(',')
+            
+            # break out
+            break
+            
+          end
+        end
+      end
+      
+      # 3. record the final boundaries
+      f2bs[f] = bs
+    end
+
+    # discretize according to each feature's boundaries
+    each_sample do |k, s|
+      s.keys.each do |f|
+        s[f] = get_index(s[f], f2bs[f])
+      end
+    end
+    
+  end # discretize_chimerge!
+  
+  private
+  
+  # get index from sorted boundaries
+  #
+  # min -- | -- | -- | ... max |
+  #        b0   b1   b2        bn(=max+1)
+  #      0    1    2   ...   n
+  #
+  def get_index(v, boundaries)
+    boundaries.each_with_index do |b, i|
+      return i if v < b
+    end
+  end # get_index
+  
+
+  # calc the chi squared value of ChiMerge
+  def calc_chisq(cs1, cs2)
+    r1 = cs1.values.sum
+    r2 = cs2.values.sum
+    n = r1+r2
+    
+    q = 0.0
+    
+    each_class do |k|
+      ck1 = 
+      ek1 = r1*(cs1[k]+cs2[k])/n
+      ek2 = r2*(cs1[k]+cs2[k])/n
+      
+      q += (cs1[k]-ek1)**2/(ek1<0.5?0.5:ek1)+
+      (cs2[k]-ek2)**2/(ek2<0.5?0.5:ek2)
+    end
+    
+    q
+  end # calc_chisq
+  
+  
+end # module

data/lib/fselector/algo_continuous/normalizer.rb

@@ -0,0 +1,59 @@
+#
+# normalize continuous feature
+#
+module Normalizer
+   # log transformation, requires positive feature values 
+   def normalize_log!(base=10)
+     each_sample do |k, s|
+       s.keys.each do |f|
+         s[f] = Math.log(s[f], base) if s[f] > 0.0
+       end
+     end
+   end
+   
+   
+   # scale to [min,max], max > min
+   def normalize_min_max!(min=0.0, max=1.0)
+     # first determine min and max for each feature
+     f2min_max = {}
+          
+     each_feature do |f|
+       fvs = get_feature_values(f)
+       f2min_max[f] = [fvs.min, fvs.max]
+     end
+     
+     # then normalize
+     each_sample do |k, s|
+       s.keys.each do |f|
+         min_v, max_v = f2min_max[f]
+         s[f] = min + (s[f]-min_v) * (max-min) / (max_v-min_v)
+       end
+     end
+   end
+   
+   
+   # by z-score
+   def normalize_zscore!
+     # first determine mean and sd for each feature
+     f2mean_sd = {}
+     
+     each_feature do |f|
+       fvs = get_feature_values(f)
+       f2mean_sd[f] = fvs.mean, fvs.sd
+     end
+     
+     # then normalize
+     each_sample do |k, s|
+       s.keys.each do |f|
+         mean, sd = f2mean_sd[f]
+         if sd.zero?
+           s[f] = 0.0
+         else
+           s[f] = (s[f]-mean)/sd
+         end
+       end
+     end
+   end
+
+  
+end # module