fselector 0.1.0

data/lib/fselector/algo_discrete/Random.rb

@@ -0,0 +1,40 @@
+#
+# FSelector: a Ruby gem for feature selection and ranking
+#
+module FSelector
+#
+# Random (Rand), no pratical use but can be used as a baseline
+#
+#  Rand = rand numbers within [0..1)
+#  
+# ref: [An extensive empirical study of feature selection metrics
+#       for text classification][url]
+# [url]: http://dl.acm.org/citation.cfm?id=944974
+#
+  class Random < BaseDiscrete
+    #
+    # initialize from an existing data structure
+    #
+    # @param [Integer] seed seed form random number
+    #   generator. provided for reproducible results, 
+    #   otherwise use current time as seed
+    #
+    def initialize(seed=nil, data=nil)
+      super(data)
+      srand(seed) if seed
+    end
+    
+    private
+    
+    # calculate contribution of each feature (f) for each class (k)
+    def calc_contribution(f)
+      each_class do |k|
+        set_feature_score(f, k, rand)
+      end
+    end # calc_contribution
+    
+    
+  end # class
+  
+  
+end # module

data/lib/fselector/algo_discrete/ReliefF_d.rb

@@ -0,0 +1,173 @@
+#
+# FSelector: a Ruby gem for feature selection and ranking
+#
+module FSelector
+# extended Relief algorithm for discrete feature (ReliefF_d)
+#
+# @note applicable to multi-class problem with missing data
+#
+# ref: [Estimating Attributes: Analysis and Extensions of RELIEF][url]
+# [url]: http://www.springerlink.com/content/fp23jh2h0426ww45/
+#
+  class ReliefF_d < BaseDiscrete
+    #
+    # new()
+    #
+    # @param [Integer] m number of samples to be used 
+    #   for estimating feature contribution. max can be
+    #   the number of training samples
+    # @param [Integer] k number of k-nearest neighbor
+    # @param [Hash] data existing data structure
+    #
+    def initialize(m=nil, k=10, data=nil)
+      super(data)
+      @m = m # use all samples
+      @k = (k || 10)  # default 10
+    end
+    
+    private
+    
+    # calculate contribution of each feature (f) across all classes
+    def calc_contribution(f)      
+      score = 0.0
+      
+      # use all samples if @m not provided
+      @m = get_sample_size if not @m
+      
+      @m.times do
+        # pick a sample at random
+        rs, rk = pick_a_sample_at_random
+        
+        # find k nearest neighbor for each class
+        nbrs = find_k_nearest_nb(rs, rk)
+        
+        # calc contribution from neighbors
+        score += calc_score(f, rs, rk, nbrs)
+      end
+      
+      s = score / @m
+      
+      set_feature_score(f, :BEST, s)
+    end # calc_contribution
+    
+    
+    # pick a sample at random
+    def pick_a_sample_at_random
+      rk = get_classes[rand(get_classes.size)]
+      rks = get_data[rk]
+      
+      [ rks[rand(rks.size)], rk ]
+    end # pick_a_sample_at_random
+    
+    # # find k nearest neighbors of sample (rs) for each class 
+    def find_k_nearest_nb(rs, rk)
+      nbrs = {}
+      
+      each_class do |k|
+        res = []
+        
+        get_data[k].each do |s|
+          next if s == rs # exclude self
+          
+          d = diff_sample(rs, s, rk, k)
+          res << [d, s]
+        end
+        
+        nbrs[k] = (res.sort { |x, y| x[0] <=> y[0] }[0...@k]).collect { |z| z[1] }
+      end
+      
+      nbrs
+    end # find_k_nearest_nb
+    
+    
+    # difference between two samples
+    def diff_sample(s1, s2, k1, k2)
+      d = 0.0
+            
+      each_feature do |f|
+        d += diff_feature(f, s1, s2, k1, k2)**2
+      end
+      
+      d
+    end # diff_sample
+    
+    
+    # difference beween the feature (f) of two samples
+    def diff_feature(f, s1, s2, k1, k2)
+      d = 0.0
+      
+      if s1.has_key?(f) and s2.has_key?(f) # no missing value
+        d = (s1[f] == s2[f]) ? 0.0 : 1.0
+      elsif not s1.has_key?(f) and not s2.has_key?(f) # two missing values
+        fvs = get_feature_values(f).uniq
+        fvs.each do |mv|
+          d -= calc_p(f, mv, k1)*calc_p(f, mv, k2)
+        end
+        d += 1
+      elsif not s1.has_key?(f) # s1: one missing value
+        # diff(f, s1, s2) = 1 - P(value(f, s2)|class(s1))
+        d = 1 - calc_p(f, s2[f], k1)
+      else # s2: one missing value
+        # diff(f, s1, s2) = 1 - P(value(f, s1)|class(s2))
+        d = 1 - calc_p(f, s1[f], k2)
+      end
+      
+      d
+    end # diff_feature
+    
+    
+    # calc probability of missing value (mv)
+    def calc_p(f, mv, k) 
+      # cache
+      if not @f2mvp
+        @f2mvp = {}
+        
+        each_feature do |f|
+          @f2mvp[f] = {}
+          
+          each_class do |k|
+            @f2mvp[f][k] = {}
+            
+            fvs = get_feature_values(f).uniq
+            fvs.each do |v|
+              n = 0.0
+              
+              get_data[k].each do |s|
+                n += 1 if s.has_key?(f) and s[f] == v
+              end
+              
+              @f2mvp[f][k][v] = n/get_data[k].size
+            end
+          end
+        end
+      end
+      
+      @f2mvp[f][k][mv]
+    end
+    
+    
+    # calc feature (f) contribution from neighbors
+    def calc_score(f, rs, rk, nbrs)
+      score = 0.0
+      
+      nbrs.each do |k, nbs|
+        if k == rk # near hit
+          nbs.each do |s|
+            score -= (diff_feature(f, rs, s, rk, k)**2/nbs.size)
+          end
+        else # near_miss
+          nbs.each do |s|
+            score += (get_data[k].size/get_sample_size.to_f *
+                     diff_feature(f, rs, s, rk, k)**2/nbs.size)
+          end
+        end
+      end
+      
+      score
+    end
+    
+    
+  end # class
+  
+  
+end # module

data/lib/fselector/algo_discrete/Relief_d.rb

@@ -0,0 +1,135 @@
+#
+# FSelector: a Ruby gem for feature selection and ranking
+#
+module FSelector
+#
+# Relief algorithm for discrete feature (Relief_d)
+#
+# @note Relief applicable only to two-class problem without missing data
+#
+# ref: [The Feature Selection Problem: Traditional Methods 
+#       and a New Algorithm][url]
+# [url]: http://www.aaai.org/Papers/AAAI/1992/AAAI92-020.pdf
+#
+  class Relief_d < BaseDiscrete
+    #
+    # new()
+    #
+    # @param [Integer] m number of samples to be used 
+    #   for estimating feature contribution. max can be
+    #   the number of training samples
+    # @param [Hash] data existing data structure
+    #
+    def initialize(m=nil, data=nil)
+      super(data)
+      @m = m # default use all samples
+    end
+    
+    private
+    
+    # calculate contribution of each feature (f) across all classes
+    def calc_contribution(f)
+      if not get_classes.size == 2
+        abort "[#{__FILE__}@#{__LINE__}]: "+
+        "Relief applicable only to two-class problems without missing data"
+      end
+      
+      # use all samples if @m not provided
+      @m = get_sample_size if not @m
+      
+      k1, k2 = get_classes
+      score = 0.0
+      
+      @m.times do        
+        # pick a sample at random
+        rs, rk = pick_a_sample_at_random
+        
+        # find the nearest neighbor for each class
+        nbrs = find_nearest_nb(rs, rk)
+        
+        # calc contribution from neighbors
+        score += calc_score(f, rs, rk, nbrs)
+      end
+      
+      s = score / @m
+      
+      set_feature_score(f, :BEST, s)
+    end # calc_contribution
+    
+    
+    # pick a sample at random
+    def pick_a_sample_at_random
+      rk = get_classes[rand(get_classes.size)]
+      rks = get_data[rk]
+      
+      [ rks[rand(rks.size)], rk ]
+    end # pick_a_sample_at_random
+    
+    
+    # find nearest neighbor sample for given sample (rs) within class (k)
+    def find_nearest_nb(rs, rk)
+      nbrs = {}
+      
+      each_class do |k|
+        nb, dmin = nil, 999
+        get_data[k].each do |s|
+          next if s == rs # exclude self
+          d = diff_sample(rs, s)
+          if d < dmin
+            dmin = d
+            nb = s
+          end
+        end
+        
+        nbrs[k] = nb
+      end
+      
+      nbrs
+    end # find_nearest_nb
+    
+    
+    # difference between two samples
+    def diff_sample(s1, s2)
+      d = 0.0
+      
+      each_feature do |f|
+        d += diff_feature(f, s1, s2)**2
+      end
+      
+      d
+    end # diff_sample
+    
+    
+    # difference beween the feature (f) of two samples
+    def diff_feature(f, s1, s2)
+      d = 0.0
+      
+      if not s1.has_key?(f) or not s2.has_key?(f)
+        abort "[#{__FILE__}@#{__LINE__}]: "+
+              "Relief does not allow missing values"
+      end
+      
+      (s1[f] == s2[f]) ? 0.0 : 1.0
+    end # diff_feature
+    
+    
+    # calc feature (f) contribution from neighbors
+    def calc_score(f, rs, rk, nbrs)
+      score = 0.0
+      
+      nbrs.each do |k, s|
+        if k == rk # near hit
+          score -= diff_feature(f, rs, s)**2
+        else # near_miss
+          score += diff_feature(f, rs, s)**2
+        end
+      end
+      
+      score
+    end # calc_score
+    
+    
+  end # class
+  
+  
+end # module

data/lib/fselector/algo_discrete/Sensitivity.rb

@@ -0,0 +1,38 @@
+#
+# FSelector: a Ruby gem for feature selection and ranking
+#
+module FSelector
+#
+# Sensitivity (SN)
+#
+#             TP        A
+#     SN = ------- = -----
+#            TP+FN     A+C
+#  
+  class Sensitivity < BaseDiscrete
+    
+    private
+    
+    # calculate contribution of each feature (f) for each class (k)
+    def calc_contribution(f)
+      each_class do |k|
+        a, c = get_A(f, k), get_C(f, k)
+        
+        s = a/(a+c)
+        
+        set_feature_score(f, k, s)
+      end
+    end # calc_contribution
+    
+    
+  end # class
+  
+  
+  
+  # shortcut so that you can use FSelector::SN instead of FSelector::Sensitivity
+  SN = Sensitivity
+  # Sensitivity, also known as Recall
+  Recall = Sensitivity
+  
+  
+end # module

data/lib/fselector/algo_discrete/Specificity.rb

@@ -0,0 +1,35 @@
+#
+# FSelector: a Ruby gem for feature selection and ranking
+#
+module FSelector
+#
+# Specificity (SP)
+#
+#             TN        D
+#     SP  = ------- = -----
+#            TN+FP     B+D
+#
+  class Specificity < BaseDiscrete
+    
+    private
+    
+    # calculate contribution of each feature (f) for each class (k)
+    def calc_contribution(f)
+      each_class do |k|
+        b, d = get_B(f, k), get_D(f, k)
+        
+        s = d/(b+d)
+        
+        set_feature_score(f, k, s)
+      end
+    end # calc_contribution
+    
+    
+  end # class
+  
+  
+  # shortcut so that you can use FSelector::SP instead of FSelector::Specificity
+  SP = Specificity
+  
+  
+end # module

data/lib/fselector/base.rb

@@ -0,0 +1,322 @@
+#
+# FSelector: a Ruby gem for feature selection and ranking
+#
+module FSelector
+  #
+  # base ranking algorithm
+  #
+  class Base
+    # include FileIO
+    include FileIO
+    
+    # initialize from an existing data structure
+    def initialize(data=nil)
+      @data = data
+      @opts = {} # store non-data information
+    end
+    
+    
+    #
+    # iterator for each class
+    #
+    #     e.g.
+    #     self.each_class do |k|
+    #       puts k
+    #     end
+    #
+    def each_class
+      if not block_given?
+        abort "[#{__FILE__}@#{__LINE__}]: "+
+              "block must be given!"
+      else
+        get_classes.each { |k| yield k }
+      end
+    end
+    
+    
+    #
+    # iterator for each feature
+    #
+    #     e.g.
+    #     self.each_feature do |f|
+    #       puts f
+    #     end
+    #
+    def each_feature
+      if not block_given?
+        abort "[#{__FILE__}@#{__LINE__}]: "+
+              "block must be given!"
+      else
+        get_features.each { |f| yield f }
+      end
+    end
+    
+    
+    #
+    # iterator for each sample with class label
+    #
+    #     e.g.
+    #     self.each_sample do |k, s|
+    #       print k
+    #       s.each { |f, v| ' '+v }
+    #       puts
+    #     end
+    #
+    def each_sample
+      if not block_given?
+        abort "[#{__FILE__}@#{__LINE__}]: "+
+              " block must be given!"
+      else      
+        get_data.each do |k, samples|
+          samples.each { |s| yield k, s }
+        end
+      end
+    end
+    
+    
+    # get classes
+    def get_classes
+      @classes ||= @data.keys
+    end
+    
+    
+    # set classes
+    def set_classes(classes)
+      if classes and classes.class == Array
+        @classes = classes
+      else
+        abort "[#{__FILE__}@#{__LINE__}]: "+
+              "classes must be a Array object!"
+      end
+    end
+
+    
+    # get unique features
+    def get_features
+      @features ||= @data.map { |x| x[1].map { |y| y.keys } }.flatten.uniq
+    end
+    
+    
+    #
+    # get feature values
+    #
+    # @param [Symbol] f feature of interest
+    #
+    def get_feature_values(f)
+      @fvs ||= {}
+      
+      if not @fvs.has_key? f
+        @fvs[f] = []
+        each_sample do |k, s|
+          @fvs[f] << s[f] if s.has_key? f
+        end
+      end
+      
+      @fvs[f]
+    end
+    
+    # set features
+    def set_features(features)
+      if features and features.class == Array
+        @features = features
+      else
+        abort "[#{__FILE__}@#{__LINE__}]: "+
+              "features must be a Array object!"
+      end
+    end
+    
+    
+    # get data
+    def get_data
+      @data
+    end
+    
+    # set data
+    def set_data(data)
+      if data and data.class == Hash
+        @data = data
+        # clear
+        @classes, @features, @fvs = nil, nil, nil
+        @scores, @ranks, @sz = nil, nil, nil
+      else
+        abort "[#{__FILE__}@#{__LINE__}]: "+
+              "data must be a Hash object!"
+      end
+    end
+    
+    
+    # get non-data information
+    def get_opt(key)
+      @opts.has_key?(key) ? @opts[key] : nil
+    end
+    
+    
+    # set non-data information as a key-value pair
+    def set_opt(key, value)
+      @opts[key] = value
+    end
+    
+    
+    # number of samples
+    def get_sample_size
+      @sz ||= get_data.values.flatten.size
+    end
+    
+    
+    #
+    # print feature scores
+    #
+    # @param [String] kclass class of interest
+    #
+    def print_feature_scores(feat=nil, kclass=nil)
+      scores = get_feature_scores
+      
+      scores.each do |f, ks|
+        next if feat and feat != f
+        
+        print "#{f} =>"
+        ks.each do |k, s|
+          if kclass
+            print " #{k}->#{s}" if k == kclass
+          else
+            print " #{k}->#{s}"
+          end
+        end
+        puts
+      end
+    end
+    
+    
+    # print feature ranks
+    def print_feature_ranks
+      ranks = get_feature_ranks
+      
+      ranks.each do |f, r|
+        puts "#{f} => #{r}"
+      end
+    end
+    
+    
+    #
+    # get scores of all features for all classes
+    #
+    # @return [Hash] \{ feature => 
+    #                \{ class_1 => score_1, class_2 => score_2, :BEST => score_best } }
+    #
+    def get_feature_scores
+      return @scores if @scores # already done
+      
+      each_feature do |f|
+        calc_contribution(f)
+      end
+      
+      # best score for feature
+      @scores.each do |f, ks|
+        # the larger, the better
+        @scores[f][:BEST] = ks.values.max
+      end
+      #@scores.each { |x,v| puts "#{x} => #{v[:BEST]}" }
+      
+      @scores
+    end
+    
+    
+    # set feature (f) score (f) for class (k)
+    def set_feature_score(f, k, s)
+      @scores ||= {}
+      @scores[f] ||= {}
+      @scores[f][k] = s
+    end
+    
+    #
+    # get the ranked features based on their best scores
+    #
+    # @return [Hash] feature ranks
+    #
+    def get_feature_ranks
+      return @ranks if @ranks # already done
+      
+      scores = get_feature_scores
+      
+      # get the ranked features
+      @ranks = {} # feature => rank
+      
+      # the larger, the better
+      sorted_features = scores.keys.sort do |x,y|
+        scores[y][:BEST] <=> scores[x][:BEST]
+      end
+      
+      sorted_features.each_with_index do |sf, si|
+        @ranks[sf] = si+1
+      end
+      
+      @ranks
+    end
+    
+    
+    #
+    # reconstruct data with feature scores satisfying cutoff
+    #
+    # @param [String] criterion 
+    #   valid criterion can be '>0.5', '>= 0.4', '==2', '<=1' or '<0.2'
+    # @param [Hash] my_scores
+    #   user customized feature scores
+    # @return [Hash] data after feature selection
+    # @note data structure will be altered
+    #
+    def select_data_by_score!(criterion, my_scores=nil)
+      # user scores or internal scores
+      scores = my_scores || get_feature_scores
+      
+      my_data = {}
+      
+      each_sample do |k, s|
+        my_data[k] ||= []
+        my_s = {}
+        
+        s.each do |f, v|
+          my_s[f] = v if eval("#{scores[f][:BEST]} #{criterion}")
+        end
+        
+        my_data[k] << my_s if not my_s.empty?
+      end
+          
+      set_data(my_data)
+    end
+        
+    
+    #
+    # reconstruct data by rank
+    #
+    # @param [String] criterion 
+    #   valid criterion can be '>11', '>= 10', '==1', '<=10' or '<20'
+    # @param [Hash] my_ranks
+    #   user customized feature ranks
+    # @return [Hash] data after feature selection
+    # @note data structure will be altered
+    #
+    def select_data_by_rank!(criterion, my_ranks=nil)
+      # user ranks or internal ranks
+      ranks = my_ranks || get_feature_ranks
+      
+      my_data = {}
+      
+      each_sample do |k, s|
+        my_data[k] ||= []
+        my_s = {}
+        
+        s.each do |f,v|
+          my_s[f] = v if eval("#{ranks[f]} #{criterion}")
+        end
+        
+        my_data[k] << my_s if not my_s.empty?
+      end
+      
+      set_data(my_data)
+    end
+  
+  
+  end # class
+
+
+end # module