fselector 0.7.0 → 0.8.0

data/.yardopts

@@ -0,0 +1,6 @@
+--readme README.md
+--markup markdown
+--default-return ""
+--title "FSelector Documentation"
+--no-private
+--files README.md,ChangeLog,LICENSE lib

data/ChangeLog

@@ -1,3 +1,8 @@
+2012-04-23	version 0.8.0
+
+  * add new algorithm FTest (FT) for continuous feature
+  * add .yardoc_opts to gem to use the MarkDown documentation syntax
+  
 2012-04-20	Tiejun Cheng	<need47@gmail.com>
 
   * update to version 0.7.0

data/README.md

@@ -8,8 +8,8 @@ FSelector: a Ruby gem for feature selection and ranking
 **Email**: [need47@gmail.com](mailto:need47@gmail.com)  
 **Copyright**: 2012  
 **License**: MIT License  
-**Latest Version**: 0.7.0  
-**Release Date**: April 20 2012
+**Latest Version**: 0.8.0  
+**Release Date**: April 23 2012
 
 Synopsis
 --------
@@ -27,7 +27,8 @@ outputs a reduced data set with only selected subset of features, which
 can later be used as the input for various machine learning softwares 
 such as LibSVM and WEKA. FSelector, as a collection of filter methods, 
 does not implement any classifier like support vector machines or 
-random forest. See below for a list of FSelector's features.
+random forest. See below for a list of FSelector's features and 
+{file:ChangeLog} for updates.
 
 Feature List
 ------------
@@ -74,6 +75,7 @@ Feature List
     SymmetricalUncertainty            SU          discrete
     BetweenWithinClassesSumOfSquare   BSS_WSS     continuous
     CFS_c                             CFS_c       continuous
+    FTest                             FT          continuous
     PMetric                           PM          continuous    two-class
     Relief_c                          Relief_c    continuous    two-class, no missing data
     ReliefF_c                         ReliefF_c   continuous
@@ -122,7 +124,7 @@ To install FSelector, use the following command:
   as a seemless bridge to access the statistical routines in the R package (http://www.r-project.org), 
   which will greatly expand the inclusion of algorithms to FSelector, especially for those relying 
   on statistical test. To this end, please pre-install the R package. RinRuby should have been 
-  auto-installed with FSelector by the above command.
+  auto-installed with FSelector via the above command.
   
 Usage
 -----

data/lib/fselector.rb

@@ -6,7 +6,7 @@ require 'rinruby'
 #
 module FSelector
   # module version
-  VERSION = '0.7.0'
+  VERSION = '0.8.0'
 end
 
 # the root dir of FSelector

data/lib/fselector/algo_base/base.rb

@@ -19,7 +19,7 @@ module FSelector
     
     
     #
-    # iterator for each class
+    # iterator for each class, a block must be given
     #
     #     e.g.
     #     self.each_class do |k|
@@ -37,7 +37,7 @@ module FSelector
     
     
     #
-    # iterator for each feature
+    # iterator for each feature, a block must be given
     #
     #     e.g.
     #     self.each_feature do |f|
@@ -55,12 +55,12 @@ module FSelector
     
     
     #
-    # iterator for each sample with class label
+    # iterator for each sample with class label, a block must be given
     #
     #     e.g.
     #     self.each_sample do |k, s|
     #       print k
-    #       s.each { |f, v| ' '+v }
+    #       s.each { |f, v| print " #{v}" }
     #       puts
     #     end
     #
@@ -119,7 +119,7 @@ module FSelector
     # @param [Symbol] f feature of interest
     # @param [Symbol] mv including missing feature values?
     #   don't include missing feature values (recorded as nils)
-    #   if mv==nil, include otherwise
+    #   if nil, include otherwise
     # @param [Symbol] ck class of interest.
     #   return feature values for all classes, otherwise return feature
     #   values for the specific class (ck)
@@ -166,7 +166,7 @@ module FSelector
     
     
     # get a copy of data, 
-    # by use of the standard Marshal library
+    # by means of the standard Marshal library
     def get_data_copy
       Marshal.load(Marshal.dump(@data)) if @data
     end
@@ -208,7 +208,7 @@ module FSelector
     # get scores of all features for all classes
     #
     # @return [Hash] \{ feature => 
-    #                \{ class_1 => score_1, class_2 => score_2, :BEST => score_best } }
+    #                \{ class\_1 => score\_1, class\_2 => score\_2, :BEST => score\_best } }
     #
     def get_feature_scores
       return @scores if @scores # already done
@@ -258,9 +258,9 @@ module FSelector
     # reconstruct data with selected features
     #
     # @return [Hash] data after feature selection
-    # @note derived class must implement its own get_subset(), 
+    # @note derived class must implement its own get\_subset(), 
     #   and data structure will be altered. For now, only the algorithms of 
-    #   CFS_c, CFS_d and FCBF implement such function
+    #   CFS\_c, CFS\_d and FCBF implemented such functions
     #
     def select_feature!
       subset = get_feature_subset

data/lib/fselector/algo_base/base_CFS.rb

@@ -10,7 +10,7 @@ module FSelector
 #   the original CFS that uses **best first search** only produces slightly better results 
 #   but demands much more computational resources
 #
-# ref: [Feature Selection for Discrete and Numeric Class Machine Learning](http://www.cs.waikato.ac.nz/ml/publications/1999/99MH-Feature-Select.pdf)
+# ref: [Feature Selection for Discrete and Numeric Class Machine Learning](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.30.5673)
 #
   class BaseCFS < Base
     

data/lib/fselector/algo_base/base_Relief.rb

@@ -19,9 +19,9 @@ module FSelector
     #   the number of training samples
     # @param [Hash] data existing data structure
     #
-    def initialize(m=nil, data=nil)
+    def initialize(m=30, data=nil)
       super(data)
-      @m = (m || 30) # default 30
+      @m = m || 30 # default 30
     end
     
     private

data/lib/fselector/algo_base/base_ReliefF.rb

@@ -17,13 +17,13 @@ module FSelector
     # @param [Integer] m number of samples to be used 
     #   for estimating feature contribution. max can be
     #   the number of training samples
-    # @param [Integer] k number of k-nearest neighbor
+    # @param [Integer] k number of k-nearest neighbors
     # @param [Hash] data existing data structure
     #
-    def initialize(m=nil, k=nil, data=nil)
+    def initialize(m=30, k=10, data=nil)
       super(data)
-      @m = (m || 30) # default 30
-      @k = (k || 10) # default 10
+      @m = m || 30 # default 30
+      @k = k || 10 # default 10
     end
     
     private

data/lib/fselector/algo_continuous/BSS_WSS.rb

@@ -5,9 +5,15 @@ module FSelector
 #
 # between-within classes sum of squares (BSS/WSS) for continous feature
 # 
-#                   sigma_i sigma_k I(yi=k)(xbar_kj - xbar_xj)2
+#                   sigma_i sigma_k I(y_i=k)(xbar_k - xbar)^2
 #     BSS_WSS(f) = ----------------------------------------------
-#                     sigma_i sigma_k I(yi=k)(x_ij - xbar_kj)2
+#                     sigma_i sigma_k I(y_i=k)(x_i - xbar_k)^2
+#
+#     where I(y_i=k) is a indicator function with value of 0 or 1
+#           xbar_k is the sample mean of class k
+#           xbar is the overall sample mean
+#           x_i is the value of sample i
+#           y_i is the class label of sample i
 #
 # ref: [Comparison of Discrimination Methods for the Classification of Tumors Using Gene Expression Data](http://amstat.tandfonline.com/doi/abs/10.1198/016214502753479248)
 #
@@ -17,18 +23,22 @@ module FSelector
     
     # calculate contribution of each feature (f) across all classes
     def calc_contribution(f)
-      xbar_xj = get_feature_values(f).mean
+      xbar = get_feature_values(f).mean
       
       a, b, s = 0.0, 0.0, 0.0
       
+      k2xbar = {} # cache
+      each_class do |k|
+        k2xbar[k] = get_feature_values(f, nil, k).mean # w/o missing values
+      end
+      
       each_sample do |k, sam|
-        # get the feature values for class k, without missing values
-        xbar_kj = get_feature_values(f, nil, k).mean
-        a += (xbar_kj - xbar_xj)**2
+        xbar_k = k2xbar[k]
+        a += (xbar_k - xbar)**2
         
         if sam.has_key? f
-          x_ij = sam[f]
-          b += (x_ij - xbar_kj)**2
+          x_i = sam[f]
+          b += (x_i - xbar_k)**2
         end
       end
       

data/lib/fselector/algo_continuous/CFS_c.rb

@@ -6,7 +6,7 @@ module FSelector
 # Correlation-based Feature Selection (CFS) algorithm for continuous feature (CFS\_c). 
 # For CFS\_c, use **select\_feature!** for feature selection
 #
-# ref: [Feature Selection for Discrete and Numeric Class Machine Learning](http://www.cs.waikato.ac.nz/ml/publications/1999/99MH-Feature-Select.pdf)
+# ref: [Feature Selection for Discrete and Numeric Class Machine Learning](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.30.5673)
 #
   class CFS_c < BaseCFS
     # include normalizer and discretizer

data/lib/fselector/algo_continuous/FTest.rb

@@ -0,0 +1,98 @@
+#
+# FSelector: a Ruby gem for feature selection and ranking
+#
+module FSelector
+#
+# F-test (FT) based on F-statistics for continous feature
+# 
+#              between-group variability
+#     FT(f) = ---------------------------
+#              within-group variability
+#
+#              sigma_k n_k*(ybar_k - ybar)^2 / (K-1)
+#           = --------------------------------------
+#              sigma_ik (y_ik - ybar_k)^2 / (N-K)
+#
+#     where n_k is the sample size of class k
+#           ybar_k is the sample mean of class k
+#           ybar is the overall smaple mean
+#           K is the number of classes
+#           y_ik is the value of sample i of class k
+#           N is the overall sample size
+#
+# ref: [Wikipedia](http://en.wikipedia.org/wiki/F-test#Formula_and_calculation) and [Minimum redundancy feature selection from microarray gene expression data](http://penglab.janelia.org/papersall/docpdf/2004_JBCB_feasel-04-06-15.pdf)
+#
+  class FTest < BaseContinuous
+        
+    private    
+    
+    # calculate contribution of each feature (f) across all classes
+    def calc_contribution2(f)
+      a, b, s = 0.0, 0.0, 0.0
+      ybar = get_feature_values(f).mean
+      kz = get_classes.size.to_f
+      sz = get_sample_size.to_f
+      
+      k2ybar = {} # cache
+      each_class do |k|
+        k2ybar[k] = get_feature_values(f, nil, k).mean        
+      end
+      
+      # a
+      each_class do |k|
+        n_k = get_data[k].size.to_f
+        a += n_k * (k2ybar[k] - ybar)**2 / (kz-1)
+      end
+      
+      # b
+      each_sample do |k, s|
+        if s.has_key? f
+          y_ik = s[f]
+          b += (y_ik - k2ybar[k])**2 / (sz-kz)
+        end
+      end
+      
+      s = a/b if not b.zero?
+      
+      set_feature_score(f, :BEST, s)
+    end # calc_contribution
+    
+    def calc_contribution(f)
+      a, b, s = 0.0, 0.0, 0.0
+      ybar = get_feature_values(f).mean
+      kz = get_classes.size.to_f
+      sz = get_sample_size.to_f
+      
+      k2ybar = {} # cache
+      each_class do |k|
+        k2ybar[k] = get_feature_values(f, nil, k).mean        
+      end
+      
+      # a
+      each_class do |k|
+        n_k = get_data[k].size.to_f
+        a += n_k * (k2ybar[k] - ybar)**2 / (kz-1)
+      end
+      
+      # b
+      each_sample do |k, s|
+        if s.has_key? f
+          y_ik = s[f]
+          b += (y_ik - k2ybar[k])**2 / (sz-kz)
+        end
+      end
+      
+      s = a/b if not b.zero?
+      
+      set_feature_score(f, :BEST, s)
+    end # calc_contribution
+    
+    
+  end # class
+  
+  
+  # shortcut so that you can use FSelector::FT instead of FSelector::FTest
+  FT = FTest
+  
+  
+end # module

data/lib/fselector/algo_discrete/BiNormalSeparation.rb

@@ -10,7 +10,7 @@ module FSelector
 #     where F'(x) is normal inverse cumulative distribution function
 #     R equivalent: qnorm
 # 
-# ref: [An extensive empirical study of feature selection metrics for text classification](http://dl.acm.org/citation.cfm?id=944974) and [Rubystats](http://rubystats.rubyforge.org)
+# ref: [An extensive empirical study of feature selection metrics for text classification](http://dl.acm.org/citation.cfm?id=944974)
 #
   class BiNormalSeparation < BaseDiscrete
         

data/lib/fselector/algo_discrete/CFS_d.rb

@@ -6,7 +6,7 @@ module FSelector
 # Correlation-based Feature Selection (CFS) algorithm for discrete feature (CFS\_d). 
 # For CFS\_d, use **select\_feature!** for feature selection
 #
-# ref: [Feature Selection for Discrete and Numeric Class Machine Learning](http://www.cs.waikato.ac.nz/ml/publications/1999/99MH-Feature-Select.pdf)
+# ref: [Feature Selection for Discrete and Numeric Class Machine Learning](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.30.5673)
 #
   class CFS_d < BaseCFS
     # include Entropy module

data/lib/fselector/algo_discrete/FastCorrelationBasedFilter.rb

@@ -15,20 +15,13 @@ module FSelector
     #
     # initialize from an existing data structure
     #
-    # @param [Float] delta predefined threshold.
-    #   if not provided, use 1/sqrt(alpha*m) where
-    #   alpha is confidence level and m is sample size
-    #   respectively.
+    # @param [Float] delta predefined threshold
     #
-    def initialize(delta=nil, data=nil)
+    def initialize(delta=0.0, data=nil)
       super(data)
       @delta = delta || 0.0
     end
     
-    # undefine superclass methods
-    undef :select_feature_by_score!
-    undef :select_feature_by_rank!
-    
     private
     
     # Fast Correlation-Based Filter(FCBF) algorithm

data/lib/fselector/algo_discrete/FishersExactTest.rb

@@ -13,7 +13,7 @@ module FSelector
 #     so that the larger is always the better (consistent with other algorithms)
 #     R equivalent: fisher.test
 #
-# ref: [Wikipedia](http://en.wikipedia.org/wiki/Fisher's_exact_test) and [Rubystats](http://rubystats.rubyforge.org)
+# ref: [Wikipedia](http://en.wikipedia.org/wiki/Fisher\'s_exact_test)
 #
   class FishersExactTest < BaseDiscrete
         

data/lib/fselector/algo_discrete/GiniIndex.rb

@@ -5,7 +5,7 @@ module FSelector
 #
 # Gini Index (GI), generalized for multi-class problem
 #
-#     GI(f) = 1 - sigma(c)(P(c|f)^2)
+#     GI(f) = 1 - sigma_c (P(c|f)^2)
 #
 # for GI, the smaller, the better, but we intentionally negate it
 # so that the larger is always the better (consistent with other algorithms)

data/lib/fselector/algo_discrete/InformationGain.rb

@@ -7,9 +7,9 @@ module FSelector
 #
 #     IG(c,f) = H(c) - H(c|f)
 #     
-#     where H(c) = -1 * sigma_i (P(ci) logP(ci))
+#     where H(c) = -1 * sigma_i (P(ci) log2 P(ci))
 #           H(c|f) = sigma_j (P(fj)*H(c|fj))
-#           H(c|fj) = -1 * sigma_k (P(ck|fj) logP(ck|fj))
+#           H(c|fj) = -1 * sigma_k (P(ck|fj) log2 P(ck|fj))
 #
 # ref: [Using Information Gain to Analyze and Fine Tune the Performance of Supply Chain Trading Agents](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.141.7895)
 #

data/lib/fselector/algo_discrete/Power.rb

@@ -22,7 +22,7 @@ module FSelector
     #
     def initialize(k=5, data=nil)
       super(data)
-      @k = k
+      @k = k || 5
     end
     
     private

data/lib/fselector/algo_discrete/Sensitivity.rb

@@ -9,6 +9,8 @@ module FSelector
 #     SN = ------- = -----
 #            TP+FN     A+C
 #  
+# ref: [Wikipedia](http://en.wikipedia.org/wiki/Sensitivity_and_specificity)
+#
   class Sensitivity < BaseDiscrete
     
     private

data/lib/fselector/algo_discrete/Specificity.rb

@@ -8,6 +8,8 @@ module FSelector
 #             TN        D
 #     SP  = ------- = -----
 #            TN+FP     B+D
+#
+# ref: [Wikipedia](http://en.wikipedia.org/wiki/Sensitivity_and_specificity)
 #
   class Specificity < BaseDiscrete
     

data/lib/fselector/algo_discrete/SymmetricalUncertainty.rb

@@ -9,10 +9,10 @@ module FSelector
 #     SU(c,f) = 2 * ------------- = ---------------
 #                    H(c) + H(f)      H(c) + H(f)
 #     
-#     where H(c) = -1 * sigma_i (P(ci) logP(ci))
+#     where H(c) = -1 * sigma_i (P(ci) log2 P(ci))
 #           H(c|f) = sigma_j (P(fj)*H(c|fj))
-#           H(c|fj) = -1 * sigma_k (P(ck|fj) logP(ck|fj))
-#           H(f) = -1 * sigma_i (P(fi) logP(fi))
+#           H(c|fj) = -1 * sigma_k (P(ck|fj) log2 P(ck|fj))
+#           H(f) = -1 * sigma_i (P(fi) log2 P(fi))
 #
 # ref: [Wikipedia](http://en.wikipedia.org/wiki/Symmetric_uncertainty)
 #

data/lib/fselector/discretizer.rb

@@ -153,9 +153,10 @@ module Discretizer
   #
   # discretize by Chi2 algorithm
   #
+  # @param [Float] delta data inconsistency rate upper bound
   # @note our implementation of Chi2 algo is **NOT** 
-  #   the exactly same as the original one and Chi2 
-  #   does some feature reduction if one feature has only one interval
+  #   the exactly same as the original one, and Chi2 
+  #   does some feature reduction if a feature has only one interval
   #
   # ref: [Chi2: Feature Selection and Discretization of Numeric Attributes](http://sci2s.ugr.es/keel/pdf/specific/congreso/liu1995.pdf)
   #
@@ -288,7 +289,7 @@ module Discretizer
   #
   # discretize by Multi-Interval Discretization (MID) algorithm
   #
-  # @note no missing feature values allowed and data structure will be altered
+  # @note no missing feature value is allowed, and data structure will be altered
   # 
   # ref: [Multi-Interval Discretization of Continuous-Valued Attributes for Classification Learning](http://www.ijcai.org/Past%20Proceedings/IJCAI-93-VOL2/PDF/022.pdf)
   #

data/lib/fselector/ensemble.rb

@@ -58,9 +58,9 @@ module FSelector
     # @param [Method] by_what by what criterion that ensemble
     #   score should be obtained from those of individual algorithms  
     #   allowed values are:  
-    #   receiver.method(:by\_min) # by min rank  
-    #   receiver.method(:by\_max) # by max rank  
-    #   receiver.method(:by\_ave) # by ave rank
+    #   - method(:by\_min) # by min score  
+    #   - method(:by\_max) # by max score  
+    #   - method(:by\_ave) # by ave score
     # @param [Integer] norm normalization  
     #   :min\_max, score scaled to [0, 1]  
     #   :zscore, score converted to zscore
@@ -96,9 +96,9 @@ module FSelector
     # @param [Method] by_what by what criterion that ensemble
     #   rank should be obtained from those of individual algorithms  
     #   allowed values are:  
-    #   method(:by\_min) # by min rank  
-    #   method(:by\_max) # by max rank  
-    #   method(:by\_ave) # by ave rank
+    #   - method(:by\_min) # by min rank  
+    #   - method(:by\_max) # by max rank  
+    #   - method(:by\_ave) # by ave rank
     #
     def ensemble_by_rank(by_what=method(:by_min))
       ranks = {}

data/lib/fselector/entropy.rb

@@ -5,7 +5,7 @@ module Entropy
   #
   # get the marginal entropy of array (X)
   #
-  #     H(X) = -1 * sigma_i (P(x_i) logP(x_i))
+  #     H(X) = -1 * sigma_i (P(x_i) log2 P(x_i))
   #
   # @param [Array] arrX array of interest
   # @return [Float] H(X)
@@ -27,7 +27,7 @@ module Entropy
   #
   #     H(X|Y) = sigma_j (P(y_j) * H(C|y_j))
   #     
-  #     where H(X|y_j) = -1 * sigma_i (P(x_i|y_j) logP(x_i|y_j))
+  #     where H(X|y_j) = -1 * sigma_i (P(x_i|y_j) log2 P(x_i|y_j))
   #
   # @param [Array] arrX the first array
   # @param [Array] arrY the second array

data/lib/fselector/fileio.rb

@@ -385,7 +385,7 @@ module FileIO
   # @param [Symbol] format sparse or regular ARFF  
   #   :sparse => sparse ARFF, otherwise regular ARFF
   #
-  def data_to_weka(fname=:stdout, format=nil)
+  def data_to_weka(fname=:stdout, format=:sparse)
     if fname == :stdout
       ofs = $stdout
     else
@@ -443,7 +443,7 @@ module FileIO
         end
         ofs.print "#{get_features.size} #{k}"
         ofs.puts "}"
-      else
+      else # regular ARFF
         each_feature do |f|
           if s.has_key? f
             ofs.print "#{s[f]},"

data/lib/fselector/normalizer.rb

@@ -6,7 +6,12 @@ module Normalizer
    def normalize_by_log!(base=10)
      each_sample do |k, s|
        s.keys.each do |f|
-         s[f] = Math.log(s[f], base) if s[f] > 0.0
+         if s[f] > 0.0
+           s[f] = Math.log(s[f], base) 
+         else
+           abort "[#{__FILE__}@#{__LINE__}]: "+
+              "feature value must be positive"
+         end
        end
      end
    end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fselector
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-20 00:00:00.000000000 Z
+date: 2012-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rinruby
-  requirement: &26347848 !ruby/object:Gem::Requirement
+  requirement: &25600464 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,7 +21,7 @@ dependencies:
         version: 2.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *26347848
+  version_requirements: *25600464
 description: FSelector is a Ruby gem that aims to integrate various feature selection/ranking
   algorithms and related functions into one single package. Welcome to contact me
   (need47@gmail.com) if you'd like to contribute your own algorithms or report a bug.
@@ -36,13 +36,12 @@ description: FSelector is a Ruby gem that aims to integrate various feature sele
 email: need47@gmail.com
 executables: []
 extensions: []
-extra_rdoc_files:
-- README.md
-- LICENSE
+extra_rdoc_files: []
 files:
 - README.md
 - ChangeLog
 - LICENSE
+- .yardopts
 - lib/fselector/algo_base/base.rb
 - lib/fselector/algo_base/base_CFS.rb
 - lib/fselector/algo_base/base_continuous.rb
@@ -51,6 +50,7 @@ files:
 - lib/fselector/algo_base/base_ReliefF.rb
 - lib/fselector/algo_continuous/BSS_WSS.rb
 - lib/fselector/algo_continuous/CFS_c.rb
+- lib/fselector/algo_continuous/FTest.rb
 - lib/fselector/algo_continuous/PMetric.rb
 - lib/fselector/algo_continuous/ReliefF_c.rb
 - lib/fselector/algo_continuous/Relief_c.rb