statsample 0.17.0 → 0.18.0

data/History.txt

@@ -1,3 +1,30 @@
+=== 0.18.0 / 2011-01-07
+* New Statsample.load_excel 
+* New Statsample.load_csv
+* Statsample::Dataset#[] accepts an array of fields and uses clone
+* New Dataset#correlation_matrix  and Statsample::Dataset#covariance_matrix
+* Statsample::Dataset.filter add labels to vectors
+* Principal Components generation complete on PCA (covariance matrix prefered)
+* Added note on Statsample::Factor::PCA about erratic signs on eigenvalues,
+* Statsample::Factor::PCA.component_matrix calculated different for covariance matrix
+* Improved summary for PCA using covariance matrix
+* New attribute :label_angle for Statsample::Graph::Boxplot
+* Fixed Scatterplots scaling problems
+* New attributes for Scatterplots: groups, minimum_x, minimum_y, maximum_x, 
+* New Statsample::Multiset#union allows to create a new dataset based on a m
+* New Statsample::Multiset#each to traverse through datasets
+* Bug fix: Vector#standarized and Vector#percentile crash on nil data
+* Bug fix: Vector#mean and Vector#sd crash on data without valid values
+* Modified methods names on Statsample::Factor::PCA : feature_vector to feature_matrix, data_transformation to principal_components
+* Added Statsample::Vector.vector_centered
+* Factor::MAP.with_dataset() implemented 
+* Bug fix: Factor::MAP with correlation matrix with non-real eigenvalues crashes * Added documentation for Graph::Histogram 
+* Added MPA to Reliability::MultiScaleAnalysis
+* Added custom names for returned vectors and datasets 
+* Updated spanish traslation 
+* Graph::Histogram updated. Custom x and y max and min, optional normal distribution drawing 
+* Updated Histogram class, with several new methods compatibles with GSL::Histogram
+
 === 0.17.0 / 2010-12-09
 * Added Statsample::Graph::Histogram and Statsample::Graph::Boxplot
 * Added Statsample::Reliability::SkillScaleAnalysis for analysis of skill based scales.

data/Manifest.txt

@@ -105,6 +105,7 @@ po/es/statsample.po
 po/statsample.pot
 references.txt
 setup.rb
+test/fixtures/bank2.dat
 test/fixtures/correlation_matrix.rb
 test/helpers_tests.rb
 test/test_anovaoneway.rb

data/Rakefile

@@ -5,7 +5,6 @@ $:.unshift(File.dirname(__FILE__)+'/lib/')
 
 require 'rubygems'
 require 'statsample'
-
 require 'hoe'
 Hoe.plugin :git
 
@@ -41,9 +40,9 @@ h=Hoe.spec('statsample') do
   #self.testlib=:minitest
 	self.rubyforge_name = "ruby-statsample"
 	self.developer('Claudio Bustos', 'clbustos@gmail.com')
-	self.extra_deps << ["spreadsheet","~>0.6.0"] <<  ["reportbuilder", "~>1.4"] << ["minimization", "~>0.2.0"] << ["fastercsv", ">0"] << ["dirty-memoize", "~>0.0"] << ["extendmatrix","~>0.3.1"] << ["statsample-bivariate-extension", ">0"] << ["rserve-client", "~>0.2.5"] << ["rubyvis", "~>0.3.3"]
+	self.extra_deps << ["spreadsheet","~>0.6.5"] <<  ["reportbuilder", "~>1.4"] << ["minimization", "~>0.2.0"] << ["fastercsv", ">0"] << ["dirty-memoize", "~>0.0"] << ["extendmatrix","~>0.3.1"] << ["statsample-bivariate-extension", ">0"] << ["rserve-client", "~>0.2.5"] << ["rubyvis", "~>0.4.0"]
   
-	self.extra_dev_deps << ["shoulda"] << ["minitest", "~>2.0"]
+	self.extra_dev_deps << ["hoe","~>0"] << ["shoulda","~>0"] << ["minitest", "~>2.0"]
   self.clean_globs << "test/images/*" << "demo/item_analysis/*" << "demo/Regression"
   self.post_install_message = <<-EOF
 ***************************************************

data/examples/histogram.rb

@@ -1,14 +1,13 @@
 #!/usr/bin/ruby
 $:.unshift(File.dirname(__FILE__)+'/../lib/')
 $:.unshift('/home/cdx/dev/reportbuilder/lib/')
-
-require 'benchmark'
 require 'statsample'
-n=1000
-a=n.times.map {|i| rand()*20}.to_scale
-hg=Statsample::Graph::Histogram.new(a, :bins=>15)
+n=3000
+rng=Distribution::Normal.rng_ugaussian
+a=n.times.map {|i| rng.call()*20}.to_scale
+hg=Statsample::Graph::Histogram.new(a, :bins=>20, :line_normal_distribution=>true )
 
 rb=ReportBuilder.new
-rb.add(a.histogram)
+#rb.add(a.histogram)
 rb.add(hg)
-puts rb.to_text
+rb.save_html('histogram.html')

data/lib/statsample.rb

@@ -118,7 +118,7 @@ module Statsample
     @@has_gsl
   end
   
-  VERSION = '0.17.0'
+  VERSION = '0.18.0'
   SPLIT_TOKEN = ","
   autoload(:Database, 'statsample/converters')
   autoload(:Anova, 'statsample/anova')
@@ -157,6 +157,30 @@ module Statsample
         false
       end
     end
+    # Import an Excel file. Cache result by default
+    def load_excel(filename, opts=Hash.new, cache=true)
+      file_ds=filename+".ds"
+      if cache and (File.exists? file_ds and File.mtime(file_ds)>File.mtime(filename))
+        ds=Statsample.load(file_ds)
+      else
+        ds=Statsample::Excel.read(filename)
+        ds.save(file_ds) if cache
+      end
+      ds
+    end
+    
+    # Import an Excel file. Cache result by default
+    def load_csv(filename, opts=Hash.new, cache=true)
+      file_ds=filename+".ds"
+      if cache and (File.exists? file_ds and File.mtime(file_ds)>File.mtime(filename))
+        ds=Statsample.load(file_ds)
+      else
+        ds=Statsample::CSV.read(filename,opts)
+        ds.save(file_ds) if cache
+      end
+      ds
+    end
+    
     
     # Create a matrix using vectors as columns.
     # Use:

data/lib/statsample/bivariate.rb

@@ -58,13 +58,6 @@ module Statsample
         # Calculate sum of squares
         ss=sum_of_squares(v1a,v2a)
         ss.quo(Math::sqrt(v1a.sum_of_squares) * Math::sqrt(v2a.sum_of_squares))
-=begin        
-        v1s,v2s=v1a.vector_standarized,v2a.vector_standarized
-        t=0
-        siz=v1s.size
-        (0...v1s.size).each {|i| t+=(v1s[i]*v2s[i]) }
-        t.quo(v2s.size-1)
-=end
       end
       alias :correlation :pearson
       # Retrieves the value for t test for a pearson correlation

data/lib/statsample/converters.rb

@@ -17,6 +17,7 @@ module Statsample
         fields=[]
         sth.column_info.each {|c|
             vectors[c['name']]=Statsample::Vector.new([])
+            vectors[c['name']].name=c['name']
             vectors[c['name']].type= (c['type_name']=='INTEGER' or c['type_name']=='DOUBLE') ? :scale : :nominal
             fields.push(c['name'])
         }
@@ -35,7 +36,7 @@ module Statsample
       #  dbh = DBI.connect("DBI:Mysql:database:localhost", "user", "password")
       #  Statsample::Database.insert(ds,dbh,"test")
       #
-      def insert(ds, dbh,table)
+      def insert(ds, dbh, table)
         require 'dbi'            
         query="INSERT INTO #{table} ("+ds.fields.join(",")+") VALUES ("+((["?"]*ds.fields.size).join(","))+")"
         sth=dbh.prepare(query)
@@ -235,6 +236,7 @@ module Statsample
         fields.each {|f|
           ds[f].name=f
         }
+        ds.name=filename
         ds
       end
     end

data/lib/statsample/dataset.rb

@@ -25,7 +25,7 @@ module Statsample
     end
     def to_s
       m="Error on iteration: "+@exp.message+"\n"+@exp.backtrace.join("\n")
-      m+="\nRow: #{@ds.i}" unless @ds.i.nil?
+      m+="\nRow ##{@ds.i}:#{@ds.case_as_hash(@ds.i)}" unless @ds.i.nil?
       m
     end
   end
@@ -140,7 +140,6 @@ module Statsample
       end
       @i=nil
     end
-    
     #
     # Returns a GSL::matrix
     #
@@ -239,6 +238,7 @@ module Statsample
         ds[f]=@vectors[f]
       }
       ds.fields=fields_to_include
+      ds.name=@name
       ds.update_valid_data
       ds
     end
@@ -419,13 +419,15 @@ module Statsample
     # if fields parameter is empty, sum all fields 
     def vector_sum(fields=nil)
       fields||=@fields
-      collect_with_index do |row, i|
+      vector=collect_with_index do |row, i|
         if(fields.find{|f| !@vectors[f].data_with_nils[i]})
           nil
         else
           fields.inject(0) {|ac,v| ac + row[v].to_f}
         end
       end
+      vector.name=_("Sum from %s") % @name
+      vector
     end
     # Check if #fields attribute is correct, after inserting or deleting vectors
     def check_fields(fields)
@@ -476,7 +478,9 @@ module Statsample
           a.push(sum.quo(size-invalids))
         end
       end
-      a.to_vector(:scale)
+      a=a.to_vector(:scale)
+      a.name=_("Means from %s") % @name
+      a
     end
     # Check vectors for type and size.
     def check_length # :nodoc:
@@ -598,8 +602,9 @@ module Statsample
     def[](i)
       if i.is_a? Range
         fields=from_to(i.begin,i.end)
-        vectors=fields.inject({}) {|a,v| a[v]=@vectors[v];a}
-        Dataset.new(vectors,fields)
+        clone(*fields)
+      elsif i.is_a? Array
+        clone(i)
       else
         raise Exception,"Vector '#{i}' doesn't exists on dataset" unless @vectors.has_key?(i)
         @vectors[i]
@@ -661,16 +666,36 @@ module Statsample
       GSL::Matrix.alloc(*rows)
       end
     end
-		
-   
+    
+    # Return a correlation matrix for fields included as parameters.
+    # By default, uses all fields of dataset
+		def correlation_matrix(fields=nil)
+      if fields
+        ds=clone(fields)
+      else
+        ds=self
+      end
+      Statsample::Bivariate.correlation_matrix(ds)
+    end
+   # Return a correlation matrix for fields included as parameters.
+    # By default, uses all fields of dataset
+		def covariance_matrix(fields=nil)
+      if fields
+        ds=clone(fields)
+      else
+        ds=self
+      end
+      Statsample::Bivariate.covariance_matrix(ds)
+    end
     
     # Create a new dataset with all cases which the block returns true
     def filter
       ds=self.dup_empty
       each {|c|
-        ds.add_case(c,false) if yield c
+        ds.add_case(c, false) if yield c
       }
       ds.update_valid_data
+      ds.name=_("%s(filtered)") % @name
       ds
     end
     
@@ -712,6 +737,8 @@ module Statsample
           #        puts "Vector #{k1}:"+v1.to_s
           v1.type=@vectors[k1].type
           v1.name=@vectors[k1].name
+          v1.labels=@vectors[k1].labels
+          
         }
       }
       ms
@@ -737,9 +764,16 @@ module Statsample
       
       ms.datasets.each do |k,ds|
         ds.update_valid_data
+        ds.name=fields.size.times.map {|i|
+          f=fields[i]
+          sk=k[i]
+          @vectors[f].labeling(sk)
+        }.join("-")
         ds.vectors.each{|k1,v1| 
           v1.type=@vectors[k1].type
           v1.name=@vectors[k1].name
+          v1.labels=@vectors[k1].labels
+          
         }
       end
       ms
@@ -805,7 +839,7 @@ module Statsample
       vr
     end
     def to_s
-      "#<"+self.class.to_s+":"+self.object_id.to_s+" @fields=["+@fields.join(",")+"] cases="+@vectors[@fields[0]].size.to_s
+      "#<"+self.class.to_s+":"+self.object_id.to_s+" @name=#{@name} @fields=["+@fields.join(",")+"] cases="+@vectors[@fields[0]].size.to_s
     end
     def inspect
       self.to_s

data/lib/statsample/factor.rb

@@ -13,7 +13,18 @@ module Statsample
   #   * Statsample::Factor::Varimax
   #   * Statsample::Factor::Equimax
   #   * Statsample::Factor::Quartimax
-  # See documentation of each class to use it
+  # * Classes for determining the number of components
+  #   * Statsample::Factor::MAP
+  #   * Statsample::Factor::ParallelAnalysis
+  #
+  # About number of components, O'Connor(2000) said:
+  #  The two procedures [PA and MAP ] complement each other nicely,
+  #  in that the MAP tends to err (when it does err) in the direction
+  #  of underextraction, whereas parallel analysis tends to err
+  #  (when it does err) in the direction of overextraction.
+  #  Optimal decisions are thus likely to be made after considering
+  #  the results of both analytic procedures. (p.10)
+
   module Factor
     # Anti-image covariance matrix.
     # Useful for inspection of desireability of data for factor analysis.

data/lib/statsample/factor/map.rb

@@ -48,6 +48,9 @@ module Statsample
       attr_reader :fm
       # Smallest average squared correlation
       attr_reader :minfm
+      def self.with_dataset(ds,opts=Hash.new)
+        new(ds.correlation_matrix,opts)
+      end
       def initialize(matrix, opts=Hash.new)
         @matrix=matrix
         opts_default={
@@ -76,10 +79,15 @@ module Statsample
         end
         minfm=fm[0]
         nfactors=0
+        @errors=[]
         fm.each_with_index do |v,s|
-          if v < minfm
-            minfm=v
-            nfactors=s
+          if v.is_a? Complex
+            @errors.push(s)
+          else
+            if v < minfm
+              minfm=v
+              nfactors=s
+            end
           end
         end
         @number_of_factors=nfactors
@@ -89,13 +97,13 @@ module Statsample
       def report_building(g) #:nodoc:
         g.section(:name=>@name) do |s|
           s.table(:name=>_("Eigenvalues"),:header=>[_("Value")]) do |t|
-            eigenvalues.each do |e|
-              t.row(["%0.6f" % e])
+            eigenvalues.each_with_index do |e,i|
+                t.row([@errors.include?(i) ? "*" : "%0.6f" % e])
             end
           end
           s.table(:name=>_("Velicer's Average Squared Correlations"), :header=>[_("number of components"),_("average square correlation")]) do |t|
             fm.each_with_index do |v,i|
-              t.row(["%d" % i, "%0.6f" % v])
+              t.row(["%d" % i, @errors.include?(i) ? "*" : "%0.6f" % v])
             end
           end
           s.text(_("The smallest average squared correlation is : %0.6f" % minfm))

data/lib/statsample/factor/parallelanalysis.rb

@@ -39,15 +39,12 @@ module Statsample
       attr_reader :ds
       # Bootstrap method. <tt>:random</tt> used by default
       # * <tt>:random</tt>: uses number of variables and cases for the dataset
-      # * <tt>:data</tt> : sample with replacement from actual data.       
-
+      # * <tt>:data</tt> : sample with replacement from actual data.
       attr_accessor :bootstrap_method
       # Uses smc on diagonal of matrixes, to perform simulation
       # of a Principal Axis analysis.
       # By default, false.
-
       attr_accessor :smc
-
       # Percentil over bootstrap eigenvalue should be accepted. 95 by default
       attr_accessor :percentil
       # Correlation matrix used with :raw_data . <tt>:correlation_matrix</tt> used by default

data/lib/statsample/factor/pca.rb

@@ -1,8 +1,14 @@
+# encoding: UTF-8
 module Statsample
 module Factor
-  # Principal Component Analysis (PCA) of a 
-  # covariance or correlation matrix. 
+  # Principal Component Analysis (PCA) of a covariance or 
+  # correlation matrix.. 
   #
+  # NOTE: Sign of second and later eigenvalues could be different
+  # using Ruby or GSL, so values for PCs and component matrix
+  # should differ, because extendmatrix and gsl's methods to calculate
+  # eigenvectors are different. Using R is worse, cause first 
+  # eigenvector could have negative values!
   # For Principal Axis Analysis, use Statsample::Factor::PrincipalAxis
   # 
   # == Usage:
@@ -26,6 +32,7 @@ module Factor
   # == References:
   # * SPSS Manual
   # * Smith, L. (2002). A tutorial on Principal Component Analysis. Available on http://courses.eas.ualberta.ca/eas570/pca_tutorial.pdf 
+  # * Härdle, W. & Simar, L. (2003). Applied Multivariate Statistical Analysis. Springer
   # 
   class PCA
     include Summarizable
@@ -43,12 +50,16 @@ module Factor
     attr_accessor :summary_parallel_analysis
     # Type of rotation. By default, Statsample::Factor::Rotation::Varimax
     attr_accessor :rotation_type
-
+    attr_accessor :type
     def initialize(matrix, opts=Hash.new)
       @use_gsl=nil
       @name=_("Principal Component Analysis")
       @matrix=matrix
-      @n_variables=@matrix.column_size
+      @n_variables=@matrix.column_size      
+      @variables_names=(@matrix.respond_to? :fields) ? @matrix.fields : @n_variables.times.map {|i| _("VAR_%d") % (i+1)}
+      
+      @type = @matrix.respond_to?(:type) ? @matrix.type : :correlation
+      
       @m=nil
       
       @rotation_type=Statsample::Factor::Varimax
@@ -65,15 +76,19 @@ module Factor
         @variables_names=@n_variables.times.map {|i| "V#{i+1}"}
       end
       calculate_eigenpairs
+      
       if @m.nil?
         # Set number of factors with eigenvalues > 1
         @m=@eigenpairs.find_all {|ev,ec| ev>=1.0}.size
       end
-
+      
     end
     def rotation
       @rotation_type.new(component_matrix)
     end
+    def total_eigenvalues
+      eigenvalues.inject(0) {|ac,v| ac+v}
+    end
     def create_centered_ds
       h={}
       @original_ds.factors.each {|f|
@@ -83,8 +98,10 @@ module Factor
       @ds=h.to_dataset
     end
     
-    # Feature vector for m factors
-    def feature_vector(m=nil)
+    # Feature matrix for +m+ factors
+    # Returns +m+ eigenvectors as columns.
+    # So, i=variable, j=component
+    def feature_matrix(m=nil)
       m||=@m
       omega_m=::Matrix.build(@n_variables, m) {0}
       m.times do |i|
@@ -92,15 +109,48 @@ module Factor
       end
       omega_m
     end
-    # data_transformation
-    def data_transformation(data_matrix, m)
+    # Returns Principal Components for +input+ matrix or dataset
+    # The number of PC to return is equal to parameter +m+. 
+    # If +m+ isn't set, m set to number of PCs selected at object creation. 
+    def principal_components(input, m=nil)
+      data_matrix=input.to_matrix
+      var_names=(data_matrix.respond_to? :fields_y) ? data_matrix.fields_y : data_matrix.column_size.times.map {|i| "VAR_%d" % (i+1)}
       m||=@m
-      raise "Data variables number should be equal to original variable number" if data_matrix.size2!=@n_variables
-      fv=feature_vector(m)
-      (fv.transpose*data_matrix.transpose).transpose
+      
+      raise "data matrix variables<>pca variables" if data_matrix.column_size!=@n_variables
+      
+      fv=feature_matrix(m)
+      pcs=(fv.transpose*data_matrix.transpose).transpose
+      pcs.extend Statsample::NamedMatrix
+      pcs.fields_y=m.times.map {|i| "PC_%d" % (i+1)}
+      pcs.to_dataset
     end
-    # Component matrix for m factors
     def component_matrix(m=nil)
+      var="component_matrix_#{type}"
+      send(var,m)
+    end
+    # Matrix with correlations between components and
+    # variables. Based on Härdle & Simar (2003, p.243)
+    def component_matrix_covariance(m=nil)
+      m||=@m
+      raise "m should be > 0" if m<1
+      ff=feature_matrix(m)
+      cm=::Matrix.build(@n_variables, m) {0}
+      @n_variables.times {|i|
+        m.times {|j|
+          cm[i,j]=ff[i,j] * Math.sqrt(eigenvalues[j] / @matrix[i,i])
+        }
+      }
+      cm.extend CovariateMatrix
+      cm.name=_("Component matrix (from covariance)")
+      cm.fields_x = @variables_names
+      cm.fields_y = m.times.map {|i| "PC_%d" % (i+1)}
+      
+      cm
+    end
+    # Matrix with correlations between components and
+    # variables
+    def component_matrix_correlation(m=nil)
       m||=@m
       raise "m should be > 0" if m<1
       omega_m=::Matrix.build(@n_variables, m) {0}
@@ -115,17 +165,17 @@ module Factor
       cm.extend CovariateMatrix
       cm.name=_("Component matrix")
       cm.fields_x = @variables_names
-      cm.fields_y = m.times.map {|i| "component_#{i+1}"}
+      cm.fields_y = m.times.map {|i| "PC_%d" % (i+1)}
       cm
     end
-    # Communalities for all variables given m factors
     def communalities(m=nil)
+      
       m||=@m
       h=[]
       @n_variables.times do |i|
         sum=0
         m.times do |j|
-          sum+=@eigenpairs[j][0].abs*@eigenpairs[j][1][i]**2
+          sum+=(@eigenpairs[j][0].abs*@eigenpairs[j][1][i]**2)
         end
         h.push(sum)
       end
@@ -135,7 +185,11 @@ module Factor
     def eigenvalues
       @eigenpairs.collect {|c| c[0] }
     end
-    
+    def eigenvectors
+      @eigenpairs.collect {|c| 
+        c[1].to_matrix
+      }
+    end
     def calculate_eigenpairs
       if @use_gsl
         calculate_eigenpairs_gsl
@@ -144,14 +198,18 @@ module Factor
       end
     end
   
-    def calculate_eigenpairs_ruby
+    def calculate_eigenpairs_ruby #:nodoc:
       @eigenpairs = @matrix.eigenpairs_ruby
     end
-    def calculate_eigenpairs_gsl
+    # Eigenvectors calculated with gsl
+    # Note: The signs of some vectors could be different of
+    # ruby generated
+    def calculate_eigenpairs_gsl #:nodoc:
       eigval, eigvec= GSL::Eigen.symmv(@matrix.to_gsl)
-      
+      #puts "***"
       ep=eigval.size.times.map {|i|
-        [eigval[i], eigvec.get_col(i)]
+        ev=eigvec.get_col(i)
+        [eigval[i], ev]
       }
       @eigenpairs=ep.sort{|a,b| a[0]<=>b[0]}.reverse
     end
@@ -159,20 +217,23 @@ module Factor
     def report_building(builder) # :nodoc:
       builder.section(:name=>@name) do |generator|
         generator.text _("Number of factors: %d") % m
-        generator.table(:name=>_("Communalities"), :header=>[_("Variable"),_("Initial"),_("Extraction")]) do |t|
+        generator.table(:name=>_("Communalities"), :header=>[_("Variable"),_("Initial"),_("Extraction"), _("%")]) do |t|
           communalities(m).each_with_index {|com, i|
-            t.row([@variables_names[i], 1.0, sprintf("%0.3f", com)])
+            perc=com*100.quo(@matrix[i,i])
+            t.row([@variables_names[i], "%0.3f" % @matrix[i,i]  , "%0.3f" % com, "%0.3f" % perc])
           }
         end
-        
+        te=total_eigenvalues
         generator.table(:name=>_("Total Variance Explained"), :header=>[_("Component"), _("E.Total"), _("%"), _("Cum. %")]) do |t|
           ac_eigen=0
           eigenvalues.each_with_index {|eigenvalue,i|
             ac_eigen+=eigenvalue
-            t.row([_("Component %d") % (i+1), sprintf("%0.3f",eigenvalue), sprintf("%0.3f%%", eigenvalue*100.quo(@n_variables)), sprintf("%0.3f",ac_eigen*100.quo(@n_variables))])
+            t.row([_("Component %d") % (i+1), sprintf("%0.3f",eigenvalue), sprintf("%0.3f%%", eigenvalue*100.quo(te)), sprintf("%0.3f",ac_eigen*100.quo(te))])
           }
         end
+        
         generator.parse_element(component_matrix(m))
+                  
         if (summary_rotation)
           generator.parse_element(rotation)
         end