statsample 0.18.0 → 1.0.0

data/lib/statsample/bivariate.rb

@@ -1,7 +1,4 @@
 require 'statsample/bivariate/pearson'
-
-
-
 module Statsample
   # Diverse methods and classes to calculate bivariate relations
   # Specific classes: 
@@ -11,7 +8,6 @@ module Statsample
   module Bivariate
     autoload(:Polychoric, 'statsample/bivariate/polychoric')
     autoload(:Tetrachoric, 'statsample/bivariate/tetrachoric')
-    
     class << self
       # Covariance between two vectors
       def covariance(v1,v2)
@@ -27,8 +23,8 @@ module Statsample
       def maximum_likehood_dichotomic(pred,real)
         preda,reala=Statsample.only_valid_clone(pred,real)                
         sum=0
-        pred.each_index{|i|
-           sum+=(real[i]*Math::log(pred[i])) + ((1-real[i])*Math::log(1-pred[i]))
+        preda.each_index{|i|
+           sum+=(reala[i]*Math::log(preda[i])) + ((1-reala[i])*Math::log(1-preda[i]))
         }
         sum
       end
@@ -101,6 +97,20 @@ module Statsample
           cdf*n_tails
         end
       end
+      
+      
+      # Predicted time for pairwise correlation matrix, in miliseconds
+      # See benchmarks/correlation_matrix.rb to see mode of calculation
+      
+      def prediction_pairwise(vars,cases)
+        ((-0.518111-0.000746*cases+1.235608*vars+0.000740*cases*vars)**2) / 100
+      end
+      # Predicted time for optimized correlation matrix, in miliseconds
+      # See benchmarks/correlation_matrix.rb to see mode of calculation
+      
+      def prediction_optimized(vars,cases)
+        ((4+0.018128*cases+0.246871*vars+0.001169*vars*cases)**2) / 100
+      end
       # Returns residual score after delete variance
       # from another variable
       # 
@@ -128,10 +138,35 @@ module Statsample
         
       end
       
+      def covariance_matrix_optimized(ds)
+        x=ds.to_gsl
+        n=x.row_size
+        m=x.column_size
+        means=((1/n.to_f)*GSL::Matrix.ones(1,n)*x).row(0)
+        centered=x-(GSL::Matrix.ones(n,m)*GSL::Matrix.diag(means))
+        ss=centered.transpose*centered
+        s=((1/(n-1).to_f))*ss
+        s
+      end
+      
       # Covariance matrix.
       # Order of rows and columns depends on Dataset#fields order
       
       def covariance_matrix(ds)
+        vars,cases=ds.fields.size,ds.cases
+        if !ds.has_missing_data? and Statsample.has_gsl? and prediction_optimized(vars,cases) < prediction_pairwise(vars,cases)
+          cm=covariance_matrix_optimized(ds)
+        else
+          cm=covariance_matrix_pairwise(ds)
+          
+        end
+        cm.extend(Statsample::CovariateMatrix)
+        cm.fields=ds.fields
+        cm
+      end
+      
+      
+      def covariance_matrix_pairwise(ds)
         cache={}
         matrix=ds.collect_matrix do |row,col|
           if (ds[row].type!=:scale or ds[col].type!=:scale)
@@ -148,15 +183,34 @@ module Statsample
             end
           end
         end
-        matrix.extend CovariateMatrix
-        matrix.fields=ds.fields
         matrix
       end
       
       # Correlation matrix.
       # Order of rows and columns depends on Dataset#fields order
-      
       def correlation_matrix(ds)
+        vars,cases=ds.fields.size,ds.cases
+        if !ds.has_missing_data? and Statsample.has_gsl? and prediction_optimized(vars,cases) < prediction_pairwise(vars,cases)
+          cm=correlation_matrix_optimized(ds)
+        else
+          cm=correlation_matrix_pairwise(ds)
+        end
+        cm.extend(Statsample::CovariateMatrix)
+        cm.fields=ds.fields
+        cm
+      end
+
+      def correlation_matrix_optimized(ds)
+        s=covariance_matrix_optimized(ds)
+        sds=GSL::Matrix.diagonal(s.diagonal.sqrt.pow(-1))
+        cm=sds*s*sds
+        # Fix diagonal
+        s.row_size.times {|i|
+          cm[i,i]=1.0
+        }
+        cm
+      end
+      def correlation_matrix_pairwise(ds)
         cache={}
         cm=ds.collect_matrix do |row,col|
           if row==col
@@ -173,9 +227,6 @@ module Statsample
             end 
           end
         end
-        cm.extend(Statsample::CovariateMatrix)
-        cm.fields=ds.fields
-        cm
       end
       
       # Retrieves the n valid pairwise.
@@ -220,7 +271,7 @@ module Statsample
         m1=ds.filter_field('c') {|c| c['d']!=f0}
         ((m1.mean-m0.mean).to_f / ds['c'].sdp) * Math::sqrt(m0.size*m1.size.to_f / ds.cases**2)
       end
-      # Kendall Rank Correlation Coefficient.
+      # Kendall Rank Correlation Coefficient (Tau a)
       # Based on Hervé Adbi article
       def tau_a(v1,v2)
         v1a,v2a=Statsample.only_valid_clone(v1,v2)
@@ -231,12 +282,15 @@ module Statsample
         delta= o1.size*2-(o2  & o1).size*2
         1-(delta * 2 / (n*(n-1)).to_f)
       end
-      # Calculates Tau b correlation.
-      #
+      # Calculates Goodman and Kruskal’s Tau b correlation.
+      # Tb is an asymmetric P-R-E measure of association for nominal scales 
+      # (Mielke, X)
+      # 
       # Tau-b defines perfect association as strict monotonicity. Although it
       # requires strict monotonicity to reach 1.0, it does not penalize ties as
       # much as some other measures.
-      #
+      # == Reference
+      # Mielke, P. GOODMAN–KRUSKAL TAU AND GAMMA. 
       # Source: http://faculty.chass.ncsu.edu/garson/PA765/assocordinal.htm
       def tau_b(matrix)
         v=pairs(matrix)

data/lib/statsample/dataset.rb

@@ -115,6 +115,10 @@ module Statsample
       ds.update_valid_data
       ds
     end
+    # Return true if any vector has missing data
+    def has_missing_data?
+      @vectors.any? {|k,v| v.has_missing_data?}
+    end
     # Creates a new dataset. A dataset is a set of ordered named vectors
     # of the same size.
     #
@@ -128,6 +132,10 @@ module Statsample
       @@n_dataset||=0
       @@n_dataset+=1
       @name=_("Dataset %d") % @@n_dataset
+      @cases=0
+      @gsl=nil
+      @i=nil
+      
       if vectors.instance_of? Array
         @fields=vectors.dup
         @vectors=vectors.inject({}){|a,x| a[x]=Statsample::Vector.new(); a}
@@ -138,17 +146,6 @@ module Statsample
         check_order
         check_length
       end
-      @i=nil
-    end
-    #
-    # Returns a GSL::matrix
-    #
-    def to_gsl_matrix
-      matrix=GSL::Matrix.alloc(cases,@vectors.size)
-      each_array do |row|
-        row.each_index{|y| matrix.set(@i,y,row[y]) }
-      end
-      matrix
     end
     # 
     # Creates a copy of the given dataset, deleting all the cases with
@@ -375,6 +372,7 @@ module Statsample
     # Check vectors and fields after inserting data. Use only 
     # after  #add_case_array or #add_case with second parameter to false
     def update_valid_data
+      @gsl=nil
       @fields.each{|f| @vectors[f].set_valid_data}
       check_length
     end
@@ -491,7 +489,6 @@ module Statsample
           size=v.size
         else
           if v.size!=size
-            p v.to_a.size
             raise Exception, "Vector #{k} have size #{v.size} and dataset have size #{size}"
           end
         end
@@ -629,7 +626,6 @@ module Statsample
     end
     # Recode a vector based on a block
     def recode!(vector_name)
-      
       0.upto(@cases-1) {|i|
         @vectors[vector_name].data[i]=yield case_as_hash(i)
       }
@@ -658,13 +654,23 @@ module Statsample
     end
     
     if Statsample.has_gsl?
-      def to_matrix_gsl
-      rows=[]
-      self.each_array{|c|
-        rows.push(c)
-      }
-      GSL::Matrix.alloc(*rows)
+      def clear_gsl
+        @gsl=nil
       end
+      
+      def to_gsl
+        if @gsl.nil?
+          if cases.nil?
+            update_valid_data
+          end
+          @gsl=GSL::Matrix.alloc(cases,fields.size)
+          self.each_array{|c|
+            @gsl.set_row(@i,c)
+          }
+        end
+        @gsl
+      end
+      
     end
     
     # Return a correlation matrix for fields included as parameters.

data/lib/statsample/dominanceanalysis.rb

@@ -107,8 +107,8 @@ module Statsample
       else
         @regression_class= UNIVARIATE_REGRESSION_CLASS
         @method_association=:r2
-        
       end
+      
       @name=nil
       opts.each{|k,v|
         self.send("#{k}=",v) if self.respond_to? k
@@ -117,7 +117,7 @@ module Statsample
       @dependent=[@dependent] unless @dependent.is_a? Array
       
       @predictors ||= input.fields-@dependent
-
+      
       @name=_("Dominance Analysis:  %s over %s") % [ @predictors.flatten.join(",") , @dependent.join(",")] if @name.nil?
       
       if input.is_a? Statsample::Dataset

data/lib/statsample/factor.rb

@@ -41,8 +41,10 @@ module Statsample
       aicm
     end
     def self.anti_image_correlation_matrix(matrix)
+      matrix=matrix.to_matrix
       s=Matrix.diag(*(matrix.inverse.diagonal)).sqrt.inverse
       aicm=s*matrix.inverse*s
+      
       aicm.extend(Statsample::CovariateMatrix)
       aicm.fields=matrix.fields if matrix.respond_to? :fields
       aicm

data/lib/statsample/factor/map.rb

@@ -48,32 +48,37 @@ module Statsample
       attr_reader :fm
       # Smallest average squared correlation
       attr_reader :minfm
+      
+      attr_accessor :use_gsl
       def self.with_dataset(ds,opts=Hash.new)
         new(ds.correlation_matrix,opts)
       end
       def initialize(matrix, opts=Hash.new)
         @matrix=matrix
         opts_default={
+          :use_gsl=>true,
           :name=>_("Velicer's MAP")
         }
         @opts=opts_default.merge(opts)
          opts_default.keys.each {|k| send("#{k}=", @opts[k]) }
       end
       def compute
-        eigen=@matrix.eigen
-        eigvect,@eigenvalues=eigen[:eigenvectors], eigen[:eigenvalues]
-        loadings=eigvect*(Matrix.diag(*@eigenvalues).sqrt)
+        gsl_m=(use_gsl and Statsample.has_gsl?) ? @matrix.to_gsl : @matrix
+        klass_m=gsl_m.class
+        eigvect,@eigenvalues=gsl_m.eigenvectors_matrix, gsl_m.eigenvalues
+        eigenvalues_sqrt=@eigenvalues.collect {|v| Math.sqrt(v)}
+        loadings=eigvect*(klass_m.diagonal(*eigenvalues_sqrt))
         fm=Array.new(@matrix.row_size)
         ncol=@matrix.column_size
-        fm[0]=(@matrix.mssq - ncol).quo(ncol*(ncol-1))
+        
+        fm[0]=(gsl_m.mssq - ncol).quo(ncol*(ncol-1))
+        
         (ncol-1).times do |m|
           puts "MAP:Eigenvalue #{m+1}" if $DEBUG
           a=loadings[0..(loadings.row_size-1),0..m]
-          partcov= @matrix - (a*a.t)
-          pc_prediag=partcov.row_size.times.map{|i|
-            1.quo(Math::sqrt(partcov[i,i]))
-          }
-          d=Matrix.diag(*pc_prediag)
+          partcov= gsl_m - (a*a.transpose)
+          
+          d=klass_m.diagonal(*(partcov.diagonal.collect {|v| Math::sqrt(1/v)}))
           pr=d*partcov*d
           fm[m+1]=(pr.mssq-ncol).quo(ncol*(ncol-1))
         end
@@ -81,7 +86,7 @@ module Statsample
         nfactors=0
         @errors=[]
         fm.each_with_index do |v,s|
-          if v.is_a? Complex
+          if defined?(Complex) and v.is_a? ::Complex
             @errors.push(s)
           else
             if v < minfm
@@ -93,6 +98,7 @@ module Statsample
         @number_of_factors=nfactors
         @fm=fm
         @minfm=minfm
+        
       end
       def report_building(g) #:nodoc:
         g.section(:name=>@name) do |s|

data/lib/statsample/factor/parallelanalysis.rb

@@ -58,7 +58,7 @@ module Statsample
       attr_accessor :no_data
       # Show extra information if true
       attr_accessor :debug
-
+      attr_accessor :use_gsl
       def initialize(ds, opts=Hash.new)
         @ds=ds
         @fields=@ds.fields
@@ -74,6 +74,7 @@ module Statsample
           :no_data=>false,
           :matrix_method=>:correlation_matrix
         }
+        @use_gsl=Statsample.has_gsl?
         @opts=opts_default.merge(opts)
         @opts[:matrix_method]==:correlation_matrix if @opts[:bootstrap_method]==:parameters
         opts_default.keys.each {|k| send("#{k}=", @opts[k]) }
@@ -120,11 +121,12 @@ module Statsample
       # Perform calculation. Shouldn't be called directly for the user
       def compute
         
+        
         @original=Statsample::Bivariate.send(matrix_method, @ds).eigenvalues unless no_data        
         @ds_eigenvalues=Statsample::Dataset.new((1..@n_variables).map{|v| "ev_%05d" % v})
         @ds_eigenvalues.fields.each {|f| @ds_eigenvalues[f].type=:scale}
         if bootstrap_method==:parameter or bootstrap_method==:random
-          rng = Distribution::Normal.rng_ugaussian
+          rng = Distribution::Normal.rng
         end
         
         @iterations.times do |i|
@@ -132,16 +134,20 @@ module Statsample
             puts "#{@name}: Iteration #{i}" if $DEBUG or debug
             # Create a dataset of dummy values
             ds_bootstrap=Statsample::Dataset.new(@ds.fields)
+            
             @fields.each do |f|
               if bootstrap_method==:random
                 ds_bootstrap[f]=@n_cases.times.map {|c| rng.call}.to_scale
               elsif bootstrap_method==:data
-                ds_bootstrap[f]=ds[f].sample_with_replacement(@n_cases).to_scale
+                ds_bootstrap[f]=ds[f].sample_with_replacement(@n_cases)
               else
                 raise "bootstrap_method doesn't recogniced"
               end
             end
+            ds_bootstrap.update_valid_data
+            
             matrix=Statsample::Bivariate.send(matrix_method, ds_bootstrap)
+            matrix=matrix.to_gsl if @use_gsl
             if smc
                 smc_v=matrix.inverse.diagonal.map{|ii| 1-(1.quo(ii))}
                 smc_v.each_with_index do |v,ii| 

data/lib/statsample/factor/pca.rb

@@ -50,7 +50,7 @@ module Factor
     attr_accessor :summary_parallel_analysis
     # Type of rotation. By default, Statsample::Factor::Rotation::Varimax
     attr_accessor :rotation_type
-    attr_accessor :type
+    attr_accessor :matrix_type
     def initialize(matrix, opts=Hash.new)
       @use_gsl=nil
       @name=_("Principal Component Analysis")
@@ -58,7 +58,7 @@ module Factor
       @n_variables=@matrix.column_size      
       @variables_names=(@matrix.respond_to? :fields) ? @matrix.fields : @n_variables.times.map {|i| _("VAR_%d") % (i+1)}
       
-      @type = @matrix.respond_to?(:type) ? @matrix.type : :correlation
+      @matrix_type = @matrix.respond_to?(:_type) ? @matrix._type : :correlation
       
       @m=nil
       
@@ -103,30 +103,45 @@ module Factor
     # So, i=variable, j=component
     def feature_matrix(m=nil)
       m||=@m
-      omega_m=::Matrix.build(@n_variables, m) {0}
-      m.times do |i|
-        omega_m.column= i, @eigenpairs[i][1]
+      if @use_gsl
+        omega_m=GSL::Matrix.zeros(@n_variables,m)
+        ev=eigenvectors
+        m.times do |i|
+          omega_m.set_column(i,ev[i])
+        end
+        omega_m
+      else
+        omega_m=::Matrix.build(@n_variables, m) {0}
+        m.times do |i|
+          omega_m.column= i, @eigenpairs[i][1]
+        end
+        omega_m
       end
-      omega_m
     end
     # Returns Principal Components for +input+ matrix or dataset
     # The number of PC to return is equal to parameter +m+. 
-    # If +m+ isn't set, m set to number of PCs selected at object creation. 
+    # If +m+ isn't set, m set to number of PCs selected at object creation.
+    # Use covariance matrix
+    
     def principal_components(input, m=nil)
-      data_matrix=input.to_matrix
-      var_names=(data_matrix.respond_to? :fields_y) ? data_matrix.fields_y : data_matrix.column_size.times.map {|i| "VAR_%d" % (i+1)}
+      if @use_gsl
+        data_matrix=input.to_gsl
+      else
+        data_matrix=input.to_matrix
+      end
       m||=@m
       
       raise "data matrix variables<>pca variables" if data_matrix.column_size!=@n_variables
       
       fv=feature_matrix(m)
       pcs=(fv.transpose*data_matrix.transpose).transpose
+      
       pcs.extend Statsample::NamedMatrix
       pcs.fields_y=m.times.map {|i| "PC_%d" % (i+1)}
       pcs.to_dataset
     end
     def component_matrix(m=nil)
-      var="component_matrix_#{type}"
+      var="component_matrix_#{matrix_type}"
       send(var,m)
     end
     # Matrix with correlations between components and
@@ -141,7 +156,7 @@ module Factor
           cm[i,j]=ff[i,j] * Math.sqrt(eigenvalues[j] / @matrix[i,i])
         }
       }
-      cm.extend CovariateMatrix
+      cm.extend NamedMatrix
       cm.name=_("Component matrix (from covariance)")
       cm.fields_x = @variables_names
       cm.fields_y = m.times.map {|i| "PC_%d" % (i+1)}
@@ -187,32 +202,13 @@ module Factor
     end
     def eigenvectors
       @eigenpairs.collect {|c| 
-        c[1].to_matrix
+        @use_gsl ? c[1].to_gsl : c[1].to_vector
       }
     end
     def calculate_eigenpairs
-      if @use_gsl
-        calculate_eigenpairs_gsl
-      else
-        calculate_eigenpairs_ruby
-      end
+      @eigenpairs= @use_gsl ? @matrix.to_gsl.eigenpairs : @matrix.to_matrix.eigenpairs_ruby 
     end
   
-    def calculate_eigenpairs_ruby #:nodoc:
-      @eigenpairs = @matrix.eigenpairs_ruby
-    end
-    # Eigenvectors calculated with gsl
-    # Note: The signs of some vectors could be different of
-    # ruby generated
-    def calculate_eigenpairs_gsl #:nodoc:
-      eigval, eigvec= GSL::Eigen.symmv(@matrix.to_gsl)
-      #puts "***"
-      ep=eigval.size.times.map {|i|
-        ev=eigvec.get_col(i)
-        [eigval[i], ev]
-      }
-      @eigenpairs=ep.sort{|a,b| a[0]<=>b[0]}.reverse
-    end
     
     def report_building(builder) # :nodoc:
       builder.section(:name=>@name) do |generator|