statsample 0.18.0 → 1.0.0

data/lib/statsample/vector/gsl.rb

@@ -0,0 +1,106 @@
+module Statsample
+  class Vector
+    module GSL_      
+      def clear_gsl
+        @gsl=nil
+      end
+      
+      def set_valid_data
+        clear_gsl
+        set_valid_data_ruby
+      end
+      def push(v)
+        # If data is GSL::Vector, should be converted first to an Array
+        if @data.is_a? GSL::Vector
+          @data=@data.to_a
+        end
+        push_ruby(v)
+      end
+      
+      def gsl
+        @gsl||=GSL::Vector.alloc(@scale_data) if @scale_data.size>0
+      end
+      
+      alias :to_gsl :gsl
+      def vector_standarized_compute(m,sd)
+        if flawed?
+          vector_standarized_compute_ruby(m,sd)
+        else
+          gsl.collect {|x| (x.to_f - m).quo(sd)}.to_scale
+        end
+      end
+      
+      def vector_centered_compute(m)
+        if flawed?
+          vector_centered_compute_ruby(m)
+        else
+          gsl.collect {|x| (x.to_f - m)}.to_scale
+        end
+      end
+      def sample_with_replacement(sample=1)
+        if(@type!=:scale)
+          sample_with_replacement_ruby(sample)
+        else
+          r = GSL::Rng.alloc(GSL::Rng::MT19937,rand(10000))
+          Statsample::Vector.new(r.sample(gsl, sample).to_a,:scale)
+        end
+      end
+      
+      def sample_without_replacement(sample=1)
+        if(@type!=:scale)
+          sample_without_replacement_ruby(sample)
+        else
+          r = GSL::Rng.alloc(GSL::Rng::MT19937,rand(10000))
+          r.choose(gsl, sample).to_a
+        end
+      end
+      def median
+        if @type!=:scale
+          median_ruby
+        else
+          sorted=GSL::Vector.alloc(@scale_data.sort)
+          GSL::Stats::median_from_sorted_data(sorted)
+        end
+      end
+      
+      def sum 
+        check_type :scale
+        gsl.nil? ? nil : gsl.sum
+      end
+      def mean
+        check_type :scale
+        gsl.nil? ? nil : gsl.mean
+      end				
+      def variance_sample(m=nil)
+        check_type :scale
+        m||=mean
+        gsl.nil? ? nil : gsl.variance_m
+      end
+      
+      def standard_deviation_sample(m=nil)
+        check_type :scale
+        m||=mean
+        gsl.nil? ? nil : gsl.sd(m)
+      end
+
+      def variance_population(m=nil) # :nodoc:
+        check_type :scale    
+        m||=mean
+        gsl.nil? ? nil : gsl.variance_with_fixed_mean(m)
+      end
+      def standard_deviation_population(m=nil) # :nodoc:
+        check_type :scale
+        m||=mean
+        gsl.nil? ? nil : gsl.sd_with_fixed_mean(m)
+      end
+      def skew # :nodoc:
+        check_type :scale
+        gsl.nil? ? nil : gsl.skew
+      end
+      def kurtosis # :nodoc:
+        check_type :scale
+        gsl.nil? ? nil : gsl.kurtosis
+      end
+    end
+  end
+end

data/references.txt

@@ -7,6 +7,7 @@ References
 * Dinneen, L., & Blakesley, B. (1973). Algorithm AS 62: A Generator for the Sampling Distribution of the Mann- Whitney U Statistic. <em>Journal of the Royal Statistical Society, 22</em>(2), 269-273
 * Dziuban, C., & Shirkey E. (1974). When is a correlation matrix appropriate for factor analysis? Some decision rules. Psychological Bulletin, 81(6), 358-361.
 * Hayton, J., Allen, D. & Scarpello, V.(2004). Factor Retention Decisions in Exploratory Factor Analysis: a Tutorial on Parallel Analysis. <i>Organizational Research Methods, 7</i> (2), 191-205.
+* Härdle, W. & Simar, L. (2003). Applied Multivariate Statistical Analysis. Springer
 * Lin, J. (2007). VARIMAX_K58 [Source code]. [http://www.johnny-lin.com/idl_code/varimax_k58.pro]
 * Liu, O., & Rijmen, F. (2008). A modified procedure for parallel analysis of ordered categorical data. Behavior Research Methods, 40(2), 556-562.
 * McGraw, K. & Wong, S.P. (1996). Forming Inferences About Some Intraclass Correlation Coefficients. Psychological methods, 1(1), 30-46.
@@ -16,8 +17,7 @@ References
 * Smith, L. (2002). A tutorial on Principal Component Analysis. Available on http://courses.eas.ualberta.ca/eas570/pca_tutorial.pdf 
 * http://en.wikipedia.org/wiki/Welch-Satterthwaite_equation
 * http://europe.isixsigma.com/library/content/c080806a.asp
-* http://snippets.dzone.com/posts/show/4666
 * http://stattrek.com/Lesson6/SRS.aspx
-* http://www.cut-the-knot.org/do_you_know/AllPerm.shtml
+* http://talkstats.com/showthread.php?t=5056
 * http://www.gnu.org/software/gsl/manual/html_node/The-histogram-struct.html
 * http://www.taygeta.com/random/gaussian.html

data/test/helpers_tests.rb

@@ -1,7 +1,6 @@
 $:.unshift(File.expand_path(File.dirname(__FILE__)+'/../lib/'))
 $:.unshift(File.expand_path(File.dirname(__FILE__)+'/'))
 
-
 require 'minitest/unit'
 require 'tempfile'
 require 'tmpdir'
@@ -17,7 +16,18 @@ module MiniTest
       include Shoulda::InstanceMethods
       extend Shoulda::ClassMethods
       include Shoulda::Assertions
-
+      
+      def self.should_with_gsl(name,&block)
+        should(name) do
+          if Statsample.has_gsl?
+            instance_eval(&block)
+          else
+            skip("Requires GSL")
+          end
+         
+        end
+      end
+      
     end
   end
 
@@ -29,6 +39,12 @@ module MiniTest
         assert_in_delta(v,obs[i],delta)
       }
     end
+    def assert_equal_vector(exp,obs,delta=1e-10,msg=nil)
+      assert_equal(exp.size, obs.size, "Different size.#{msg}")
+      exp.size.times {|i|
+        assert_in_delta(exp[i],obs[i],delta, "Different element #{i}. \nExpected:\n#{exp}\nObserved:\n#{obs}.#{msg}")
+      }
+    end
     def assert_equal_matrix(exp,obs,delta=1e-10,msg=nil)
        assert_equal(exp.row_size, obs.row_size, "Different row size.#{msg}")
        assert_equal(exp.column_size, obs.column_size, "Different column size.#{msg}")

data/test/test_analysis.rb

@@ -0,0 +1,118 @@
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
+require 'mocha'
+class StatsampleAnalysisTestCase < MiniTest::Unit::TestCase
+  context(Statsample::Analysis) do
+    should "store() should create and store Statsample::Analysis::Suite" do
+      Statsample::Analysis.store(:first) do
+        a=1
+      end
+      assert(Statsample::Analysis.stored_analysis[:first])
+      assert(Statsample::Analysis.stored_analysis[:first].is_a? Statsample::Analysis::Suite)
+    end
+    should "store last created analysis" do
+      an=Statsample::Analysis.store(:first) do
+        a=1
+      end
+      assert_equal(an,Statsample::Analysis.last)
+    end
+    context(Statsample::Analysis::Suite) do
+      should "echo() uses output#puts with same arguments" do
+        an=Statsample::Analysis::Suite.new(:output)
+        obj=mock()
+        obj.expects(:puts).with(:first,:second).once
+        an.output=obj
+        an.echo(:first,:second)
+      end
+      should "summary() should call object.summary" do
+        an=Statsample::Analysis::Suite.new(:summary)
+        obj=stub('summarizable',:summary=>'summary')
+        assert_equal(obj.summary,an.summary(obj))
+      end
+      should "attach() allows to call objects on objects which respond to fields" do
+        an=Statsample::Analysis::Suite.new(:summary)
+        ds={'x'=>stub(:mean=>10),'y'=>stub(:mean=>12)}
+        ds.expects(:fields).returns(%w{x y}).at_least_once
+        an.attach(ds)
+        assert_equal(10,an.x.mean)
+        assert_equal(12,an.y.mean)
+        assert_raise(RuntimeError) {
+          an.z
+        }
+      end
+      should "attached objects should be called LIFO" do
+        an=Statsample::Analysis::Suite.new(:summary)
+        ds1={'x'=>stub(:mean=>100),'y'=>stub(:mean=>120),'z'=>stub(:mean=>13)}
+        ds1.expects(:fields).returns(%w{x y z}).at_least_once
+        ds2={'x'=>stub(:mean=>10),'y'=>stub(:mean=>12)}
+        ds2.expects(:fields).returns(%w{x y}).at_least_once
+        an.attach(ds1)
+        an.attach(ds2)
+        assert_equal(10,an.x.mean)
+        assert_equal(12,an.y.mean)        
+        assert_equal(13,an.z.mean)        
+      end
+      
+      should "detach() without arguments drop latest object" do
+        an=Statsample::Analysis::Suite.new(:summary)
+        ds1={'x'=>stub(:mean=>100),'y'=>stub(:mean=>120),'z'=>stub(:mean=>13)}
+        ds1.expects(:fields).returns(%w{x y z}).at_least_once
+        ds2={'x'=>stub(:mean=>10),'y'=>stub(:mean=>12)}        
+        ds2.expects(:fields).returns(%w{x y}).at_least_once
+        an.attach(ds1)
+        an.attach(ds2)
+        assert_equal(10,an.x.mean)
+        an.detach
+        assert_equal(100, an.x.mean)
+      end
+      should "detach() with argument drop select object" do
+        an=Statsample::Analysis::Suite.new(:summary)
+        ds1={'x'=>1}
+        ds1.expects(:fields).returns(%w{x}).at_least_once
+        ds2={'x'=>2,'y'=>3}
+        ds2.expects(:fields).returns(%w{x y}).at_least_once
+        ds3={'y'=>4}
+        ds3.expects(:fields).returns(%w{y}).at_least_once
+        
+        an.attach(ds3)
+        an.attach(ds2)
+        an.attach(ds1)
+        assert_equal(1,an.x)
+        assert_equal(3,an.y)
+        an.detach(ds2)
+        assert_equal(4,an.y)
+      end
+      should "perform a simple analysis" do
+        output=mock()
+        output.expects(:puts).with(5.5)
+        an=Statsample::Analysis.store(:simple, :output=>output) do
+          ds=data_frame(:x=>c(1..10),:y=>c(1..10))
+          attach(ds)
+          echo x.mean
+        end
+        an.run
+      end
+      should "rnorm returns a random normal distribution vector" do
+        an=Statsample::Analysis::Suite.new(:simple)
+        v=an.rnorm(1000)
+        assert_in_delta(0,v.mean,0.09)
+        assert_in_delta(1,v.sd,0.09)
+        v=an.rnorm(1000,5,10)
+        assert_in_delta(5,v.mean,0.9)
+        assert_in_delta(10,v.sd,0.9)
+      end
+    end
+    context(Statsample::Analysis::SuiteReportBuilder) do
+      should "echo() use add on rb object" do
+        an=Statsample::Analysis::SuiteReportBuilder.new(:puts_to_add)
+        an.rb.expects(:add).with(:first).twice
+        an.echo(:first, :first)
+      end
+      should "summary() uses add on rb object" do
+        an=Statsample::Analysis::SuiteReportBuilder.new(:summary_to_add)
+        an.rb.expects(:add).with(:first).once
+        an.summary(:first)
+      end
+    end
+    
+  end
+end

data/test/test_anovatwoway.rb

@@ -1,4 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 class StatsampleAnovaTwoWayTestCase < MiniTest::Unit::TestCase
   context(Statsample::Anova::TwoWay) do
     setup do

data/test/test_anovatwowaywithdataset.rb

@@ -1,4 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 # Reference:
 # * http://www.uwsp.edu/psych/Stat/13/anova-2w.htm#III
 class StatsampleAnovaTwoWayWithVectorsTestCase < MiniTest::Unit::TestCase

data/test/test_anovawithvectors.rb

@@ -1,5 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 class StatsampleAnovaOneWayWithVectorsTestCase < MiniTest::Unit::TestCase
   context(Statsample::Anova::OneWayWithVectors) do
     context("when initializing") do

data/test/test_bartlettsphericity.rb

@@ -1,5 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 
 class StatsampleBartlettSphericityTestCase < MiniTest::Unit::TestCase
   include Statsample::Test

data/test/test_bivariate.rb

@@ -1,31 +1,21 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 class StatsampleBivariateTestCase < MiniTest::Unit::TestCase
   should "method sum of squares should be correct" do
     v1=[1,2,3,4,5,6].to_vector(:scale)
     v2=[6,2,4,10,12,8].to_vector(:scale)
     assert_equal(23.0, Statsample::Bivariate.sum_of_squares(v1,v2))
   end
-  should "return same covariance with ruby and gls implementation" do
-    if Statsample.has_gsl?
-      v1=20.times.collect {|a| rand()}.to_scale
-      v2=20.times.collect {|a| rand()}.to_scale
-      assert_in_delta(Statsample::Bivariate.covariance(v1,v2), Statsample::Bivariate.covariance_slow(v1,v2), 0.001)
-    else
-      skip "Bivariate::covariance not tested (needs GSL)"
-    end
-
+  should_with_gsl "return same covariance with ruby and gls implementation" do
+    v1=20.times.collect {|a| rand()}.to_scale
+    v2=20.times.collect {|a| rand()}.to_scale
+    assert_in_delta(Statsample::Bivariate.covariance(v1,v2), Statsample::Bivariate.covariance_slow(v1,v2), 0.001)
   end
 
-  should "return same correlation with ruby and gls implementation" do
-    if Statsample.has_gsl?
-      v1=20.times.collect {|a| rand()}.to_scale
-      v2=20.times.collect {|a| rand()}.to_scale
+  should_with_gsl "return same correlation with ruby and gls implementation" do
+    v1=20.times.collect {|a| rand()}.to_scale
+    v2=20.times.collect {|a| rand()}.to_scale
 
-      assert_in_delta(GSL::Stats::correlation(v1.gsl, v2.gsl), Statsample::Bivariate.pearson_slow(v1,v2), 1e-10)
-    else
-      skip "Not tested gsl versus ruby correlation (needs GSL)"
-    end
+    assert_in_delta(GSL::Stats::correlation(v1.gsl, v2.gsl), Statsample::Bivariate.pearson_slow(v1,v2), 1e-10)
   end
   should "return correct pearson correlation" do
     v1=[6,5,4,7,8,4,3,2].to_vector(:scale)
@@ -49,7 +39,7 @@ class StatsampleBivariateTestCase < MiniTest::Unit::TestCase
     assert_in_delta(Statsample::Bivariate.prop_pearson(r.t,8,:both), r.probability, 0.001)
     assert(r.summary.size>0)
   end
-  should "return correct correlation_matrix" do
+  should "return correct correlation_matrix with nils values" do
     v1=[6,5,4,7,8,4,3,2].to_vector(:scale)
     v2=[2,3,7,8,6,4,3,2].to_vector(:scale)
     v3=[6,2,  1000,1000,5,4,7,8].to_vector(:scale)
@@ -69,6 +59,60 @@ class StatsampleBivariateTestCase < MiniTest::Unit::TestCase
     end
     #assert_equal(expected,obt)
   end
+  should_with_gsl "return same values for optimized and pairwise covariance matrix" do
+      cases=100
+      v1=Statsample::Vector.new_scale(cases) {rand()}
+      v2=Statsample::Vector.new_scale(cases) {rand()}
+      v3=Statsample::Vector.new_scale(cases) {rand()}
+      v4=Statsample::Vector.new_scale(cases) {rand()}
+      v5=Statsample::Vector.new_scale(cases) {rand()}
+
+      ds={'v1'=>v1,'v2'=>v2,'v3'=>v3,'v4'=>v4,'v5'=>v5}.to_dataset
+      
+      cor_opt=Statsample::Bivariate.covariance_matrix_optimized(ds)
+      
+      cor_pw =Statsample::Bivariate.covariance_matrix_pairwise(ds)
+      assert_equal_matrix(cor_opt,cor_pw,1e-15)
+  end
+  should_with_gsl "return same values for optimized and pairwise correlation matrix" do
+    
+    cases=100
+    v1=Statsample::Vector.new_scale(cases) {rand()}
+    v2=Statsample::Vector.new_scale(cases) {rand()}
+    v3=Statsample::Vector.new_scale(cases) {rand()}
+    v4=Statsample::Vector.new_scale(cases) {rand()}
+    v5=Statsample::Vector.new_scale(cases) {rand()}
+
+    ds={'v1'=>v1,'v2'=>v2,'v3'=>v3,'v4'=>v4,'v5'=>v5}.to_dataset
+    
+    cor_opt=Statsample::Bivariate.correlation_matrix_optimized(ds)
+    
+    cor_pw =Statsample::Bivariate.correlation_matrix_pairwise(ds)
+    assert_equal_matrix(cor_opt,cor_pw,1e-15)
+    
+  end
+  should "return correct correlation_matrix without nils values" do
+    v1=[6,5,4,7,8,4,3,2].to_vector(:scale)
+    v2=[2,3,7,8,6,4,3,2].to_vector(:scale)
+    v3=[6,2,  1000,1000,5,4,7,8].to_vector(:scale)
+    v4=[2,4,6,7,  3,7,8,6].to_vector(:scale)
+    ds={'v1'=>v1,'v2'=>v2,'v3'=>v3,'v4'=>v4}.to_dataset
+    c=Proc.new {|n1,n2|Statsample::Bivariate.pearson(n1,n2)}
+    expected=Matrix[ [c.call(v1,v1),c.call(v1,v2),c.call(v1,v3),c.call(v1,v4)], [c.call(v2,v1),c.call(v2,v2),c.call(v2,v3),c.call(v2,v4)], [c.call(v3,v1),c.call(v3,v2),c.call(v3,v3),c.call(v3,v4)],
+      [c.call(v4,v1),c.call(v4,v2),c.call(v4,v3),c.call(v4,v4)]
+    ]
+    obt=Statsample::Bivariate.correlation_matrix(ds)
+    for i in 0...expected.row_size
+      for j in 0...expected.column_size
+        #puts expected[i,j].inspect
+        #puts obt[i,j].inspect
+        assert_in_delta(expected[i,j], obt[i,j],0.0001, "#{expected[i,j].class}!=#{obt[i,j].class}  ")
+      end
+    end
+    #assert_equal(expected,obt)
+  end
+
+  
   should "return correct value for prop pearson" do
     assert_in_delta(0.42, Statsample::Bivariate.prop_pearson(Statsample::Bivariate.t_r(0.084,94), 94),0.01)
     assert_in_delta(0.65, Statsample::Bivariate.prop_pearson(Statsample::Bivariate.t_r(0.046,95), 95),0.01)
@@ -115,7 +159,5 @@ class StatsampleBivariateTestCase < MiniTest::Unit::TestCase
     assert_in_delta(0.636,Statsample::Bivariate.gamma(m),0.001)
     m2=Matrix[[15,12,6,5],[12,8,10,8],[4,6,9,10]]
     assert_in_delta(0.349,Statsample::Bivariate.gamma(m2),0.001)
-
-
   end
 end