statsample 0.18.0 → 1.0.0

data/test/test_codification.rb

@@ -1,5 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 class StatsampleCodificationTestCase < MiniTest::Unit::TestCase
 
   def initialize(*args)

data/test/test_crosstab.rb

@@ -1,5 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 class StatsampleCrosstabTestCase < MiniTest::Unit::TestCase
 
   def initialize(*args)

data/test/test_csv.rb

@@ -1,8 +1,7 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 class StatsampleCSVTestCase < MiniTest::Unit::TestCase
   def setup
-    @ds=Statsample::CSV.read(File.dirname(__FILE__)+"/test_csv.csv")
+    @ds=Statsample::CSV.read(File.dirname(__FILE__)+"/fixtures/test_csv.csv")
   end
   def test_read
     assert_equal(6,@ds.cases)
@@ -22,7 +21,7 @@ class StatsampleCSVTestCase < MiniTest::Unit::TestCase
     assert_equal(nil,@ds['age'][5])
   end
   def test_repeated
-    ds=Statsample::CSV.read(File.dirname(__FILE__)+"/../data/repeated_fields.csv")
+    ds=Statsample::CSV.read(File.dirname(__FILE__)+"/fixtures/repeated_fields.csv")
     assert_equal(%w{id name_1 age_1 city a1 name_2 age_2},ds.fields)
     age=[3,4,5,6,nil,8].to_vector(:scale)
     assert_equal(age,ds['age_2'])

data/test/test_dataset.rb

@@ -1,5 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 class StatsampleDatasetTestCase < MiniTest::Unit::TestCase
   def setup
     @ds=Statsample::Dataset.new({'id' => Statsample::Vector.new([1,2,3,4,5]), 'name'=>Statsample::Vector.new(%w{Alex Claude Peter Franz George}), 'age'=>Statsample::Vector.new([20,23,25,27,5]),
@@ -19,7 +18,15 @@ class StatsampleDatasetTestCase < MiniTest::Unit::TestCase
     a=Statsample.load(outfile.path)
     assert_equal(@ds,a)
   end
-
+  def test_gsl
+    if Statsample.has_gsl?
+      matrix=GSL::Matrix[[1,2],[3,4],[5,6]]
+      ds=Statsample::Dataset.new('v1'=>[1,3,5].to_vector,'v2'=>[2,4,6].to_vector)
+      assert_equal(matrix,ds.to_gsl)
+    else
+      skip("Gsl needed")
+    end
+  end
   def test_matrix
     matrix=Matrix[[1,2],[3,4],[5,6]]
     ds=Statsample::Dataset.new('v1'=>[1,3,5].to_vector,'v2'=>[2,4,6].to_vector)
@@ -124,6 +131,20 @@ class StatsampleDatasetTestCase < MiniTest::Unit::TestCase
     mva=[2,3,0,1,0,1].to_vector(:scale)
     assert_equal(mva,ds.vector_missing_values)
   end
+  
+  def test_has_missing_values
+    a1=[1  ,nil ,3 ,4  , 5,nil].to_vector(:scale)
+    a2=[10 ,nil ,20,20 ,20,30].to_vector(:scale)
+    b1=[nil,nil ,1 ,1  ,1 ,2].to_vector(:scale)
+    b2=[2  ,2   ,2 ,nil,2 ,3].to_vector(:scale)
+    c= [nil,2   , 4,2   ,2 ,2].to_vector(:scale)
+    ds={'a1'=>a1,'a2'=>a2,'b1'=>b1,'b2'=>b2,'c'=>c}.to_dataset
+    assert(ds.has_missing_data?)
+    clean=ds.dup_only_valid
+    assert(!clean.has_missing_data?)
+  end
+  
+  
   def test_vector_count_characters
     a1=[1  ,"abcde"  ,3  ,4  , 5,nil].to_vector(:scale)
     a2=[10 ,20.3     ,20 ,20 ,20,30].to_vector(:scale)

data/test/test_dominance_analysis.rb

@@ -1,5 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 class StatsampleDominanceAnalysisTestCase < MiniTest::Unit::TestCase
   def test_dominance_univariate
     # Example from Budescu (1993)

data/test/test_factor.rb

@@ -35,17 +35,18 @@ class StatsampleFactorTestCase < MiniTest::Unit::TestCase
   end
   def test_principalcomponents_ruby_gsl
     
-    ran=Distribution::Normal.rng_ugaussian
+    ran=Distribution::Normal.rng
     
 #    @r=::Rserve::Connection.new
 
     samples=20
-    (3..7).each {|k|
+    [3,5,7].each {|k|
       v={}
       v["x0"]=samples.times.map { ran.call()}.to_scale.centered
       (1...k).each {|i|
         v["x#{i}"]=samples.times.map {|ii| ran.call()*0.5+v["x#{i-1}"][ii]*0.5}.to_scale.centered
       }
+      
       ds=v.to_dataset
       cm=ds.covariance_matrix
 #      @r.assign('ds',ds)
@@ -82,7 +83,7 @@ class StatsampleFactorTestCase < MiniTest::Unit::TestCase
   
   end  
   def principalcomponents(gsl)
-    ran=Distribution::Normal.rng_ugaussian
+    ran=Distribution::Normal.rng
     samples=50
     x1=samples.times.map { ran.call()}.to_scale
     x2=samples.times.map {|i| ran.call()*0.5+x1[i]*0.5}.to_scale
@@ -94,9 +95,10 @@ class StatsampleFactorTestCase < MiniTest::Unit::TestCase
     assert_in_delta(1+r,pca.eigenvalues[0],1e-10)
     assert_in_delta(1-r,pca.eigenvalues[1],1e-10)
     hs=1.0 / Math.sqrt(2)
-    assert_equal_matrix(hs*Matrix[[1],[1]],pca.eigenvectors[0])
-    m_1=gsl ? Matrix[[-1],[1]] : Matrix[[1],[-1]]
-    assert_equal_matrix(hs*m_1, pca.eigenvectors[1])    
+    assert_equal_vector(Vector[1, 1]*hs, pca.eigenvectors[0])
+    m_1=gsl ? Vector[-1,1] : Vector[1,-1]
+    
+    assert_equal_vector(hs*m_1, pca.eigenvectors[1])    
     
     pcs=pca.principal_components(ds)
     exp_pc_1=ds.collect_with_index {|row,i|
@@ -134,69 +136,6 @@ class StatsampleFactorTestCase < MiniTest::Unit::TestCase
       assert_in_delta(expected[i], Statsample::Factor.kmo_univariate(m,i),0.01)
     }
   end
-  def test_parallelanalysis_with_data
-    if Statsample.has_gsl?
-      samples=100
-      variables=10
-      iterations=50
-      rng = Distribution::Normal.rng_ugaussian
-      f1=samples.times.collect {rng.call}.to_scale
-      f2=samples.times.collect {rng.call}.to_scale    
-      vectors={}
-      variables.times do |i|
-        if i<5
-          vectors["v#{i}"]=samples.times.collect {|nv|
-            f1[nv]*5+f2[nv]*2+rng.call
-          }.to_scale
-        else
-          vectors["v#{i}"]=samples.times.collect {|nv|
-            f2[nv]*5+f1[nv]*2+rng.call
-          }.to_scale
-        end
-        
-      end
-      ds=vectors.to_dataset
-      
-      pa1=Statsample::Factor::ParallelAnalysis.new(ds, :bootstrap_method=>:data, :iterations=>iterations)
-      pa2=Statsample::Factor::ParallelAnalysis.with_random_data(samples,variables,:iterations=>iterations,:percentil=>95)
-      3.times do |n|
-        var="ev_0000#{n+1}"
-        assert_in_delta(pa1.ds_eigenvalues[var].mean,pa2.ds_eigenvalues[var].mean,0.04)
-      end
-    else
-      skip("Too slow without GSL")
-    end
-    
-  end
-  def test_parallelanalysis
-    pa=Statsample::Factor::ParallelAnalysis.with_random_data(305,8,:iterations=>100,:percentil=>95)
-    assert_in_delta(1.2454, pa.ds_eigenvalues['ev_00001'].mean, 0.01)
-    assert_in_delta(1.1542, pa.ds_eigenvalues['ev_00002'].mean, 0.01)
-    assert_in_delta(1.0836, pa.ds_eigenvalues['ev_00003'].mean, 0.01)
-    #puts pa.summary
-    assert(pa.summary.size>0)
-    #pa=Statsample::Factor::ParallelAnalysis.with_random_data(305,8,100, 95, true)
-    #puts pa.summary
-  end
-  def test_map
-    #fields=%w{height arm.span forearm lower.leg weight bitro.diameter chest.girth chest.width}
-    m=Matrix[ 
-          [ 1, 0.846, 0.805, 0.859, 0.473, 0.398, 0.301, 0.382],
-          [ 0.846, 1, 0.881, 0.826, 0.376, 0.326, 0.277, 0.415],
-          [ 0.805, 0.881, 1, 0.801, 0.38, 0.319, 0.237, 0.345],
-          [ 0.859, 0.826, 0.801, 1, 0.436, 0.329, 0.327, 0.365],
-          [ 0.473, 0.376, 0.38, 0.436, 1, 0.762, 0.73, 0.629],
-          [ 0.398, 0.326, 0.319, 0.329, 0.762, 1, 0.583, 0.577],
-          [ 0.301, 0.277, 0.237, 0.327, 0.73, 0.583, 1, 0.539],
-          [ 0.382, 0.415, 0.345, 0.365, 0.629, 0.577, 0.539, 1]
-    ]
-    map=Statsample::Factor::MAP.new(m)
-    assert_in_delta(map.minfm, 0.066445,0.00001)
-    assert_equal(map.number_of_factors, 2)
-    assert_in_delta(map.fm[0], 0.312475,0.00001)
-    assert_in_delta(map.fm[1], 0.245121,0.00001)
-
-  end
   # Tested with SPSS and R
   def test_pca
       a=[2.5, 0.5, 2.2, 1.9, 3.1, 2.3, 2.0, 1.0, 1.5, 1.1].to_scale

data/test/test_factor_map.rb

@@ -0,0 +1,43 @@
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
+#require 'rserve'
+#require 'statsample/rserve_extension'
+
+class StatsampleFactorMpaTestCase < MiniTest::Unit::TestCase
+  context Statsample::Factor::MAP do
+    setup do
+      m=Matrix[ 
+            [ 1, 0.846, 0.805, 0.859, 0.473, 0.398, 0.301, 0.382],
+            [ 0.846, 1, 0.881, 0.826, 0.376, 0.326, 0.277, 0.415],
+            [ 0.805, 0.881, 1, 0.801, 0.38, 0.319, 0.237, 0.345],
+            [ 0.859, 0.826, 0.801, 1, 0.436, 0.329, 0.327, 0.365],
+            [ 0.473, 0.376, 0.38, 0.436, 1, 0.762, 0.73, 0.629],
+            [ 0.398, 0.326, 0.319, 0.329, 0.762, 1, 0.583, 0.577],
+            [ 0.301, 0.277, 0.237, 0.327, 0.73, 0.583, 1, 0.539],
+            [ 0.382, 0.415, 0.345, 0.365, 0.629, 0.577, 0.539, 1]
+      ]
+      @map=Statsample::Factor::MAP.new(m)
+    end
+    should "return correct values with pure ruby" do
+      @map.use_gsl=false
+      map_assertions(@map)
+    end
+    should_with_gsl "return correct values with gsl" do
+      require 'ruby-prof'
+
+      @map.use_gsl=true
+      map_assertions(@map)    
+    end
+    
+    
+  end
+  
+  def map_assertions(map)
+      assert_in_delta(map.minfm, 0.066445,0.00001)
+      assert_equal(map.number_of_factors, 2)
+      assert_in_delta(map.fm[0], 0.312475,0.00001)
+      assert_in_delta(map.fm[1], 0.245121,0.00001)  
+    end
+  
+  
+end
+

data/test/test_factor_pa.rb

@@ -0,0 +1,54 @@
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
+#require 'rserve'
+#require 'statsample/rserve_extension'
+
+class StatsampleFactorTestCase < MiniTest::Unit::TestCase
+  include Statsample::Fixtures
+  # Based on Hardle and Simar
+  def setup
+    @fixtures_dir=File.expand_path(File.dirname(__FILE__)+"/fixtures")
+  end
+  def test_parallelanalysis_with_data
+    if Statsample.has_gsl?
+      samples=100
+      variables=10
+      iterations=50
+      rng = Distribution::Normal.rng
+      f1=samples.times.collect {rng.call}.to_scale
+      f2=samples.times.collect {rng.call}.to_scale    
+      vectors={}
+      variables.times do |i|
+        if i<5
+          vectors["v#{i}"]=samples.times.collect {|nv|
+            f1[nv]*5+f2[nv]*2+rng.call
+          }.to_scale
+        else
+          vectors["v#{i}"]=samples.times.collect {|nv|
+            f2[nv]*5+f1[nv]*2+rng.call
+          }.to_scale
+        end
+        
+      end
+      ds=vectors.to_dataset
+      
+      pa1=Statsample::Factor::ParallelAnalysis.new(ds, :bootstrap_method=>:data, :iterations=>iterations)
+      pa2=Statsample::Factor::ParallelAnalysis.with_random_data(samples,variables,:iterations=>iterations,:percentil=>95)
+      3.times do |n|
+        var="ev_0000#{n+1}"
+        assert_in_delta(pa1.ds_eigenvalues[var].mean, pa2.ds_eigenvalues[var].mean,0.04)
+      end
+    else
+      skip("Too slow without GSL")
+    end
+    
+  end
+  def test_parallelanalysis
+    pa=Statsample::Factor::ParallelAnalysis.with_random_data(305,8,:iterations=>100,:percentil=>95)
+    assert_in_delta(1.2454, pa.ds_eigenvalues['ev_00001'].mean, 0.01)
+    assert_in_delta(1.1542, pa.ds_eigenvalues['ev_00002'].mean, 0.01)
+    assert_in_delta(1.0836, pa.ds_eigenvalues['ev_00003'].mean, 0.01)
+    assert(pa.summary.size>0)
+    #pa=Statsample::Factor::ParallelAnalysis.with_random_data(305,8,100, 95, true)
+    #puts pa.summary
+  end  
+end

data/test/test_ggobi.rb

@@ -1,4 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 require 'ostruct'
 class StatsampleGGobiTestCase < MiniTest::Unit::TestCase
 

data/test/test_gsl.rb

@@ -1,22 +1,16 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 class StatsampleGSLTestCase < MiniTest::Unit::TestCase
-  def test_matrix_to_gsl
-    if Statsample.has_gsl?
-      a=[1,2,3,4,20].to_vector(:scale)
-      b=[3,2,3,4,50].to_vector(:scale)
-      c=[6,2,3,4,3].to_vector(:scale)
-      ds={'a'=>a,'b'=>b,'c'=>c}.to_dataset
-      gsl=ds.to_matrix.to_gsl
-      assert_equal(5,gsl.size1)
-      assert_equal(3,gsl.size2)
-      matrix=gsl.to_matrix
-      assert_equal(5,matrix.row_size)
-      assert_equal(3,matrix.column_size)
-    else
-      skip("Needs GSL extension")
-    end
+  should_with_gsl "matrix with gsl" do
+    a=[1,2,3,4,20].to_vector(:scale)
+    b=[3,2,3,4,50].to_vector(:scale)
+    c=[6,2,3,4,3].to_vector(:scale)
+    ds={'a'=>a,'b'=>b,'c'=>c}.to_dataset
+    gsl=ds.to_matrix.to_gsl
+    assert_equal(5,gsl.size1)
+    assert_equal(3,gsl.size2)
+    matrix=gsl.to_matrix
+    assert_equal(5,matrix.row_size)
+    assert_equal(3,matrix.column_size)
   end
 end
 

data/test/test_histogram.rb

@@ -1,5 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 
 
 class StatsampleHistogramTestCase < MiniTest::Unit::TestCase

data/test/test_logit.rb

@@ -1,21 +1,65 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
+$:.unshift("/home/cdx/dev/rserve-client/lib")
 class StatsampleLogitTestCase < MiniTest::Unit::TestCase
-  def test_logit_1
-    crime=File.dirname(__FILE__)+'/../data/test_binomial.csv'
-    ds=Statsample::CSV.read(crime)
-    lr=Statsample::Regression::Binomial::Logit.new(ds,'y')
-    assert_in_delta(-38.8669,lr.log_likehood,0.001)
-    assert_in_delta(-5.3658,lr.constant,0.001)
+  context Statsample::Regression::Binomial::Logit do
+    should "return correct values for example" do
+      crime=File.dirname(__FILE__)+'/fixtures/test_binomial.csv'
+      ds=Statsample::CSV.read(crime)
+      lr=Statsample::Regression::Binomial::Logit.new(ds,'y')
+      assert_in_delta(-38.8669,lr.log_likehood,0.001)
+      assert_in_delta(-5.3658,lr.constant,0.001)
+  
+      exp_coeffs={"a"=>0.3270,"b"=>0.8147, "c"=>-0.4031}
+      exp_coeffs.each{|k,v|
+        assert_in_delta(v,lr.coeffs[k],0.001)
+      }
+      exp_errors={'a'=>0.4390,'b'=>0.4270,'c'=>0.3819}
+      exp_errors.each{|k,v|
+        assert_in_delta(v,lr.coeffs_se[k],0.001)
+      }
+      assert_equal(7,lr.iterations)
+      end
+    end
+    begin
+      require 'rserve'
+      require 'statsample/rserve_extension'
+      should "return same similat values to as R gml" do
+       
+        r=Rserve::Connection.new
+        ran=Distribution::Normal.rng
+        samples=100
+        a,b,c=ran.call,ran.call,ran.call
+        logit=lambda {|x| Math.exp(x) / (1+Math.exp(x))}
+        
+        x1=Statsample::Vector.new_scale(samples) {ran.call}
+        x2=Statsample::Vector.new_scale(samples) {ran.call}
+        x3=Statsample::Vector.new_scale(samples) {ran.call}
+
+        y= Statsample::Vector.new_scale(samples) {|i| logit.call(x1[i]*a+x2[i]*b+x3[i]*c+ran.call)}
+        # Generate R object
+        ds={'x1'=>x1,'x2'=>x2,'x3'=>x3,'y'=>y}.to_dataset
+        r.assign('ds',ds)
+        r.eval("mylogit<- glm(ds$y~ds$x1+ds$x2+ds$x3, family=binomial(link='logit'), na.action=na.pass)")
+        
+        r_logit=r.eval('summary(mylogit)')        
+        r_coeffs=r_logit.as_list['coefficients'].to_ruby
+        ruby_logit=Statsample::Regression::Binomial::Logit.new(ds,'y')
 
-    exp_coeffs={"a"=>0.3270,"b"=>0.8147, "c"=>-0.4031}
-    exp_coeffs.each{|k,v|
-      assert_in_delta(v,lr.coeffs[k],0.001)
-    }
-    exp_errors={'a'=>0.4390,'b'=>0.4270,'c'=>0.3819}
-    exp_errors.each{|k,v|
-      assert_in_delta(v,lr.coeffs_se[k],0.001)
-    }
-    assert_equal(7,lr.iterations)
-  end
+        assert_in_delta(r_coeffs[0,0],  ruby_logit.constant,1e-4)
+        assert_in_delta(r_coeffs[0,1],  ruby_logit.constant_se,5e-3)
+        
+        %w{x1 x2 x3}.each_with_index do |f,i|
+          assert_in_delta(r_coeffs[i+1,0], ruby_logit.coeffs[f],1e-4)
+          assert_in_delta(r_coeffs[i+1,1], ruby_logit.coeffs_se[f],5e-3)
+          
+        end
+        
+        r.close
+        
+      end
+      
+    rescue LoadError
+      puts "Require rserve extension"
+    
+  end  
 end

data/test/test_matrix.rb

@@ -1,5 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 
 class StatsampleMatrixTestCase < MiniTest::Unit::TestCase
   
@@ -24,17 +23,17 @@ class StatsampleMatrixTestCase < MiniTest::Unit::TestCase
     a=Matrix[[1.0, 0.3, 0.2], [0.3, 1.0, 0.5], [0.2, 0.5, 1.0]]
     a.extend Statsample::CovariateMatrix
     a.fields=%w{a b c}
-    assert_equal(:correlation, a.type)
+    assert_equal(:correlation, a._type)
 
     assert_equal(Matrix[[0.5],[0.3]], a.submatrix(%w{c a}, %w{b}))
     assert_equal(Matrix[[1.0, 0.2] , [0.2, 1.0]], a.submatrix(%w{c a}))
-    assert_equal(:correlation, a.submatrix(%w{c a}).type)
+    assert_equal(:correlation, a.submatrix(%w{c a})._type)
 
     a=Matrix[[20,30,10], [30,60,50], [10,50,50]]
 
     a.extend Statsample::CovariateMatrix
 
-    assert_equal(:covariance, a.type)
+    assert_equal(:covariance, a._type)
 
     a=50.times.collect {rand()}.to_scale
     b=50.times.collect {rand()}.to_scale