statsample 0.18.0 → 1.0.0

data/test/test_mle.rb

@@ -1,9 +1,8 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 class StatsampleMLETestCase < MiniTest::Unit::TestCase
   def setup
-    @file_binomial=File.dirname(__FILE__)+'/../data/test_binomial.csv'
-    @crime=File.dirname(__FILE__)+'/../data/crime.txt'
+    @file_binomial=File.dirname(__FILE__)+'/fixtures/test_binomial.csv'
+    @crime=File.dirname(__FILE__)+'/fixtures/crime.txt'
     @cases=100
     a=Array.new()
     b=Array.new()

data/test/test_regression.rb

@@ -1,4 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 
 class StatsampleRegressionTestCase < MiniTest::Unit::TestCase
   context "Example with missing data" do
@@ -21,6 +21,22 @@ class StatsampleRegressionTestCase < MiniTest::Unit::TestCase
       assert_in_delta(0.064, @lr.constant_se,0.001,"constant se")
   end
   end
+  should "return an error if data is linearly dependent" do 
+    samples=100
+    
+    a,b=rand,rand
+    
+    x1=samples.times.map { rand}.to_scale
+    x2=samples.times.map {rand}.to_scale
+    x3=samples.times.map {|i| x1[i]*(1+a)+x2[i]*(1+b)}.to_scale
+    y=samples.times.map {|i| x1[i]+x2[i]+x3[i]+rand}.to_scale
+
+    ds={'x1'=>x1,'x2'=>x2,'x3'=>x3,'y'=>y}.to_dataset
+
+    assert_raise(Statsample::Regression::LinearDependency) {
+        Statsample::Regression::Multiple::RubyEngine.new(ds,'y')
+    }
+  end
   def test_parameters
     @x=[13,20,10,33,15].to_vector(:scale)
     @y=[23,18,35,10,27	].to_vector(:scale)
@@ -34,10 +50,13 @@ class StatsampleRegressionTestCase < MiniTest::Unit::TestCase
 
   end
   def _test_simple_regression(reg)
+  
     assert_in_delta(40.009, reg.a,0.001)
     assert_in_delta(-0.957, reg.b,0.001)
     assert_in_delta(4.248,reg.standard_error,0.002)
+    assert(reg.summary)
   end
+  
   def test_summaries
     a=10.times.map{rand(100)}.to_scale
     b=10.times.map{rand(100)}.to_scale
@@ -166,8 +185,8 @@ class StatsampleRegressionTestCase < MiniTest::Unit::TestCase
     @c=[11,22,30,40,50,65,78,79,99,100].to_vector(:scale)
     @y=[3,4,5,6,7,8,9,10,20,30].to_vector(:scale)
     ds={'a'=>@a,'b'=>@b,'c'=>@c,'y'=>@y}.to_dataset
-
     cor=Statsample::Bivariate.correlation_matrix(ds)
+    
     lr=Statsample::Regression::Multiple::MatrixEngine.new(cor,'y', :y_mean=>@y.mean, :x_mean=>{'a'=>ds['a'].mean, 'b'=>ds['b'].mean, 'c'=>ds['c'].mean}, :cases=>@a.size, :y_sd=>@y.sd , :x_sd=>{'a' => @a.sd, 'b' => @b.sd, 'c' => @c.sd})
     assert_nil(lr.constant_se)
     assert_nil(lr.constant_t)

data/test/test_reliability.rb

@@ -1,4 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 class StatsampleReliabilityTestCase < MiniTest::Unit::TestCase
   context Statsample::Reliability do
     should "return correct r according to Spearman-Brown prophecy" do
@@ -197,8 +197,8 @@ class StatsampleReliabilityTestCase < MiniTest::Unit::TestCase
         assert_in_delta(var_mean, @ia.variances_mean)
         assert_equal(@x1.mean, @ia.item_statistics['x1'][:mean])
         assert_equal(@x4.mean, @ia.item_statistics['x4'][:mean])
-        assert_equal(@x1.sds, @ia.item_statistics['x1'][:sds])
-        assert_equal(@x4.sds, @ia.item_statistics['x4'][:sds])
+        assert_in_delta(@x1.sds, @ia.item_statistics['x1'][:sds],1e-14)
+        assert_in_delta(@x4.sds, @ia.item_statistics['x4'][:sds],1e-14)
         ds2=@ds.clone
         ds2.delete_vector('x1')
         vector_sum=ds2.vector_sum

data/test/test_reliability_icc.rb

@@ -1,5 +1,5 @@
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 $reliability_icc=nil
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
 class StatsampleReliabilityIccTestCase < MiniTest::Unit::TestCase
   context Statsample::Reliability::ICC do
     setup do

data/test/test_reliability_skillscale.rb

@@ -1,5 +1,6 @@
-$reliability_icc=nil
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
+
+
 class StatsampleReliabilitySkillScaleTestCase < MiniTest::Unit::TestCase
   context Statsample::Reliability::SkillScaleAnalysis do
     setup do
@@ -10,9 +11,12 @@ class StatsampleReliabilitySkillScaleTestCase < MiniTest::Unit::TestCase
       @b=cases.times.map {options[rand(5)]}.to_vector
       @c=cases.times.map {options[rand(5)]}.to_vector
       @d=cases.times.map {options[rand(5)]}.to_vector
-      @e=cases.times.map {rand()>0.8 ? nil : options[rand(5)]}.to_vector
+      @e=cases.times.map {|i|
+        i==0 ? options[rand(0)] : 
+          rand()>0.8 ? nil : options[rand(5)]
+      }.to_vector
       @ds={'id'=>@id,'a'=>@a,'b'=>@b,'c'=>@c,'d'=>@d,'e'=>@e}.to_dataset
-      @key={'a'=>options[rand(5)], 'b'=>options[rand(5)], 'c'=>options[rand(5)], 'd'=>options[rand(5)],'e'=>options[rand(5)]}
+      @key={'a'=>"a", 'b'=>options[rand(5)], 'c'=>options[rand(5)], 'd'=>options[rand(5)],'e'=>options[rand(5)]}
       @ssa=Statsample::Reliability::SkillScaleAnalysis.new(@ds, @key)
       @ac=@a.map {|v| v==@key['a'] ? 1 : 0}.to_scale
       @bc=@b.map {|v| v==@key['b'] ? 1 : 0}.to_scale
@@ -34,6 +38,18 @@ class StatsampleReliabilitySkillScaleTestCase < MiniTest::Unit::TestCase
       assert_equal(cdsm.vector_sum, @ssa.vector_sum)
       assert_equal(cdsm.vector_mean, @ssa.vector_mean)
     end
+    should "not crash on rare case" do
+      a=Statsample::Vector["c","c","a","a","c","a","b","c","c","b","a","d","a","d","a","a","d","e","c","d"]
+      b=Statsample::Vector["e","b","e","b","c","d","a","e","e","c","b","e","e","b","d","c","e","b","b","d"]
+      c=Statsample::Vector["e","b","e","c","e","c","b","d","e","c","a","a","b","d","e","c","b","a","a","e"]
+      d=Statsample::Vector["a","b","d","d","e","b","e","b","d","c","e","a","c","d","c","c","e","d","d","b"]
+      e=Statsample::Vector["a","b",nil,"d","c","c","d",nil,"d","d","e","e",nil,nil,nil,"d","c",nil,"e","d"]
+      key={"a"=>"a", "b"=>"e", "c"=>"d", "d"=>"c", "e"=>"d"}
+      ds=Statsample::Dataset.new("a"=>a,"b"=>b,"c"=>c,"d"=>d,"e"=>e)
+      ssa=Statsample::Reliability::SkillScaleAnalysis.new(ds, key)
+      assert(ssa.summary)
+    end
+    
     should "return valid summary" do
       assert(@ssa.summary.size>0)
     end

data/test/test_resample.rb

@@ -1,5 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 
 class StatsampleResampleTestCase < MiniTest::Unit::TestCase
   def initialize(*args)

data/test/test_rserve_extension.rb

@@ -1,5 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 begin
   require 'rserve'
   require 'statsample/rserve_extension'

data/test/test_srs.rb

@@ -1,5 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 
 class StatsampleSrsTestCase < MiniTest::Unit::TestCase
   def test_std_error

data/test/test_statistics.rb

@@ -1,5 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 class StatsampleStatisicsTestCase < MiniTest::Unit::TestCase
 
   def initialize(*args)

data/test/test_stest.rb

@@ -1,5 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 
 class StatsampleTestTestCase < MiniTest::Unit::TestCase
   def test_chi_square_matrix_with_expected

data/test/test_stratified.rb

@@ -1,5 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 
 class StatsampleStratifiedTestCase < MiniTest::Unit::TestCase
 

data/test/test_test_f.rb

@@ -1,5 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 class StatsampleTestFTestCase < MiniTest::Unit::TestCase
   context(Statsample::Test::F) do 
     setup do

data/test/test_test_t.rb

@@ -1,5 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 class StatsampleTestTTestCase < MiniTest::Unit::TestCase
   include Statsample::Test
   include Math

data/test/test_umannwhitney.rb

@@ -1,5 +1,4 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 
 class StatsampleUMannWhitneyTestCase < MiniTest::Unit::TestCase
   include Statsample::Test

data/test/test_vector.rb

@@ -51,8 +51,7 @@ class StatsampleTestVector < MiniTest::Unit::TestCase
       assert_equal(av,av2)
       assert_same(av,av2)
       assert_same(bv,bv2)
-    end
-    
+    end 
   end
   context Statsample::Vector do
     setup do 
@@ -60,6 +59,14 @@ class StatsampleTestVector < MiniTest::Unit::TestCase
       @c.name="Test Vector"
       @c.missing_values=[-99]
     end
+    should_with_gsl "be created with GSL::Vector" do
+      gsl=GSL::Vector[1,2,3,4,5]
+      v=Statsample::Vector.new(gsl)
+      assert_equal([1,2,3,4,5], v.to_a)
+      refute(v.flawed?)
+      
+    end
+
     context "using matrix operations" do
       setup do
         @a=[1,2,3,4,5].to_scale
@@ -84,6 +91,20 @@ class StatsampleTestVector < MiniTest::Unit::TestCase
         @data=(10.times.map{rand(100)})+[nil]
         @original=Statsample::Vector.new(@data, :scale)
       end
+      should "be the sample using []" do
+        second=Statsample::Vector[*@data]
+        assert_equal(@original, second)
+      end
+      should "[] returns same results as R-c()" do
+        reference=[0,4,5,6,10].to_scale
+        assert_equal(reference, Statsample::Vector[0,4,5,6,10])
+        assert_equal(reference, Statsample::Vector[0,4..6,10])
+        assert_equal(reference, Statsample::Vector[[0],[4,5,6],[10]])
+        assert_equal(reference, Statsample::Vector[[0],[4,[5,[6]]],[10]])
+        
+        assert_equal(reference, Statsample::Vector[[0],[4,5,6].to_vector,[10]])
+        
+      end
       should "be the same usign #to_vector" do
         lazy1=@data.to_vector(:scale)
         assert_equal(@original,lazy1)
@@ -94,6 +115,24 @@ class StatsampleTestVector < MiniTest::Unit::TestCase
         assert_equal(:scale,lazy2.type)
         assert_equal(@data.find_all{|v| !v.nil?},lazy2.valid_data)
       end
+      should "could use new_scale with size only" do
+        v1=10.times.map {nil}.to_scale
+        v2=Statsample::Vector.new_scale(10)
+        assert_equal(v1,v2)
+        
+      end
+      should "could use new_scale with size and value" do
+        a=rand
+        v1=10.times.map {a}.to_scale
+        v2=Statsample::Vector.new_scale(10,a)
+        assert_equal(v1,v2)
+      end
+      should "could use new_scale with func" do
+        v1=10.times.map {|i| i*2}.to_scale
+        v2=Statsample::Vector.new_scale(10) {|i| i*2}
+        assert_equal(v1,v2)
+      end
+      
     end
     
     context "#split_by_separator" do
@@ -207,23 +246,53 @@ class StatsampleTestVector < MiniTest::Unit::TestCase
     should "have a summary with name on it" do
       assert_match(/#{@c.name}/, @c.summary)
     end
+    
+    should "GSL::Vector based should push correcty" do
+      if Statsample.has_gsl?
+        v=GSL::Vector[1,2,3,4,5].to_scale
+        v.push(nil)
+        assert_equal([1,2,3,4,5,nil], v.to_a)
+        assert(v.flawed?)
+      else
+        skip("Requires GSL")
+      end
+    end
+
+    
     should "split correctly" do
       a = Statsample::Vector.new(["a","a,b","c,d","a,d","d",10,nil],:nominal)
       assert_equal([%w{a},%w{a b},%w{c d},%w{a d},%w{d},[10],nil], a.splitted)      
     end
-  end
-
-
-  def test_types
-    @c.type=:nominal
-    assert_raise NoMethodError do
-      @c.median
+    should "multiply correct for scalar" do
+      a = [1,2,3].to_scale
+      assert_equal([5,10,15].to_scale, a*5)
     end
-    @c.type=:ordinal
-    assert_raise NoMethodError do
-      @c.mean
+    should "multiply correct with other vector" do
+      a = [1,2,3].to_scale
+      b = [2,4,6].to_scale
+      
+      assert_equal([2,8,18].to_scale, a*b)
+    end
+    should "sum correct for scalar" do
+      a = [1,2,3].to_scale
+      assert_equal([11,12,13].to_scale, a+10)
+    end
+    
+    should "raise NoMethodError when method requires ordinal and vector is nominal" do
+      @c.type=:nominal
+      assert_raise(::NoMethodError) { @c.median }
+    end
+    
+    should "raise NoMethodError when method requires scalar and vector is ordinal" do      
+      @c.type=:ordinal
+      assert_raise(::NoMethodError) { @c.mean }
     end
+
+    
   end
+
+
+
   def test_nominal
     assert_equal(@c[1],5)
     assert_equal({ 1=>1,2=>1,3=>1,4=>1,5=>5,6=>2,7=>1,8=>1, 9=>1,10=>1},@c.frequencies)
@@ -308,12 +377,41 @@ class StatsampleTestVector < MiniTest::Unit::TestCase
     assert_equal(0,vs.mean)
     assert_equal(1,vs.sds)
   end
+  
   def test_vector_standarized_with_zero_variance
     v1=100.times.map {|i| 1}.to_scale
     exp=100.times.map {nil}.to_scale
     assert_equal(exp,v1.standarized)
   end
 
+    def test_check_type
+    v=Statsample::Vector.new
+    v.type=:nominal
+    assert_raise(NoMethodError) { v.check_type(:scale)}
+    assert_raise(NoMethodError) { v.check_type(:ordinal)}
+    assert(v.check_type(:nominal).nil?)
+    
+    v.type=:ordinal
+    
+    assert_raise(NoMethodError) { v.check_type(:scale)}
+    
+    assert(v.check_type(:ordinal).nil?)
+    assert(v.check_type(:nominal).nil?)
+    
+
+    v.type=:scale
+    assert(v.check_type(:scale).nil?)
+    assert(v.check_type(:ordinal).nil?)
+    assert(v.check_type(:nominal).nil?)
+
+    v.type=:date
+    assert_raise(NoMethodError) { v.check_type(:scale)}
+    assert_raise(NoMethodError) { v.check_type(:ordinal)}
+    assert_raise(NoMethodError) { v.check_type(:nominal)}
+    
+  end
+  
+  
   def test_add
     a=Statsample::Vector.new([1,2,3,4,5], :scale)
     b=Statsample::Vector.new([11,12,13,14,15], :scale)
@@ -391,11 +489,12 @@ class StatsampleTestVector < MiniTest::Unit::TestCase
   def test_gsl
     if Statsample.has_gsl?
       a=Statsample::Vector.new([1,2,3,4,"STRING"], :scale)
+      
       assert_equal(2,a.mean)
-      assert_equal(a.variance_sample_slow,a.variance_sample)
-      assert_equal(a.standard_deviation_sample_slow,a.sds)
-      assert_equal(a.variance_population_slow,a.variance_population)
-      assert_equal(a.standard_deviation_population_slow,a.standard_deviation_population)
+      assert_equal(a.variance_sample_ruby,a.variance_sample)
+      assert_equal(a.standard_deviation_sample_ruby,a.sds)
+      assert_equal(a.variance_population_ruby,a.variance_population)
+      assert_equal(a.standard_deviation_population_ruby,a.standard_deviation_population)
       assert_nothing_raised do
         a=[].to_vector(:scale)
       end
@@ -408,8 +507,8 @@ class StatsampleTestVector < MiniTest::Unit::TestCase
       assert_equal(3.5, b.mean)
       assert_equal(6,b.gsl.size)
       c=[10,20,30,40,50,100,1000,2000,5000].to_scale
-      assert_in_delta(c.skew,     c.skew_slow     ,0.0001)
-      assert_in_delta(c.kurtosis, c.kurtosis_slow ,0.0001)
+      assert_in_delta(c.skew,     c.skew_ruby     ,0.0001)
+      assert_in_delta(c.kurtosis, c.kurtosis_ruby ,0.0001)
     end
   end
   def test_vector_matrix

data/test/test_xls.rb

@@ -1,9 +1,8 @@
-require(File.dirname(__FILE__)+'/helpers_tests.rb')
-
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_tests.rb'))
 class StatsampleExcelTestCase < MiniTest::Unit::TestCase
   context "Excel reader" do
     setup do 
-      @ds=Statsample::Excel.read(File.dirname(__FILE__)+"/test_xls.xls")
+      @ds=Statsample::Excel.read(File.dirname(__FILE__)+"/fixtures/test_xls.xls")
     end
     should "set the number of cases" do
       assert_equal(6,@ds.cases)

data/web/Rakefile

@@ -0,0 +1,39 @@
+# -*- ruby -*-
+require 'rake'
+require 'fileutils'
+directory "examples"
+
+def get_base(f)
+  f.sub(File.dirname(__FILE__)+"/../examples/","").gsub("/","_").gsub(".rb","")
+end
+
+
+EXAMPLES=Dir.glob(File.dirname(__FILE__)+"/../examples/**/*.rb").map {|v|     [v, get_base(v)]
+}.find_all{|v| !v[0].include?"_data"}
+
+EXAMPLES_BASE=EXAMPLES.map {|v| v[1]}
+
+
+desc "Build all html, rtf and pdf files"
+task :build_site do
+  ruby "build_site.rb"
+end
+
+
+task :clean do
+  Dir.glob(File.dirname(__FILE__)+"/examples/*.pdf").each do |t|
+    FileUtils.rm t
+  end
+  Dir.glob(File.dirname(__FILE__)+"/examples/*.html").each do |t|
+    FileUtils.rm t
+  end
+  Dir.glob(File.dirname(__FILE__)+"/examples/*.rtf").each do |t|
+    FileUtils.rm t
+  end
+  Dir.glob(File.dirname(__FILE__)+"/examples/images/*.*").each do |t|
+    FileUtils.rm t
+  end
+end
+
+
+load 'upload_task.rb' if File.exists? "upload_task.rb"