statsample 0.3.4 → 0.4.0

data/test/test_anova.rb

@@ -21,12 +21,7 @@ class StatsampleAnovaTestCase < Test::Unit::TestCase
         assert_in_delta(23.568,@anova.f,0.001)
         anova2=Statsample::Anova::OneWay.new([@v1,@v1,@v1,@v1,@v2])
         assert_in_delta(3.960, anova2.f,0.001)
-
-	if HAS_GSL
 		assert(@anova.significance<0.01)
 		assert_in_delta(0.016, anova2.significance,0.001)
-	else
-		puts "Skipped OneWay#significance (no GSL)"
-	end
     end
 end

data/test/test_dataset.rb

@@ -294,6 +294,18 @@ class StatsampleDatasetTestCase < Test::Unit::TestCase
         assert_equal(exp1,res)
         res=ds.verify('id',t1,t2,t3)
         assert_equal(exp2,res)
-
+    end
+    def test_compute_operation
+        v1=[1,2,3,4].to_vector(:scale)
+        v2=[4,3,2,1].to_vector(:scale)
+        v3=[10,20,30,40].to_vector(:scale)
+        vscale=[1.quo(2),1,3.quo(2),2].to_vector(:scale)
+        vsum=[1+4+10.0,2+3+20.0,3+2+30.0,4+1+40.0].to_vector(:scale)
+        vmult=[1*4,2*3,3*2,4*1].to_vector(:scale)
+        ds={'v1'=>v1,'v2'=>v2,'v3'=>v3}.to_dataset
+        assert_equal(vscale,ds.compute("v1/2"))        
+        assert_equal(vsum,ds.compute("v1+v2+v3"))
+        assert_equal(vmult,ds.compute("v1*v2"))
+        
     end
 end

data/test/test_distribution.rb

@@ -0,0 +1,57 @@
+$:.unshift(File.dirname(__FILE__)+'/../lib/')
+require 'distribution'
+require 'test/unit'
+begin
+    require 'rbgsl'
+    NOT_GSL=false
+rescue
+    NOT_GSL=true
+end
+class DistributionTestCase < Test::Unit::TestCase
+    def test_chi
+        if !NOT_GSL
+        [2,3,4,5].each{|k|
+            chis=rand()*10
+            area=Distribution::ChiSquare.cdf(chis, k)
+            assert_in_delta(area, GSL::Cdf.chisq_P(chis,k),0.0001)
+            assert_in_delta(chis, Distribution::ChiSquare.p_value(area,k),0.0001,"Error on prob #{area} and k #{k}")
+        }
+        end
+    end
+    def test_t
+        if !NOT_GSL
+            [-2,0.1,0.5,1,2].each{|t|
+                [2,5,10].each{|n|
+                    area=Distribution::T.cdf(t,n)
+                    assert_in_delta(area, GSL::Cdf.tdist_P(t,n),0.0001)
+                    assert_in_delta(Distribution::T.p_value(area,n), GSL::Cdf.tdist_Pinv(area,n),0.0001)
+                    
+                }
+            }
+        end
+    end
+    def test_normal
+        if !NOT_GSL
+            [-2,0.1,0.5,1,2].each{|x|
+                    area=Distribution::Normal.cdf(x)
+                    assert_in_delta(area, GSL::Cdf.ugaussian_P(x),0.0001)
+                    assert_in_delta(Distribution::Normal.p_value(area), GSL::Cdf.ugaussian_Pinv(area),0.0001)
+            }
+        end
+    end
+    def test_f
+        if !NOT_GSL
+            [0.1,0.5,1,2,10,20,30].each{|f|
+                [2,5,10].each{|n2|
+                [2,5,10].each{|n1|
+                    area=Distribution::F.cdf(f,n1,n2)
+                    assert_in_delta(area, GSL::Cdf.fdist_P(f,n1,n2),0.0001)
+                    assert_in_delta(Distribution::F.p_value(area,n1,n2), GSL::Cdf.fdist_Pinv(area,n1,n2),0.0001)
+                    
+                }
+                }
+            }
+        end
+    end
+
+end

data/test/test_gsl.rb

@@ -0,0 +1,22 @@
+$:.unshift(File.dirname(__FILE__)+'/../lib/')
+require 'statsample'
+require 'test/unit'
+require 'matrix_extension'
+class StatsampleGSLTestCase < Test::Unit::TestCase
+    def test_matrix_to_gsl
+        if HAS_GSL
+        a=[1,2,3,4,20].to_vector(:scale)
+        b=[3,2,3,4,50].to_vector(:scale)
+        c=[6,2,3,4,3].to_vector(:scale)
+        ds={'a'=>a,'b'=>b,'c'=>c}.to_dataset
+        gsl=ds.to_matrix.to_gsl
+        assert_equal(5,gsl.size1)
+        assert_equal(3,gsl.size2)
+        matrix=gsl.to_matrix
+        assert_equal(5,matrix.row_size)
+        assert_equal(3,matrix.column_size)
+        end
+    end
+end
+
+

data/test/test_logit.rb

@@ -0,0 +1,22 @@
+$:.unshift(File.dirname(__FILE__)+'/../lib/')
+require 'statsample'
+require 'test/unit'
+class StatsampleLogitTestCase < Test::Unit::TestCase
+    def test_logit_1
+        crime=File.dirname(__FILE__)+'/../data/test_binomial.csv'
+        ds=Statsample::CSV.read(crime)
+        lr=Statsample::Regression::Binomial::Logit.new(ds,'y')
+        assert_in_delta(-38.8669,lr.log_likehood,0.001)
+        assert_in_delta(-5.3658,lr.constant,0.001)
+        
+        exp_coeffs={"a"=>0.3270,"b"=>0.8147, "c"=>-0.4031}
+        exp_coeffs.each{|k,v|
+            assert_in_delta(v,lr.coeffs[k],0.001)
+        }
+        exp_errors={'a'=>0.4390,'b'=>0.4270,'c'=>0.3819}
+        exp_errors.each{|k,v|
+            assert_in_delta(v,lr.coeffs_se[k],0.001)
+        }
+        assert_equal(7,lr.iterations)               
+    end
+end

data/test/test_mle.rb

@@ -0,0 +1,140 @@
+$:.unshift(File.dirname(__FILE__)+'/../lib/')
+require 'statsample'
+require 'test/unit'
+class StatsampleMLETestCase < Test::Unit::TestCase
+    def setup
+        @file_binomial=File.dirname(__FILE__)+'/../data/test_binomial.csv'
+        @crime=File.dirname(__FILE__)+'/../data/crime.txt'
+        @cases=100
+        a=Array.new()
+        b=Array.new()
+        c=Array.new()
+        y=Array.new()
+
+        @cases.times{|i|
+            a.push(2*rand()-i)
+            b.push(2*rand()-5+i)
+            c.push(2*rand()+i)
+            y_val=i+(rand()*@cases.quo(2) - @cases.quo(4))
+            y.push(y_val<(@cases/2.0) ? 0.0 : 1.0)
+        }
+        a=a.to_vector(:scale)
+        b=b.to_vector(:scale)
+        c=c.to_vector(:scale)
+        y=y.to_vector(:scale)
+        
+        @ds_indep={'a'=>a,'b'=>b,'c'=>c}.to_dataset
+        constant=([1.0]*@cases).to_vector(:scale)
+        @ds_indep_2={'constant'=>constant,'a'=>a,'b'=>b,'c'=>c}.to_dataset
+        @ds_indep_2.fields=%w{constant a b c}
+        @mat_x=@ds_indep_2.to_matrix
+        @mat_y=y.to_matrix(:vertical)
+        @ds=@ds_indep.dup
+        @ds.add_vector('y',y)
+    end
+    def test_normal
+        y=Array.new()
+        y=@ds_indep.collect{|row|
+            row['a']*5+row['b']+row['c']+rand()*3
+        }
+        constant=([1]*@cases).to_vector(:scale)
+        ds_indep_2=@ds_indep.dup
+        ds_indep_2['constant']=constant
+        ds_indep_2.fields=%w{constant a b c}
+        mat_x=ds_indep_2.to_matrix
+        mat_y=y.to_matrix(:vertical)
+        mle=Statsample::MLE::Normal.new()
+        mle.verbose=false
+        coeffs_nr=mle.newton_raphson(mat_x,mat_y)
+        #p coeffs_nr
+        ds=@ds_indep.dup
+        ds.add_vector('y',y)
+        lr=Statsample::Regression::Multiple.listwise(ds,'y')
+        lr_constant = lr.constant
+        lr_coeffs   = lr.coeffs
+        assert_in_delta(coeffs_nr[0,0], lr_constant,0.0000001)
+        assert_in_delta(coeffs_nr[1,0], lr_coeffs["a"],0.0000001)
+        assert_in_delta(coeffs_nr[2,0], lr_coeffs["b"],0.0000001)
+        assert_in_delta(coeffs_nr[3,0], lr_coeffs["c"],0.0000001)
+    end
+    
+    def test_probit
+        ds=Statsample::CSV.read(@file_binomial)
+        constant=([1.0]*ds.cases).to_vector(:scale)
+        ds_indep={'constant'=>constant, 'a'=>ds['a'],'b'=>ds['b'], 'c'=>ds['c']}.to_dataset(%w{constant a b c})
+        mat_x=ds_indep.to_matrix
+        mat_y=ds['y'].to_matrix(:vertical)
+        mle=Statsample::MLE::Probit.new
+        b_probit=mle.newton_raphson(mat_x,mat_y)
+        ll=mle.log_likehood(mat_x,mat_y,b_probit)
+        
+        b_exp=[-3.0670,0.1763,0.4483,-0.2240]
+        b_exp.each_index{|i|
+            assert_in_delta(b_exp[i], b_probit[i,0], 0.001)
+        }
+        assert_in_delta(-38.31559,ll,0.0001)
+    end
+    def test_logit_crime
+        ds=Statsample::PlainText.read(@crime, %w{crimerat maleteen south educ police60 police59 labor  males pop nonwhite unemp1  unemp2 median belowmed})
+        constant=([1.0]*ds.cases).to_vector(:scale)
+        ds2=ds.dup(%w{maleteen south educ police59})
+        ds2['constant']=constant
+        ds2.fields=%w{constant maleteen south educ police59}
+        mat_x=ds2.to_matrix
+        mat_y=(ds.compute "(crimerat>=110) ? 1:0").to_matrix(:vertical)
+        mle=Statsample::MLE::Logit.new
+        b=mle.newton_raphson(mat_x,mat_y)
+        ll=mle.log_likehood(mat_x,mat_y,b)
+        assert_in_delta(-18.606959,ll,0.001)
+        exp=[-17.701, 0.0833,-1.117, 0.0229, 0.0581]
+        exp.each_index{|i|
+            assert_in_delta(exp[i],b[i,0],0.001)
+        }
+        assert_equal(5,mle.iterations)        
+    end
+    def atest_logit_alglib
+        if(HAS_ALGIB)
+            ds=Statsample::CSV.read(@file_binomial)
+            constant=([1.0]*ds.cases).to_vector(:scale)
+            
+            ds_indep={'constant'=>constant, 'a'=>ds['a'],'b'=>ds['b'], 'c'=>ds['c']}.to_dataset(%w{constant a b c} )
+            
+            mat_x=ds_indep.to_matrix
+            mat_y=ds['y'].to_matrix(:vertical)
+            log=Alglib::Logit.build_from_matrix(ds.to_matrix)
+            coeffs=log.unpack[0]
+            b_alglib=Matrix.columns([[-coeffs[3], -coeffs[0], -coeffs[1], -coeffs[2]]])
+            mle=Statsample::MLE::Logit.new
+            ll_alglib=mle.log_likehood(mat_x,mat_y,b_alglib)
+            b_newton=mle.newton_raphson(mat_x,mat_y)
+            ll_pure_ruby=mle.log_likehood(mat_x,mat_y,b_newton)
+            #p b_alglib
+            #p b_newton
+            
+            assert_in_delta(ll_alglib,ll_pure_ruby,1)
+        end
+
+    end
+    def atest_logit1
+        log=Alglib::Logit.build_from_matrix(@ds.to_matrix)
+        coeffs=log.unpack[0]
+        b=Matrix.columns([[-coeffs[3],-coeffs[0],-coeffs[1],-coeffs[2]]])
+#        puts "Coeficientes beta alglib:"
+        #p b
+        mle_alglib=Statsample::MLE::ln_mle(Statsample::MLE::Logit, @mat_x,@mat_y,b)
+#       puts "MLE Alglib:"
+        #p mle_alglib
+#        Statsample::CSV.write(ds,"test_binomial.csv")
+
+
+
+#        puts "iniciando newton"
+        coeffs_nr=Statsample::MLE.newton_raphson(@mat_x,@mat_y, Statsample::MLE::Logit)
+        #p coeffs_nr
+        mle_pure_ruby=Statsample::MLE::ln_mle(Statsample::MLE::Logit, @mat_x,@mat_y,coeffs_nr)
+        #p mle_pure_ruby
+
+        #puts "Malo: #{mle_malo} Bueno: #{mle_bueno} : #{mle_malo-mle_bueno}"
+    end
+end
+

data/test/test_r.rb

@@ -0,0 +1,9 @@
+$:.unshift(File.dirname(__FILE__)+'/../lib/')
+require 'statsample'
+require 'test/unit'
+
+class StatsampleRTestCase < Test::Unit::TestCase
+    def test_basic
+        # Nothing yet
+    end
+end

data/test/test_regression.rb

@@ -95,11 +95,7 @@ class StatsampleRegressionTestCase < Test::Unit::TestCase
         assert_in_delta(0.955,lr.r,0.001)
         assert_in_delta(0.913,lr.r2,0.001)
         assert_in_delta(20.908, lr.f,0.001)
-	if HAS_GSL
         assert_in_delta(0.001, lr.significance, 0.001)
-	else
-		puts "#{lr.class}#significance not tested (not GSL)"
-	end
         assert_in_delta(0.226,lr.tolerance("a"),0.001)
         coeffs_se={"a"=>1.171,"b"=>1.129,"c"=>0.072}
         ccoeffs_se=lr.coeffs_se
@@ -133,4 +129,16 @@ class StatsampleRegressionTestCase < Test::Unit::TestCase
             assert_in_delta(residuals[i],c_residuals[i],0.001)
         }
     end
+    def test_ds_by_exp
+        @a=  [1,3,2,4,3,5,4,6,5,7].to_vector(:scale)
+        @b=  [3,3,4,4,5,5,6,6,4,4].to_vector(:scale)
+        @c=  [11,22,30,40,50,65,78,79,99,100].to_vector(:scale)
+        @d=%w{a  b  c  a  a  c  a   a c  a}.to_vector(:nominal)
+        @y=[3,4,5,6,7,8,9,10,20,30].to_vector(:scale)
+        ds={'a'=>@a,'b'=>@b,'c'=>@c,'d'=>@d,'y'=>@y}.to_dataset
+        #puts Statsample::Regression::Multiple.ds_by_exp(ds,"a+b=y")
+        #puts Statsample::Regression::Multiple.ds_by_exp(ds,"a+b*d=y")
+        #puts Statsample::Regression::Multiple.ds_by_exp(ds,"c+d=y")
+    end
+
 end

data/test/test_srs.rb

@@ -4,12 +4,8 @@ require 'test/unit'
 
 class StatsampleSrsTestCase < Test::Unit::TestCase
     def test_std_error
-	if HAS_GSL
         assert_equal(384,Statsample::SRS.estimation_n0(0.05,0.5,0.95).to_i)
         assert_equal(108,Statsample::SRS.estimation_n(0.05,0.5,150,0.95).to_i)
-	else
-		puts "Statsample::SRS.estimation_n0 not tested (needs ruby-gsl)"
-	end
         assert_in_delta(0.0289,Statsample::SRS.proportion_sd_kp_wor(0.5,100,150),0.001)
     end
 end

data/test/test_stata.rb

@@ -0,0 +1,11 @@
+$:.unshift(File.dirname(__FILE__)+'/../lib/')
+#require 'stata'
+require 'statsample'
+require 'test/unit'
+
+class StatsampleStataTestCase < Test::Unit::TestCase
+    def test_basic
+        #datafile=File.dirname(__FILE__)+"/../data/logit.dta"
+        #p Stata.open(datafile)
+    end
+end

data/test/test_statistics.rb

@@ -64,7 +64,6 @@ class StatsampleStatisicsTestCase < Test::Unit::TestCase
 #assert_equal(expected,obt)
     end
     def test_prop_pearson
-	if HAS_GSL    
         assert_in_delta(0.42, Statsample::Bivariate.prop_pearson(Statsample::Bivariate.t_r(0.084,94), 94),0.01)
         assert_in_delta(0.65, Statsample::Bivariate.prop_pearson(Statsample::Bivariate.t_r(0.046,95), 95),0.01)
         r=0.9
@@ -80,11 +79,6 @@ class StatsampleStatisicsTestCase < Test::Unit::TestCase
         assert(Statsample::Bivariate.prop_pearson(t,n,:both)<0.05)
         assert(Statsample::Bivariate.prop_pearson(t,n,:right)>0.05)
         assert(Statsample::Bivariate.prop_pearson(t,n,:left)<0.05)
-
-        
-	else
-		puts "Bivariate.prop_pearson not tested (no ruby-gsl)"
-	end
     end
 	def test_covariance
 		if HAS_GSL
@@ -130,11 +124,7 @@ class StatsampleStatisicsTestCase < Test::Unit::TestCase
         v=([42]*23+[41]*4+[36]*1+[32]*1+[29]*1+[27]*2+[23]*1+[19]*1+[16]*2+[15]*2+[14,11,10,9,7]+ [6]*3+[5]*2+[4,3]).to_vector(:scale)
         assert_equal(50,v.size)
         assert_equal(1471,v.sum())
-	if HAS_GSL
         limits=Statsample::SRS.mean_confidence_interval_z(v.mean(), v.sds(), v.size,676,0.80)
-       else
-	       puts "SRS.mean_confidence_interval_z not tested (no ruby-gsl)"
-	       end
     end
     def test_estimation_proportion
         # total
@@ -148,14 +138,9 @@ class StatsampleStatisicsTestCase < Test::Unit::TestCase
         sam=100
         prop=0.37
         a=0.95
-	if HAS_GSL
         l= Statsample::SRS.proportion_confidence_interval_z(prop, sam, pop, a)
         assert_in_delta(0.28,l[0],0.01)
         assert_in_delta(0.46,l[1],0.01)
-	else
-	       puts "SRS.proportion_confidence_interval_z not tested (no ruby-gsl)"
-		
-		end
     end
     def test_ml
         if(true)

data/test/test_vector.rb

@@ -16,6 +16,17 @@ class StatsampleVectorTestCase < Test::Unit::TestCase
     def test_product
         a=[1,2,3,4,5].to_vector(:scale)
         assert_equal(120,a.product)
+    end
+    def test_matrix
+        a=[1,2,3,4,5].to_vector(:scale)
+        mh=Matrix[[1,2,3,4,5]]
+        mv=Matrix.columns([[1,2,3,4,5]])
+        assert_equal(mh,a.to_matrix)
+        assert_equal(mv,a.to_matrix(:vertical))
+        # 3*4 + 2*5 = 22
+        a=[3,2].to_vector(:scale)
+        b=[4,5].to_vector(:scale)
+        assert_equal(22,(a.to_matrix*b.to_matrix(:vertical))[0,0])
     end
 	def test_missing_values
 		@c.missing_values=[10]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: statsample
 version: !ruby/object:Gem::Version 
-  version: 0.3.4
+  version: 0.4.0
 platform: ruby
 authors: 
 - Claudio Bustos
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-08-21 00:00:00 -04:00
+date: 2009-09-10 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -60,7 +60,9 @@ files:
 - Manifest.txt
 - README.txt
 - bin/statsample
+- data/crime.txt
 - data/locale/es/LC_MESSAGES/statsample.mo
+- data/test_binomial.csv
 - demo/benchmark.rb
 - demo/chi-square.rb
 - demo/crosstab.rb
@@ -76,11 +78,17 @@ files:
 - demo/strata_proportion.rb
 - demo/stratum.rb
 - demo/t-student.rb
+- lib/distribution.rb
+- lib/distribution/chisquare.rb
+- lib/distribution/f.rb
+- lib/distribution/normal.rb
+- lib/distribution/t.rb
+- lib/matrix_extension.rb
 - lib/spss.rb
+- lib/statistics2.rb
 - lib/statsample.rb
 - lib/statsample/anova.rb
 - lib/statsample/bivariate.rb
-- lib/statsample/chidistribution.rb
 - lib/statsample/codification.rb
 - lib/statsample/combination.rb
 - lib/statsample/converters.rb
@@ -94,9 +102,15 @@ files:
 - lib/statsample/graph/svghistogram.rb
 - lib/statsample/graph/svgscatterplot.rb
 - lib/statsample/htmlreport.rb
+- lib/statsample/mle.rb
+- lib/statsample/mle/logit.rb
+- lib/statsample/mle/normal.rb
+- lib/statsample/mle/probit.rb
 - lib/statsample/multiset.rb
 - lib/statsample/regression.rb
-- lib/statsample/regression/logit.rb
+- lib/statsample/regression/binomial.rb
+- lib/statsample/regression/binomial/logit.rb
+- lib/statsample/regression/binomial/probit.rb
 - lib/statsample/regression/multiple.rb
 - lib/statsample/regression/multiple/alglibengine.rb
 - lib/statsample/regression/multiple/gslengine.rb
@@ -118,7 +132,11 @@ files:
 - test/test_csv.csv
 - test/test_csv.rb
 - test/test_dataset.rb
+- test/test_distribution.rb
 - test/test_ggobi.rb
+- test/test_gsl.rb
+- test/test_logit.rb
+- test/test_mle.rb
 - test/test_multiset.rb
 - test/test_regression.rb
 - test/test_reliability.rb
@@ -163,17 +181,23 @@ test_files:
 - test/test_anova.rb
 - test/test_codification.rb
 - test/test_crosstab.rb
+- test/test_distribution.rb
 - test/test_svg_graph.rb
 - test/test_csv.rb
+- test/test_gsl.rb
 - test/test_combination.rb
+- test/test_mle.rb
 - test/test_resample.rb
 - test/test_stratified.rb
 - test/test_vector.rb
 - test/test_srs.rb
+- test/test_stata.rb
 - test/test_ggobi.rb
 - test/test_xls.rb
+- test/test_logit.rb
 - test/test_statistics.rb
 - test/test_reliability.rb
 - test/test_dataset.rb
+- test/test_r.rb
 - test/test_regression.rb
 - test/test_multiset.rb