statsample 0.6.5 → 0.6.7

data/lib/statsample.rb

@@ -23,8 +23,8 @@ $:.unshift(File.expand_path(File.dirname(__FILE__)+"/../ext"))
 
 require 'matrix'
 require 'distribution'
+raise "Install reportbuilder ~>0.2.0" unless gem 'reportbuilder','~>0.2.0'
 require 'reportbuilder'
-
 class Numeric
   def square ; self * self ; end
 end
@@ -88,28 +88,32 @@ rescue LoadError
     end
   end
 end
-
-begin
-  require 'rbgsl'
-  HAS_GSL=true
-rescue LoadError
-  HAS_GSL=false
-end
-begin 
-  require 'alglib'
-  HAS_ALGIB=true
-rescue LoadError
-  HAS_ALGIB=false
-end
-# ++
-# Modules for statistical analysis
-# See first: 
-# * Converter : several modules to import and export data
-# * Vector: The base class of all analysis
-# * Dataset: An union of vectors.
+# Library for statistical analysis on Ruby
+#
+# * Classes for manipulation and storage of data:
+# * Module Statsample::Bivariate provides covariance and pearson, spearman, point biserial, tau a, tau b, gamma, tetrachoric (see Bivariate::Tetrachoric) and polychoric (see Bivariate::Polychoric) correlations. Include methods to create correlation and covariance matrices
+# * Multiple types of regression on Statsample::Regression
+# * Factorial Analysis algorithms on Statsample::Factor module.
+# * Dominance Analysis. Based on Budescu and Azen papers.link[http://psycnet.apa.org/journals/met/8/2/129/]. 
+# * Module Statsample::Codification, to help to codify open questions
+# * Converters to import and export data from databases, csv and excel files.
+# * Module Statsample::Crosstab provides function to create crosstab for categorical data
+# * Reliability analysis provides functions to analyze scales.
+# * Module Statsample::SRS (Simple Random Sampling) provides a lot of functions to estimate standard error for several type of samples
+# * Interfaces to gdchart, gnuplot and SVG::Graph 
 #
 module Statsample
-  VERSION = '0.6.5'
+  begin
+    require 'rbgsl'
+    def self.has_gsl?
+      true
+    end
+  rescue LoadError
+    def self.has_gsl?
+      false
+    end
+  end
+  VERSION = '0.6.7'
   SPLIT_TOKEN = ","
   autoload(:Database, 'statsample/converters')
   autoload(:Anova, 'statsample/anova')
@@ -137,16 +141,56 @@ module Statsample
   autoload(:Test, 'statsample/test')
   autoload(:Factor, 'statsample/factor')
   
-  def self.load(filename)
-    if File.exists? filename
-      o=false
-      File.open(filename,"r") {|fp| o=Marshal.load(fp) }
-      o
-    else
-      false
+  
+  
+  class << self
+    # Load a object saved on a file.
+    def load(filename)
+      if File.exists? filename
+        o=false
+        File.open(filename,"r") {|fp| o=Marshal.load(fp) }
+        o
+      else
+        false
+      end
     end
-  end
-    
+    # Create a matrix using vectors as columns.
+    # Use:
+    #
+    #   matrix=Statsample.vector_cols_matrix(v1,v2)
+    def vector_cols_matrix(*vs)
+      # test
+      size=vs[0].size
+      vs.each{|v|
+        raise ArgumentError,"Arguments should be Vector" unless v.instance_of? Statsample::Vector
+        raise ArgumentError,"Vectors size should be the same" if v.size!=size
+      }
+      Matrix.rows((0...size).to_a.collect() {|i|
+        vs.collect{|v| v[i]}
+      })
+    end
+    # Returns a duplicate of the input vectors, without missing data
+    # for any of the vectors.
+    # 
+    #  a=[1,2,3,6,7,nil,3,5].to_scale
+    #  b=[nil,nil,5,6,4,5,10,2].to_scale
+    #  c=[2,4,6,7,4,5,6,7].to_scale
+    #  a2,b2,c2=Statsample.only_valid(a,b,c)
+    #  => [#<Statsample::Scale:0xb748c8c8 @data=[3, 6, 7, 3, 5]>, 
+    #        #<Statsample::Scale:0xb748c814 @data=[5, 6, 4, 10, 2]>, 
+    #        #<Statsample::Scale:0xb748c760 @data=[6, 7, 4, 6, 7]>]
+    #
+    def only_valid(*vs)
+      i=1
+      h=vs.inject({}) {|a,v| a["v#{i}"]=v;i+=1;a}
+      ds=Statsample::Dataset.new(h).dup_only_valid
+      ds.vectors.values
+    end
+  end  
+  
+  
+  
+  
 	module Util
     # Reference: http://www.itl.nist.gov/div898/handbook/eda/section3/normprpl.htm
     def normal_order_statistic_medians(i,n)

data/test/test_bivariate.rb

@@ -33,7 +33,7 @@ class StatsampleBivariateTestCase < Test::Unit::TestCase
       poly  = Statsample::Bivariate::Polychoric.new(matrix)
       poly.compute_two_step_mle_drasgow_ruby
       assert_in_delta(tetra.r,poly.r,0.0001)
-      if HAS_GSL
+      if Statsample.has_gsl?
         poly.compute_two_step_mle_drasgow_gsl
         assert_in_delta(tetra.r,poly.r,0.0001)
       end
@@ -59,7 +59,7 @@ class StatsampleBivariateTestCase < Test::Unit::TestCase
       
 
       
-    if HAS_GSL  
+    if Statsample.has_gsl?  
       poly.method=:polychoric_series
       poly.compute
       
@@ -93,6 +93,7 @@ class StatsampleBivariateTestCase < Test::Unit::TestCase
     else
       puts "Two-step optimized, polychoric series and Joint method for Polychoric  requires GSL"
     end
+    assert(poly.summary)
   end
   def test_tetrachoric
     a,b,c,d=0,0,0,0
@@ -134,7 +135,7 @@ class StatsampleBivariateTestCase < Test::Unit::TestCase
     tc2  = Statsample::Bivariate::Tetrachoric.new_with_vectors(x,y)
     assert_equal(tc1.r,tc2.r)
     assert_equal(tc1.se,tc2.se)
-    
+    assert(tc.summary)
   end
   def test_matrix_correlation
     v1=[6,5,4,7,8,4,3,2].to_vector(:scale)
@@ -174,7 +175,7 @@ class StatsampleBivariateTestCase < Test::Unit::TestCase
     assert(Statsample::Bivariate.prop_pearson(t,n,:left)<0.05)
   end
   def test_covariance
-  if HAS_GSL
+  if Statsample.has_gsl?
     v1=[6,5,4,7,8,4,3,2].to_vector(:scale)
     v2=[2,3,7,8,6,4,3,2].to_vector(:scale)
     assert_in_delta(Statsample::Bivariate.covariance(v1,v2), Statsample::Bivariate.covariance_slow(v1,v2), 0.001)

data/test/test_combination.rb

@@ -18,7 +18,7 @@ class StatsampleCombinationTestCase < Test::Unit::TestCase
         assert_equal(expected,a)
     end
     def test_gsl_versus_ruby
-        if HAS_GSL
+        if Statsample.has_gsl?
         k=3
         n=10
         gsl=Statsample::Combination.new(k,n,false)

data/test/test_dataset.rb

@@ -13,7 +13,7 @@ class StatsampleDatasetTestCase < Test::Unit::TestCase
     assert_equal(%w{id name age city a1}, @ds.fields)
   end
   def test_saveload
-    outfile=Tempfile.new("/dataset.ds")
+    outfile=Tempfile.new("dataset.ds")
   @ds.save(outfile.path)
   a=Statsample.load(outfile.path)
   assert_equal(@ds,a)
@@ -389,4 +389,4 @@ class StatsampleDatasetTestCase < Test::Unit::TestCase
   
   end
     
-end
+end

data/test/test_factor.rb

@@ -2,11 +2,12 @@ $:.unshift(File.dirname(__FILE__)+'/../lib/')
 require 'statsample'
 require 'test/unit'
 class StatsampleFactorTestCase < Test::Unit::TestCase
+    # Tested with SPSS and R
     def test_pca
-      if HAS_GSL
+      if Statsample.has_gsl?
         require 'gsl'
         a=[2.5, 0.5, 2.2, 1.9, 3.1, 2.3, 2.0, 1.0, 1.5, 1.1].to_scale
-        b=[2.4,0.7,2.9,2.2,3.0,2.7,1.6,1.1,1.6,0.9].to_scale
+        b=[2.4, 0.7, 2.9, 2.2, 3.0, 2.7, 1.6, 1.1, 1.6, 0.9].to_scale
         a.recode! {|c| c-a.mean}
         b.recode! {|c| c-b.mean}
         ds={'a'=>a,'b'=>b}.to_dataset
@@ -16,20 +17,66 @@ class StatsampleFactorTestCase < Test::Unit::TestCase
         expected_eigenvalues.each_with_index{|ev,i|
           assert_in_delta(ev,pca.eigenvalues[i],0.001)
         }
+        expected_communality=[0.590, 0.694]
+        expected_communality.each_with_index{|ev,i|
+          assert_in_delta(ev,pca.communalities[i],0.001)
+        }
+        expected_cm=[0.768, 0.833]
+        
+        obs=pca.component_matrix(1).column(0).to_a
+        expected_cm.each_with_index{|ev,i|
+          assert_in_delta(ev,obs[i],0.001)
+        }
+
         expected_fm_1=GSL::Matrix[[0.677], [0.735]]
         expected_fm_2=GSL::Matrix[[0.677,0.735], [0.735, -0.677]]
         _test_matrix(expected_fm_1,pca.feature_vector(1))
         _test_matrix(expected_fm_2,pca.feature_vector(2))
+        assert(pca.summary)
       else
         puts "PCA not tested. Requires GSL"
       end
     end
+    
+    # Tested with R
+    def test_principalaxis
+      if Statsample.has_gsl?
+        require 'gsl'
+        matrix=Matrix[
+        [1.0, 0.709501601093587, 0.877596585880047, 0.272219316266807],  [0.709501601093587, 1.0, 0.291633797330304, 0.871141831433844], [0.877596585880047, 0.291633797330304, 1.0, -0.213373722977167], [0.272219316266807, 0.871141831433844, -0.213373722977167, 1.0]]
+        fa=Statsample::Factor::PrincipalAxis.new(matrix,:m=>1)
+        cm=Matrix[[0.923],[0.912],[0.507],[0.483]].to_gsl
+        _test_matrix(cm,fa.component_matrix)
+        h2=[0.852,0.832,0.257,0.233]
+        h2.each_with_index{|ev,i|
+          assert_in_delta(ev,fa.communalities[i],0.001)
+        }
+        eigen1=2.175
+        assert_in_delta(eigen1, fa.eigenvalues[0],0.001)
+        
+        fa=Statsample::Factor::PrincipalAxis.new(matrix,:smc=>false)
+        assert_raise RuntimeError do
+          fa.iterate
+          
+        end
+        
+        
+        
+        assert(fa.summary)
+        
+      else
+        puts "Principal Axis not tested. Requires GSL"
+      end
+    end
+    
+    
     def test_rotation_varimax
-      if HAS_GSL
+      if Statsample.has_gsl?
         a = Matrix[ [ 0.4320,  0.8129,  0.3872]  ,
          [0.7950, -0.5416,  0.2565]  ,
          [0.5944,  0.7234, -0.3441],
          [0.8945, -0.3921, -0.1863] ]
+         
          expected= Matrix[[-0.0204423,     0.938674,    -0.340334],
          [0.983662, 0.0730206, 0.134997],
          [0.0826106, 0.435975, -0.893379],
@@ -37,9 +84,9 @@ class StatsampleFactorTestCase < Test::Unit::TestCase
          varimax=Statsample::Factor::Varimax.new(a)
          varimax.iterate
          _test_matrix(expected,varimax.rotated)
-       else
-         puts "Rotation not tested. Requires GSL"
-       end
+      else
+        puts "Rotation not tested. Requires GSL"
+      end
     end
     def _test_matrix(a,b)
       a.size1.times {|i|

data/test/test_gsl.rb

@@ -3,7 +3,7 @@ require 'statsample'
 require 'test/unit'
 class StatsampleGSLTestCase < Test::Unit::TestCase
   def test_matrix_to_gsl
-    if HAS_GSL
+    if Statsample.has_gsl?
       a=[1,2,3,4,20].to_vector(:scale)
       b=[3,2,3,4,50].to_vector(:scale)
       c=[6,2,3,4,3].to_vector(:scale)

data/test/test_mle.rb

@@ -49,7 +49,7 @@ class StatsampleMLETestCase < Test::Unit::TestCase
         #p coeffs_nr
         ds=@ds_indep.dup
         ds.add_vector('y',y)
-        lr=Statsample::Regression::Multiple.listwise(ds,'y')
+        lr=Statsample::Regression.multiple(ds,'y')
         lr_constant = lr.constant
         lr_coeffs   = lr.coeffs
         assert_in_delta(coeffs_nr[0,0], lr_constant,0.0000001)

data/test/test_regression.rb

@@ -2,17 +2,24 @@ $:.unshift(File.dirname(__FILE__)+'/../lib/')
 require 'statsample'
 require 'test/unit'
 class StatsampleRegressionTestCase < Test::Unit::TestCase
-	def initialize(*args)
-		@x=[13,20,10,33,15].to_vector(:scale)
-		@y=[23,18,35,10,27	].to_vector(:scale)
-		@reg=Statsample::Regression::Simple.new_from_vectors(@x,@y)
-		super
-	end
 	def test_parameters
-		assert_in_delta(40.009, @reg.a,0.001)
-		assert_in_delta(-0.957, @reg.b,0.001)
-		assert_in_delta(4.248,@reg.standard_error,0.002)
+    @x=[13,20,10,33,15].to_vector(:scale)
+		@y=[23,18,35,10,27	].to_vector(:scale)
+		reg=Statsample::Regression::Simple.new_from_vectors(@x,@y)
+    _test_simple_regression(reg)
+    ds={'x'=>@x,'y'=>@y}.to_dataset
+		reg=Statsample::Regression::Simple.new_from_dataset(ds,'x','y')
+    _test_simple_regression(reg)
+    reg=Statsample::Regression.simple(@x,@y)
+    _test_simple_regression(reg)
+    
 	end
+  def _test_simple_regression(reg)
+    assert_in_delta(40.009, reg.a,0.001)
+		assert_in_delta(-0.957, reg.b,0.001)
+		assert_in_delta(4.248,reg.standard_error,0.002)
+  end
+   
   def test_multiple_dependent
     complete=Matrix[
     [1,0.53,0.62,0.19,-0.09,0.08,0.02,-0.12,0.08],
@@ -54,7 +61,7 @@ class StatsampleRegressionTestCase < Test::Unit::TestCase
 
 
   def test_multiple_regression_gsl
-    if HAS_GSL
+    if Statsample.has_gsl?
       @a=[1,3,2,4,3,5,4,6,5,7].to_vector(:scale)
       @b=[3,3,4,4,5,5,6,6,4,4].to_vector(:scale)
       @c=[11,22,30,40,50,65,78,79,99,100].to_vector(:scale)
@@ -78,29 +85,7 @@ class StatsampleRegressionTestCase < Test::Unit::TestCase
   end
 
 
-  def test_multiple_regression_alglib
-    if HAS_ALGIB
-      @a=[1,3,2,4,3,5,4,6,5,7].to_vector(:scale)
-      @b=[3,3,4,4,5,5,6,6,4,4].to_vector(:scale)
-      @c=[11,22,30,40,50,65,78,79,99,100].to_vector(:scale)
-      @y=[3,4,5,6,7,8,9,10,20,30].to_vector(:scale)
-      ds={'a'=>@a,'b'=>@b,'c'=>@c,'y'=>@y}.to_dataset
-      lr=Statsample::Regression::Multiple::AlglibEngine.new(ds,'y')
-      model_test(lr,'alglib')
-      predicted=[1.7857, 6.0989, 3.2433, 7.2908, 4.9667, 10.3428, 8.8158, 10.4717, 23.6639, 25.3198]
-      c_predicted=lr.predicted
-      predicted.each_index{|i|
-          assert_in_delta(predicted[i],c_predicted[i],0.001)
-      }
-      residuals=[1.2142, -2.0989, 1.7566, -1.29085, 2.033, -2.3428, 0.18414, -0.47177, -3.66395, 4.6801]
-      c_residuals=lr.residuals
-      residuals.each_index{|i|
-          assert_in_delta(residuals[i],c_residuals[i],0.001)
-      }
-    else
-    puts "Regression::Multiple::AlglibEngine not tested (no Alglib)"
-    end
-  end
+  
   def model_test_matrix(lr,name='undefined')
     
     stan_coeffs={'a'=>0.151,'b'=>-0.547,'c'=>0.997}

data/test/test_statistics.rb

@@ -3,9 +3,9 @@ require 'statsample'
 require 'test/unit'
 class StatsampleStatisicsTestCase < Test::Unit::TestCase
 
-	def initialize(*args)
-		super
-	end
+  def initialize(*args)
+    super
+  end
   def test_recode_repeated
     a=%w{a b c c d d d e}
     exp=["a","b","c_1","c_2","d_1","d_2","d_3","e"]
@@ -21,19 +21,11 @@ class StatsampleStatisicsTestCase < Test::Unit::TestCase
     assert(!"1212-1212-1".is_number?)
     assert(!"a10".is_number?)
     assert(!"".is_number?)
-  
-  end
-  def test_chi_square
-    real=Matrix[[95,95],[45,155]]
-    expected=Matrix[[68,122],[72,128]]
-    assert_nothing_raised do
-      chi=Statsample::Test.chi_square(real,expected)
-    end
-    chi=Statsample::Test.chi_square(real,expected)
-    assert_in_delta(32.53,chi,0.1)
+
   end
-	
-  def test_estimation_mean              
+
+
+  def test_estimation_mean
     v=([42]*23+[41]*4+[36]*1+[32]*1+[29]*1+[27]*2+[23]*1+[19]*1+[16]*2+[15]*2+[14,11,10,9,7]+ [6]*3+[5]*2+[4,3]).to_vector(:scale)
     assert_equal(50,v.size)
     assert_equal(1471,v.sum())
@@ -45,7 +37,7 @@ class StatsampleStatisicsTestCase < Test::Unit::TestCase
     sam=200
     prop=0.19
     assert_in_delta(81.8, Statsample::SRS.proportion_total_sd_ep_wor(prop, sam, pop), 0.1)
-    
+
     # confidence limits
     pop=500
     sam=100
@@ -57,13 +49,15 @@ class StatsampleStatisicsTestCase < Test::Unit::TestCase
   end
   def test_ml
     if(true)
-    real=[1,1,1,1].to_vector(:scale)
-    
-    pred=[0.0001,0.0001,0.0001,0.0001].to_vector(:scale)
-    # puts  Statsample::Bivariate.maximum_likehood_dichotomic(pred,real)
-    
+      real=[1,1,1,1].to_vector(:scale)
+
+      pred=[0.0001,0.0001,0.0001,0.0001].to_vector(:scale)
+      # puts  Statsample::Bivariate.maximum_likehood_dichotomic(pred,real)
+
     end
   end
+
+
   def test_simple_linear_regression
     a=[1,2,3,4,5,6].to_vector(:scale)
     b=[6,2,4,10,12,8].to_vector(:scale)
@@ -74,17 +68,5 @@ class StatsampleStatisicsTestCase < Test::Unit::TestCase
     assert_in_delta(1.314,reg.b,0.001)
     assert_in_delta(0.657,reg.r,0.001)
     assert_in_delta(0.432,reg.r2,0.001)
-	end
-  def test_u_mannwhitney
-    a=[1,2,3,4,5,6].to_scale
-    b=[0,5,7,9,10,11].to_scale
-    assert_equal(7.5, Statsample::Test.u_mannwhitney(a,b).u)
-    assert_equal(7.5, Statsample::Test.u_mannwhitney(b,a).u)
-    a=[1, 7,8,9,10,11].to_scale
-    b=[2,3,4,5,6,12].to_scale
-    assert_equal(11, Statsample::Test.u_mannwhitney(a,b).u)
-    
-    
-    
   end
 end

data/test/test_stest.rb

@@ -0,0 +1,35 @@
+$:.unshift(File.dirname(__FILE__)+'/../lib/')
+require 'statsample'
+require "tempfile"
+require 'test/unit'
+
+class StatsampleStatisticalTestCase < Test::Unit::TestCase
+  def test_chi_square
+    real=Matrix[[95,95],[45,155]]
+    expected=Matrix[[68,122],[72,128]]
+    assert_nothing_raised do
+      chi=Statsample::Test.chi_square(real,expected)
+    end
+    chi=Statsample::Test.chi_square(real,expected)
+    assert_in_delta(32.53,chi,0.1)
+  end
+  def test_u_mannwhitney
+    a=[1,2,3,4,5,6].to_scale
+    b=[0,5,7,9,10,11].to_scale
+    assert_equal(7.5, Statsample::Test.u_mannwhitney(a,b).u)
+    assert_equal(7.5, Statsample::Test.u_mannwhitney(b,a).u)
+    a=[1, 7,8,9,10,11].to_scale
+    b=[2,3,4,5,6,12].to_scale
+    assert_equal(11, Statsample::Test.u_mannwhitney(a,b).u)
+  end
+  
+  
+  def test_levene
+    
+    a=[1,2,3,4,5,6,7,8,100,10].to_scale
+    b=[30,40,50,60,70,80,90,100,110,120].to_scale    
+    levene=Statsample::Test::Levene.new([a,b])
+    assert_in_delta(0.778, levene.f, 0.001)
+    assert_in_delta(0.389, levene.probability, 0.001)
+  end
+end

data/test/test_svg_graph.rb

@@ -15,7 +15,7 @@ class StatsampleSvgGraphTestCase < Test::Unit::TestCase
   super
   end
   def test_histogram
-    if HAS_GSL
+    if Statsample.has_gsl?
       ar=(1..1000).to_a.collect {|a|
     rand(10)
     }.to_vector(:scale)
@@ -47,7 +47,7 @@ class StatsampleSvgGraphTestCase < Test::Unit::TestCase
     vector.svggraph_frequencies(file,800,600,SVG::Graph::PieNoOp,:graph_title=>'Pie')
     assert(File.exists?(file))		
     vector.type=:scale
-    if HAS_GSL
+    if Statsample.has_gsl?
     file=Tempfile.new("svg_histogram.svg").path
       hist=vector.svggraph_histogram(5)
       File.open(file,"wb") {|fp|