statsample 0.18.0 → 1.0.0

data/History.txt

@@ -1,3 +1,26 @@
+=== 1.0.0 / 2011-01-27
+
+* Added Statsample::Analysis, a beautiful DSL to perform fast statistical analysis using statsample. See directory /examples
+* Created benchmarks directory 
+* Removed Distribution module from statsample and moved to a gem. Changes on code to reflect new API
+* Optimized simple regression.  Better library detection
+* New 'should_with_gsl' to test methods with gsl. Refactored Factor::MAP
+* Almost complete GSL cleanup on Vector
+* Updated some doc on Vector
+* Used GSL::Matrix on Factor classes when available
+* SkillScaleAnalysis doesn't crash with one or more vectors with 0 variance
+* Modified examples using Statsample::Analysis
+* Simplified eigen calculations
+* Updated some examples. Added correlation matrix speed suite
+* Correlation matrix optimized. Better specs 
+* Optimized correlation matrix. Use gsl matrix algebra or pairwise correlations depending on empiric calculated equations. See benchmarks/correlation_matrix.rb to see implementation of calculation 
+* Moved tests fixtures from data to test/fixtures
+* Fixed some errors on tests
+* Bug fix: constant_se on binomial regression have an error 
+* All test should work on ruby 1.9.3 
+* New Vector.[] and Vector.new_scale
+* Detect linearly dependent predictors on OLS. 
+
 === 0.18.0 / 2011-01-07
 * New Statsample.load_excel 
 * New Statsample.load_csv

data/Manifest.txt

@@ -3,14 +3,19 @@ LICENSE.txt
 Manifest.txt
 README.txt
 Rakefile
+benchmarks/correlation_matrix_15_variables.rb
+benchmarks/correlation_matrix_5_variables.rb
+benchmarks/correlation_matrix_methods/correlation_matrix.ds
+benchmarks/correlation_matrix_methods/correlation_matrix.html
+benchmarks/correlation_matrix_methods/correlation_matrix.rb
+benchmarks/correlation_matrix_methods/correlation_matrix.xls
+benchmarks/correlation_matrix_methods/correlation_matrix_gsl_ruby.ods
+benchmarks/correlation_matrix_methods/correlation_matrix_with_graphics.ods
+benchmarks/correlation_matrix_methods/results.ds
+benchmarks/factor_map.rb
+benchmarks/helpers_benchmark.rb
 bin/statsample
-data/crime.txt
-data/hartman_23.matrix
 data/locale/es/LC_MESSAGES/statsample.mo
-data/repeated_fields.csv
-data/test_binomial.csv
-data/tetmat_matrix.txt
-data/tetmat_test.txt
 doc_latex/manual/equations.tex
 examples/boxplot.rb
 examples/correlation_matrix.rb
@@ -23,7 +28,6 @@ examples/levene.rb
 examples/multiple_regression.rb
 examples/multivariate_correlation.rb
 examples/parallel_analysis.rb
-examples/parallel_analysis_tetrachoric.rb
 examples/polychoric.rb
 examples/principal_axis.rb
 examples/reliability.rb
@@ -34,15 +38,11 @@ examples/u_test.rb
 examples/vector.rb
 examples/velicer_map_test.rb
 grab_references.rb
-lib/distribution.rb
-lib/distribution/chisquare.rb
-lib/distribution/f.rb
-lib/distribution/normal.rb
-lib/distribution/normalbivariate.rb
-lib/distribution/normalmultivariate.rb
-lib/distribution/t.rb
 lib/spss.rb
 lib/statsample.rb
+lib/statsample/analysis.rb
+lib/statsample/analysis/suite.rb
+lib/statsample/analysis/suitereportbuilder.rb
 lib/statsample/anova.rb
 lib/statsample/anova/oneway.rb
 lib/statsample/anova/twoway.rb
@@ -91,6 +91,7 @@ lib/statsample/reliability/scaleanalysis.rb
 lib/statsample/reliability/skillscaleanalysis.rb
 lib/statsample/resample.rb
 lib/statsample/rserve_extension.rb
+lib/statsample/shorthand.rb
 lib/statsample/srs.rb
 lib/statsample/test.rb
 lib/statsample/test/bartlettsphericity.rb
@@ -100,6 +101,7 @@ lib/statsample/test/levene.rb
 lib/statsample/test/t.rb
 lib/statsample/test/umannwhitney.rb
 lib/statsample/vector.rb
+lib/statsample/vector/gsl.rb
 po/es/statsample.mo
 po/es/statsample.po
 po/statsample.pot
@@ -107,7 +109,16 @@ references.txt
 setup.rb
 test/fixtures/bank2.dat
 test/fixtures/correlation_matrix.rb
+test/fixtures/crime.txt
+test/fixtures/hartman_23.matrix
+test/fixtures/repeated_fields.csv
+test/fixtures/test_binomial.csv
+test/fixtures/test_csv.csv
+test/fixtures/test_xls.xls
+test/fixtures/tetmat_matrix.txt
+test/fixtures/tetmat_test.txt
 test/helpers_tests.rb
+test/test_analysis.rb
 test/test_anovaoneway.rb
 test/test_anovatwoway.rb
 test/test_anovatwowaywithdataset.rb
@@ -116,12 +127,12 @@ test/test_bartlettsphericity.rb
 test/test_bivariate.rb
 test/test_codification.rb
 test/test_crosstab.rb
-test/test_csv.csv
 test/test_csv.rb
 test/test_dataset.rb
-test/test_distribution.rb
 test/test_dominance_analysis.rb
 test/test_factor.rb
+test/test_factor_map.rb
+test/test_factor_pa.rb
 test/test_ggobi.rb
 test/test_gsl.rb
 test/test_histogram.rb
@@ -144,4 +155,4 @@ test/test_test_t.rb
 test/test_umannwhitney.rb
 test/test_vector.rb
 test/test_xls.rb
-test/test_xls.xls
+web/Rakefile

data/Rakefile

@@ -40,9 +40,10 @@ h=Hoe.spec('statsample') do
   #self.testlib=:minitest
 	self.rubyforge_name = "ruby-statsample"
 	self.developer('Claudio Bustos', 'clbustos@gmail.com')
-	self.extra_deps << ["spreadsheet","~>0.6.5"] <<  ["reportbuilder", "~>1.4"] << ["minimization", "~>0.2.0"] << ["fastercsv", ">0"] << ["dirty-memoize", "~>0.0"] << ["extendmatrix","~>0.3.1"] << ["statsample-bivariate-extension", ">0"] << ["rserve-client", "~>0.2.5"] << ["rubyvis", "~>0.4.0"]
+	self.extra_deps << ["spreadsheet","~>0.6.5"] <<  ["reportbuilder", "~>1.4"] << ["minimization", "~>0.2.0"] << ["fastercsv", ">0"] << ["dirty-memoize", "~>0.0"] << ["extendmatrix","~>0.3.1"] << ["statsample-bivariate-extension", ">0"] << ["rserve-client", "~>0.2.5"] << ["rubyvis", "~>0.4.0"] << ["distribution", "~>0.2.0"]
+  
+	self.extra_dev_deps << ["hoe","~>0"] << ["shoulda","~>0"] << ["minitest", "~>2.0"] << ["rserve-client", "~>0"] << ["gettext", "~>0"] << ["mocha", "~>0"] << ["hoe-git", "~>0"]
   
-	self.extra_dev_deps << ["hoe","~>0"] << ["shoulda","~>0"] << ["minitest", "~>2.0"]
   self.clean_globs << "test/images/*" << "demo/item_analysis/*" << "demo/Regression"
   self.post_install_message = <<-EOF
 ***************************************************

data/benchmarks/correlation_matrix_15_variables.rb

@@ -0,0 +1,31 @@
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_benchmark.rb'))
+
+extend BenchPress
+cases=250
+vars=20
+
+
+name "gsl matrix based vs. manual ruby correlation matrix (#{vars} vars, #{cases} cases)"
+author 'Clbustos'
+date '2011-01-18'
+summary "
+A correlation matrix could be constructed using matrix algebra or
+mannualy, calculating covariances, means and sd for each pair of vectors.
+In this test, we test the calculation using #{vars} variables with 
+#{cases} cases on each vector
+"
+
+reps 200 #number of repetitions
+
+ds=vars.times.inject({}) {|ac,v|
+ac["x#{v}"]=Statsample::Vector.new_scale(cases) {rand()}
+ac
+}.to_dataset
+    
+measure "Statsample::Bivariate.correlation_matrix_optimized" do
+  Statsample::Bivariate.correlation_matrix_optimized(ds)
+end
+
+measure "Statsample::Bivariate.correlation_matrix_pairwise" do
+  Statsample::Bivariate.correlation_matrix_pairwise(ds)
+end

data/benchmarks/correlation_matrix_5_variables.rb

@@ -0,0 +1,32 @@
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_benchmark.rb'))
+
+extend BenchPress
+cases=500
+vars=5
+
+
+name "gsl matrix based vs. manual ruby correlation matrix (#{vars} vars, #{cases} cases)"
+author 'Clbustos'
+date '2011-01-18'
+summary "
+A correlation matrix could be constructed using matrix algebra or
+mannualy, calculating covariances, means and sd for each pair of vectors.
+In this test, we test the calculation using #{vars} variables with 
+#{cases} cases on each vector
+"
+
+reps 200 #number of repetitions
+
+
+ds=vars.times.inject({}) {|ac,v|
+ac["x#{v}"]=Statsample::Vector.new_scale(cases) {rand()}
+ac
+}.to_dataset
+    
+measure "Statsample::Bivariate.correlation_matrix_optimized" do
+  Statsample::Bivariate.correlation_matrix_optimized(ds)
+end
+
+measure "Statsample::Bivariate.correlation_matrix_pairwise" do
+  Statsample::Bivariate.correlation_matrix_pairwise(ds)
+end

data/benchmarks/correlation_matrix_methods/correlation_matrix.html

@@ -0,0 +1,93 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html;charset=utf-8" >
+<title>Correlation matrix analysis</title>
+        <style>
+        body {
+          margin:0;
+          padding:1em;
+        }
+        table {
+          border-collapse: collapse;
+
+        }
+        table td {
+          border: 1px solid black;
+        }
+        .section {
+          margin:0.5em;
+        }
+        </style>
+
+</head><body>
+<h1>Correlation matrix analysis</h1><div id='toc'><div class='title'>List of contents</div>
+<ul>
+<li><a href='#toc_1'>Multiple reggresion of cases,vars,c_v on time_optimized</a></li>
+<ul>
+<li><a href='#toc_2'>ANOVA</a></li>
+</ul>
+<li><a href='#toc_3'>Multiple reggresion of cases,vars,c_v on time_pairwise</a></li>
+<ul>
+<li><a href='#toc_4'>ANOVA</a></li>
+</ul>
+</ul>
+</div>
+<div class='tot'><div class='title'>List of tables</div><ul><li><a href='#table_1'>ANOVA Table</a></li><li><a href='#table_2'>Beta coefficients</a></li><li><a href='#table_3'>ANOVA Table</a></li><li><a href='#table_4'>Beta coefficients</a></li></ul></div>
+  <div class='section'><h2>Multiple reggresion of cases,vars,c_v on time_optimized</h2><a name='toc_1'></a>
+    <p>Engine: Statsample::Regression::Multiple::RubyEngine</p>
+    <p>Cases(listwise)=63(63)</p>
+    <p>R=0.978844</p>
+    <p>R^2=0.958137</p>
+    <p>R^2 Adj=0.956008</p>
+    <p>Std.Error R=3.092024</p>
+    <p>Equation=4.031667 + 0.018039cases + 0.244790vars + 0.001197c_v</p>
+    <div class='section'><h3>ANOVA</h3><a name='toc_2'></a>
+      <a name='table_1'></a><table><caption>ANOVA Table</caption><thead><th>source</th><th>ss</th><th>df</th><th>ms</th><th>f</th><th>p</th></thead>
+<tbody>
+<tr><td>Regression</td><td>12910.098</td><td>3</td><td>4303.366</td><td>450.114</td><td>0.000</td></tr>
+<tr><td>Error</td><td>564.076</td><td>59</td><td>9.561</td><td></td><td></td></tr>
+<tr><td>Total</td><td>13474.174</td><td>62</td><td>4312.927</td><td></td><td></td></tr>
+</tbody>
+</table>
+
+    </div>
+    <a name='table_2'></a><table><caption>Beta coefficients</caption><thead><th>coeff</th><th>b</th><th>beta</th><th>se</th><th>t</th></thead>
+<tbody>
+<tr><td>Constant</td><td>4.031667</td><td>-</td><td>0.752604</td><td>5.356953</td></tr>
+<tr><td>cases</td><td>0.018039</td><td>0.381587</td><td>0.001961</td><td>9.200093</td></tr>
+<tr><td>vars</td><td>0.244790</td><td>0.224390</td><td>0.036055</td><td>6.789335</td></tr>
+<tr><td>c_v</td><td>0.001197</td><td>0.584174</td><td>0.000094</td><td>12.738410</td></tr>
+</tbody>
+</table>
+
+  </div>
+  <div class='section'><h2>Multiple reggresion of cases,vars,c_v on time_pairwise</h2><a name='toc_3'></a>
+    <p>Engine: Statsample::Regression::Multiple::RubyEngine</p>
+    <p>Cases(listwise)=63(63)</p>
+    <p>R=0.999637</p>
+    <p>R^2=0.999275</p>
+    <p>R^2 Adj=0.999238</p>
+    <p>Std.Error R=0.538365</p>
+    <p>Equation=-0.520303 + -0.000708cases + 1.234451vars + 0.000735c_v</p>
+    <div class='section'><h3>ANOVA</h3><a name='toc_4'></a>
+      <a name='table_3'></a><table><caption>ANOVA Table</caption><thead><th>source</th><th>ss</th><th>df</th><th>ms</th><th>f</th><th>p</th></thead>
+<tbody>
+<tr><td>Regression</td><td>23554.271</td><td>3</td><td>7851.424</td><td>27089.134</td><td>0.000</td></tr>
+<tr><td>Error</td><td>17.100</td><td>59</td><td>0.290</td><td></td><td></td></tr>
+<tr><td>Total</td><td>23571.372</td><td>62</td><td>7851.714</td><td></td><td></td></tr>
+</tbody>
+</table>
+
+    </div>
+    <a name='table_4'></a><table><caption>Beta coefficients</caption><thead><th>coeff</th><th>b</th><th>beta</th><th>se</th><th>t</th></thead>
+<tbody>
+<tr><td>Constant</td><td>-0.520303</td><td>-</td><td>0.131039</td><td>-3.970594</td></tr>
+<tr><td>cases</td><td>-0.000708</td><td>-0.011324</td><td>0.000341</td><td>-2.074007</td></tr>
+<tr><td>vars</td><td>1.234451</td><td>0.855546</td><td>0.006278</td><td>196.641087</td></tr>
+<tr><td>c_v</td><td>0.000735</td><td>0.271138</td><td>0.000016</td><td>44.912972</td></tr>
+</tbody>
+</table>
+
+  </div>
+</body></html>

data/benchmarks/correlation_matrix_methods/correlation_matrix.rb

@@ -0,0 +1,75 @@
+# This test create a database to adjust the best algorithm
+# to use on correlation matrix
+require(File.expand_path(File.dirname(__FILE__)+'/../helpers_benchmark.rb'))
+require 'statsample'
+require 'benchmark'
+
+def create_dataset(vars,cases) 
+  ran=Distribution::Normal.rng
+  ds=vars.times.inject({}) {|ac,v|
+    ac["x#{v}"]=Statsample::Vector.new_scale(cases) {ran.call}
+  ac
+  }.to_dataset
+end
+
+def prediction_pairwise(vars,cases)
+	Statsample::Bivariate.prediction_pairwise(vars,cases) / 10
+end
+def prediction_optimized(vars,cases)
+	Statsample::Bivariate.prediction_optimized(vars,cases) / 10
+end
+
+
+
+if !File.exists?("correlation_matrix.ds") or File.mtime(__FILE__) > File.mtime("correlation_matrix.ds")
+reps=100 #number of repetitions
+ds_sizes=[5,10,30,50,100,150,200,500,1000]
+ds_vars=[3,4,5,10,20,30,40]
+#ds_sizes=[5,10]
+#ds_vars=[3,5,20]
+rs=Statsample::Dataset.new(%w{cases vars time_optimized time_pairwise})
+
+ds_sizes.each do |cases|
+  ds_vars.each do |vars|
+      ds=create_dataset(vars,cases)
+      time_optimized= Benchmark.realtime do
+        reps.times { 
+        Statsample::Bivariate.correlation_matrix_optimized(ds) 
+        ds.clear_gsl
+        }
+      end
+      
+      time_pairwise= Benchmark.realtime do
+        reps.times { 
+        Statsample::Bivariate.correlation_matrix_pairwise(ds)
+        }
+      end
+      
+      puts "Cases:#{cases}, vars:#{vars} -> opt:%0.3f (%0.3f) | pair: %0.3f (%0.3f)" % [time_optimized, prediction_optimized(vars,cases), time_pairwise, prediction_pairwise(vars,cases)]
+      
+      rs.add_case({'cases'=>cases,'vars'=>vars,'time_optimized'=>Math.sqrt(time_optimized*1000),'time_pairwise'=>Math.sqrt(time_pairwise*1000)})
+    end
+  end
+  
+else
+  rs=Statsample.load("correlation_matrix.ds")
+end
+
+
+rs.fields.each {|f| rs[f].type=:scale}
+
+rs['c_v']=rs.collect {|row| row['cases']*row['vars']}
+
+rs.update_valid_data
+rs.save("correlation_matrix.ds")
+Statsample::Excel.write(rs,"correlation_matrix.xls")
+
+
+
+rb=ReportBuilder.new(:name=>"Correlation matrix analysis")
+
+rb.add(Statsample::Regression.multiple(rs[['cases','vars','time_optimized','c_v']],'time_optimized', :digits=>6))
+rb.add(Statsample::Regression.multiple(rs[['cases','vars','time_pairwise','c_v']],'time_pairwise', :digits=>6))
+
+
+rb.save_html("correlation_matrix.html")

data/benchmarks/factor_map.rb

@@ -0,0 +1,37 @@
+require(File.expand_path(File.dirname(__FILE__)+'/helpers_benchmark.rb'))
+
+extend BenchPress
+
+
+name "Statsample::Factor::Map with and without GSL"
+author 'Clbustos'
+date '2011-01-18'
+summary "Velicer's MAP uses a lot of Matrix algebra. How much we can improve the timing using GSL?
+"
+
+reps 20 #number of repetitions
+
+m=Matrix[ 
+        [ 1, 0.846, 0.805, 0.859, 0.473, 0.398, 0.301, 0.382],
+        [ 0.846, 1, 0.881, 0.826, 0.376, 0.326, 0.277, 0.415],
+        [ 0.805, 0.881, 1, 0.801, 0.38, 0.319, 0.237, 0.345],
+        [ 0.859, 0.826, 0.801, 1, 0.436, 0.329, 0.327, 0.365],
+        [ 0.473, 0.376, 0.38, 0.436, 1, 0.762, 0.73, 0.629],
+        [ 0.398, 0.326, 0.319, 0.329, 0.762, 1, 0.583, 0.577],
+        [ 0.301, 0.277, 0.237, 0.327, 0.73, 0.583, 1, 0.539],
+        [ 0.382, 0.415, 0.345, 0.365, 0.629, 0.577, 0.539, 1]
+  ]
+  
+map=Statsample::Factor::MAP.new(m)
+
+
+measure "Statsample::Factor::MAP without GSL" do
+  map.use_gsl=false
+  map.compute
+end
+
+measure "Statsample::Factor::MAP with GSL" do
+  map.use_gsl=true
+  map.compute
+end
+

data/benchmarks/helpers_benchmark.rb

@@ -0,0 +1,5 @@
+$:.unshift(File.expand_path(File.dirname(__FILE__)+'/../lib/'))
+$:.unshift(File.expand_path(File.dirname(__FILE__)+'/'))
+
+require 'statsample'
+require 'bench_press'

data/examples/boxplot.rb

@@ -1,17 +1,16 @@
 #!/usr/bin/ruby
 $:.unshift(File.dirname(__FILE__)+'/../lib/')
-$:.unshift('/home/cdx/dev/reportbuilder/lib/')
-
-require 'benchmark'
 require 'statsample'
-n=100
-a=(n-1).times.map {|i| rand()*20+50}
-b=n.times.map {|i| rand()*10+50}.to_scale
-c=n.times.map {|i| rand()*5+50}.to_scale
- 
-a.push(30)
-a=a.to_scale
-sp=Statsample::Graph::Boxplot.new(:vectors=>[a,b,c],:width=>300, :height=>300, :groups=>%w{first first second}, :minimum=>0)
-rb=ReportBuilder.new
-rb.add(sp)
-puts rb.to_text
+Statsample::Analysis.store(Statsample::Graph::Boxplot) do 
+  n=30
+  a=rnorm(n-1,50,10)
+  b=rnorm(n, 30,5)
+  c=rnorm(n,5,1)
+  a.push(2)
+  boxplot(:vectors=>[a,b,c],:width=>300, :height=>300, :groups=>%w{first first second}, :minimum=>0)
+  
+end
+
+if __FILE__==$0
+  Statsample::Analysis.run
+end