statsample 1.5.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2bdb8a5e29a75f62af49b094ac026bbc01597707
-  data.tar.gz: 9459ac7bf01e6d20e81a23286dab63f63bdbcea3
+  metadata.gz: 244a3b640d29832affc12304a77aa447f4f2df0d
+  data.tar.gz: 08261176b4763367e65036c486e699491e209e7d
 SHA512:
-  metadata.gz: de03429dbc2c7a1bf0e0acfd0afcebcb55f53630688d635bdfa549eca988bfde40f601e056ae10946f064e86e769bc3be0b50888e1a3209ab243d0253e5b9ed6
-  data.tar.gz: dd71de3f239d8202f3716e4e268f5199cce24e6436e1b017d82970a3bf27da1749bc00a0ab35d030fbd1f63b717aab49c513bb51f8728807a565206cf81d30f1
+  metadata.gz: 125beac1e267c030c6d4b70b84543a7888f3b8fcb43e6475913b00f7332dd5ce01bb6330813c3c99f5d649add677a339dc8ef366b466f2710aeaaf4bf20c0cdf
+  data.tar.gz: 402c505e62785e2f1eacb28b8798fed26193ed54171f7dce2a106584800ffd3de657013559c55df9e20fe5827fe6309c53f526213fc9ec23bfa2a95bf086d38e

data/.build.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+git clone https://github.com/SciRuby/nmatrix.git
+cd nmatrix
+gem build nmatrix.gemspec
+gem install nmatrix-0.1.0.gem
+cd ..
+rm -rf nmatrix
+git clone https://github.com/v0dro/gsl-nmatrix
+cd gsl-nmatrix
+gem build gsl-nmatrix.gemspec
+gem install gsl-nmatrix-1.17.gem
+cd ..
+rm -rf gsl-nmatrix
+

data/.gitignore

@@ -12,3 +12,4 @@ examples/images/*
 examples/*.html
 web/upload_task.rb
 .idea
+*.gem

data/.travis.yml

@@ -1,16 +1,28 @@
 language:
   ruby
 
+env:
+  - CPLUS_INCLUDE_PATH=/usr/include/atlas C_INCLUDE_PATH=/usr/include/atlas
+
 rvm:
-  - 1.9.3
-  - 2.0
-  - 2.1
-  - 2.2
+  - '1.9.3'
+  - '2.0'
+  - '2.1'
+  - '2.2'
+
+matrix:
+  fast_finish:
+    true
+
+script: "bundle exec rake test"
+
+install:
+  - gem install bundler
+  - ./.build.sh
+  - bundle install
 
-script:
-  bundle exec rake test
-  
 before_install:
   - sudo apt-get update -qq
+  - sudo apt-get install -qq libatlas-base-dev
   - sudo apt-get install -y libgsl0-dev r-base r-base-dev
   - sudo Rscript -e "install.packages(c('Rserve','irr'),,'http://cran.us.r-project.org')"

data/CONTRIBUTING.md

@@ -0,0 +1,33 @@
+# Contributing guide
+
+## Installing statsample development dependencies
+
+If you want to run the full test suite, you will need the latest unreleased nmatrix and gsl-nmatrix ruby gems. They will be released upstream soon but please follow this procdure for now.
+
+Keep in mind that either nmatrix OR gsl-nmatrix are NOT NECESSARY for using statsample. They are just required for an optional speed up. 
+
+Statsample also works with [rb-gsl](https://github.com/blackwinter/rb-gsl), though installing that will cause a problem if you have any nmatrix dependent code because narray and nmatrix have a namespace problem.
+
+To install dependencies, execute the following commands:
+
+  `export CPLUS_INCLUDE_PATH=/usr/include/atlas` 
+  `export C_INCLUDE_PATH=/usr/include/atlas`
+  `sudo apt-get update -qq`
+  `sudo apt-get install -qq libatlas-base-dev`
+  `sudo apt-get --purge remove liblapack-dev liblapack3 liblapack3gf`
+  `sudo apt-get install -y libgsl0-dev r-base r-base-dev`
+  `sudo Rscript -e "install.packages(c('Rserve','irr'),,'http://cran.us.r-project.org')"`
+
+Then execute the .build.sh script to clone and install the latest nmatrix and gsl-nmatrix on your system:
+
+  `./.build.sh`
+
+Then finally install remaining dependencies:
+
+  `bundle install`
+
+And run the test suite (should be all green):
+
+  `bundle exec rake test`
+
+If you have problems installing nmatrix, please consult the [nmatrix installation wiki](https://github.com/SciRuby/nmatrix/wiki/Installation) or the [mailing list](https://groups.google.com/forum/#!forum/sciruby-dev).

data/History.txt

@@ -1,3 +1,8 @@
+=== 2.0.0 / 2015-06-20
+  * Added dependency on daru and replaced Statsample::Vector and Dataset with
+    Daru::Vector and Daru::DataFrame.
+  * NMatrix and gsl-nmatrix are used as development dependencies.
+
 === 1.5.0 / 2015-06-11
   * Made sure all methods work properly with and without GSL.
   * Statsample works with either rb-gsl or gsl-nmatrix.

data/README.md

@@ -32,18 +32,56 @@ If you need to work on Structural Equation Modeling, you could see +statsample-s
 ```bash
 $ [sudo] gem install statsample-sem
 ```
+# Testing
+
+See CONTRIBUTING for information on testing and contributing to statsample.
 
 # Documentation
 
 You can see the latest documentation in [rubydoc.info](http://www.rubydoc.info/github/sciruby/statsample/master).
 
+# Usage
+
+## Notebooks
+
+You can see some iruby notebooks here:
+
+### Statistics
+
+* [Correlation Matrix with daru and statsample](http://nbviewer.ipython.org/github/SciRuby/sciruby-notebooks/blob/master/Statistics/Correlation%20Matrix%20with%20daru%20and%20statsample.ipynb)
+* [Dominance Analysis with statsample](http://nbviewer.ipython.org/github/SciRuby/sciruby-notebooks/blob/master/Statistics/Dominance%20Analysis%20with%20statsample.ipynb)
+* [Reliability ICC](http://nbviewer.ipython.org/github/v0dro/sciruby-notebooks/blob/master/Statistics/Reliability%20ICC%20with%20statsample.ipynb)
+* [Levene Test](http://nbviewer.ipython.org/github/v0dro/sciruby-notebooks/blob/master/Statistics/Levene%20Test.ipynb)
+* [Multiple Regression](http://nbviewer.ipython.org/github/v0dro/sciruby-notebooks/blob/master/Statistics/Multiple%20Regression.ipynb)
+* [Parallel Analysis on PCA](http://nbviewer.ipython.org/github/SciRuby/sciruby-notebooks/blob/master/Statistics/Parallel%20Analysis%20on%20PCA.ipynb)
+* [Polychoric Analysis](http://nbviewer.ipython.org/github/SciRuby/sciruby-notebooks/blob/master/Statistics/Polychoric%20Correlation.ipynb)
+* [Reliability Scale and Multiscale Analysis](https://github.com/SciRuby/sciruby-notebooks/blob/master/Statistics/Reliability%20Scale%20Analysis.ipynb)
+* [Velicer MAP Test](http://nbviewer.ipython.org/github/SciRuby/sciruby-notebooks/blob/master/Statistics/Velicer%20MAP%20test.ipynb)
+
+### Visualizations
+
+* [Creating Boxplots with daru and statsample](http://nbviewer.ipython.org/github/SciRuby/sciruby-notebooks/blob/master/Visualization/Boxplot%20with%20daru%20and%20statsample.ipynb)
+* [Creating A Histogram](http://nbviewer.ipython.org/github/SciRuby/sciruby-notebooks/blob/master/Visualization/Creating%20a%20Histogram.ipynb)
+* [Creating a Scatterplot](http://nbviewer.ipython.org/github/SciRuby/sciruby-notebooks/blob/master/Visualization/Scatterplot%20with%20statsample.ipynb)
+
+### Working with DataFrame and Vector
+
+* [Creating Vectors and DataFrames with daru](http://nbviewer.ipython.org/github/SciRuby/sciruby-notebooks/blob/master/Data%20Analysis/Creation%20of%20Vector%20and%20DataFrame.ipynb)
+* [Detailed Usage of Daru::Vector](http://nbviewer.ipython.org/github/SciRuby/sciruby-notebooks/blob/master/Data%20Analysis/Usage%20of%20Vector.ipynb)
+* [Detailed Usage of Daru::DataFrame](http://nbviewer.ipython.org/github/SciRuby/sciruby-notebooks/blob/master/Data%20Analysis/Usage%20of%20DataFrame.ipynb)
+* [Visualizing Data with Daru::DataFrame](http://nbviewer.ipython.org/github/SciRuby/sciruby-notebooks/blob/master/Visualization/Visualizing%20data%20with%20daru%20DataFrame.ipynb)
+
+## Examples
+
+See the /examples directory for some use cases. The notebooks listed above have mostly
+the same examples, and they look better so you might want to see that first.
+
 # Description
 
 A suite for basic and advanced statistics on Ruby. Tested on CRuby 1.9.3, 2.0.0 and 2.1.1. See `.travis.yml` for more information.
 
 Include:
 - Descriptive statistics: frequencies, median, mean, standard error, skew, kurtosis (and many others).
-- Imports and exports datasets from and to Excel, CSV and plain text files.
 - Correlations: Pearson's r, Spearman's rank correlation (rho), point biserial, tau a, tau b and  gamma.  Tetrachoric and Polychoric correlation provides by +statsample-bivariate-extension+ gem.
 - Intra-class correlation
 - Anova: generic and vector-based One-way ANOVA and Two-way ANOVA, with contrasts for One-way ANOVA.
@@ -75,8 +113,7 @@ Include:
 # Features
 
 - Classes for manipulation and storage of data:
-  - Statsample::Vector: An extension of an array, with statistical methods like sum, mean and standard deviation
-  - Statsample::Dataset: a group of Statsample::Vector, analog to a excel spreadsheet or a dataframe on R. The base of almost all operations on statsample.
+  - Uses [daru](https://github.com/v0dro/daru) for storing data and basic statistics.
   - Statsample::Multiset: multiple datasets with same fields and type of vectors
 - Anova module provides generic Statsample::Anova::OneWay and vector based Statsample::Anova::OneWayWithVectors. Also you can create contrast using Statsample::Anova::Contrast
 - Module Statsample::Bivariate provides covariance and pearson, spearman, point biserial, tau a, tau b, gamma, tetrachoric (see Bivariate::Tetrachoric) and polychoric (see Bivariate::Polychoric) correlations. Include methods to create correlation and covariance matrices
@@ -100,10 +137,7 @@ Include:
   - Statsample::DominanceAnalysis class can report dominance analysis for a sample, using uni or multivariate dependent variables
   - Statsample::DominanceAnalysis::Bootstrap can execute bootstrap analysis to determine dominance stability, as recomended by  Azen & Budescu (2003) link[http://psycnet.apa.org/journals/met/8/2/129/].
 - Module Statsample::Codification, to help to codify open questions
-- Converters to import and export data:
-  - Statsample::Database : Can create sql to create tables, read and insert data
-  - Statsample::CSV : Read and write CSV files
-  - Statsample::Excel : Read and write Excel files
+- Converters to export data:
   - Statsample::Mx    : Write Mx Files
   - Statsample::GGobi : Write Ggobi files
 - Module Statsample::Crosstab provides function to create crosstab for categorical data
@@ -130,52 +164,6 @@ Include:
 - Gem <tt>statsample-glm</tt> provides you with GML method, to work with Logistic, Poisson and Gaussian regression ,using ML or IRWLS.
 - Close integration with gem <tt>reportbuilder</tt>, to easily create reports on text, html and rtf formats.
 
-# Usage
-
-See the [examples folder](https://github.com/clbustos/statsample/tree/master/examples/) too.
-
-## Boxplot
-
-```ruby
-require 'statsample'
-
-ss_analysis(Statsample::Graph::Boxplot) do
-  n = 30
-  a = rnorm(n-1, 50, 10)
-  b = rnorm(n, 30, 5)
-  c = rnorm(n, 5, 1)
-  a.push(2)
-  boxplot(vectors: [a, b, c],
-          width: 300,
-          height: 300,
-          groups: %w{first first second},
-          minimum: 0)
-end
-
-Statsample::Analysis.run # Open svg file on *nix application defined
-```
-
-## Correlation matrix
-
-```ruby
-require 'statsample'
-# Note R like generation of random gaussian variable
-# and correlation matrix
-
-ss_analysis("Statsample::Bivariate.correlation_matrix") do
-  samples = 1000
-  ds = data_frame(
-    'a' => rnorm(samples),
-    'b' => rnorm(samples),
-    'c' => rnorm(samples),
-    'd' => rnorm(samples))
-  cm = cor(ds)
-  summary(cm)
-end
-
-Statsample::Analysis.run_batch # Echo output to console
-```
-
 # Resources
 
 - Source code on github :: http://github.com/sciruby/statsample

data/benchmarks/correlation_matrix_15_variables.rb

@@ -4,7 +4,6 @@ extend BenchPress
 cases=250
 vars=20
 
-
 name "gsl matrix based vs. manual ruby correlation matrix (#{vars} vars, #{cases} cases)"
 author 'Clbustos'
 date '2011-01-18'
@@ -17,10 +16,12 @@ In this test, we test the calculation using #{vars} variables with
 
 reps 200 #number of repetitions
 
-ds=vars.times.inject({}) {|ac,v|
-ac["x#{v}"]=Statsample::Vector.new_numeric(cases) {rand()}
-ac
-}.to_dataset
+ds = Daru::DataFrame.new(
+  vars.times.inject({}) do |ac,v|
+    ac["x#{v}".to_sym]=Daru::Vector.new_with_size(cases) {rand()}
+    ac
+  end
+)
     
 measure "Statsample::Bivariate.correlation_matrix_optimized" do
   Statsample::Bivariate.correlation_matrix_optimized(ds)

data/benchmarks/correlation_matrix_5_variables.rb

@@ -17,11 +17,12 @@ In this test, we test the calculation using #{vars} variables with
 
 reps 200 #number of repetitions
 
-
-ds=vars.times.inject({}) {|ac,v|
-ac["x#{v}"]=Statsample::Vector.new_numeric(cases) {rand()}
-ac
-}.to_dataset
+ds = Daru::DataFrame.new(
+  vars.times.inject({}) do |ac,v|
+    ac["x#{v}".to_sym]=Daru::Vector.new_with_size(cases) {rand()}
+    ac
+  end
+)
     
 measure "Statsample::Bivariate.correlation_matrix_optimized" do
   Statsample::Bivariate.correlation_matrix_optimized(ds)

data/benchmarks/correlation_matrix_methods/correlation_matrix.rb

@@ -5,11 +5,13 @@ require 'statsample'
 require 'benchmark'
 
 def create_dataset(vars,cases) 
-  ran=Distribution::Normal.rng
-  ds=vars.times.inject({}) {|ac,v|
-    ac["x#{v}"]=Statsample::Vector.new_numeric(cases) {ran.call}
-  ac
-  }.to_dataset
+  ran = Distribution::Normal.rng
+  ds  = Daru::DataFrame.new(
+    vars.times.inject({}) do |ac,v|
+      ac["x#{v}".to_sym] = Daru::Vector.new_with_size(cases) {ran.call}
+      ac
+    end
+  )
 end
 
 def prediction_pairwise(vars,cases)
@@ -19,19 +21,17 @@ def prediction_optimized(vars,cases)
 	Statsample::Bivariate.prediction_optimized(vars,cases) / 10
 end
 
-
-
 if !File.exists?("correlation_matrix.ds") or File.mtime(__FILE__) > File.mtime("correlation_matrix.ds")
 reps=100 #number of repetitions
 ds_sizes=[5,10,30,50,100,150,200,500,1000]
 ds_vars=[3,4,5,10,20,30,40]
 #ds_sizes=[5,10]
 #ds_vars=[3,5,20]
-rs=Statsample::Dataset.new(%w{cases vars time_optimized time_pairwise})
+rs = Daru::DataFrame.new({}, order: [:cases, :vars, :time_optimized, :time_pairwise])
 
 ds_sizes.each do |cases|
   ds_vars.each do |vars|
-      ds=create_dataset(vars,cases)
+      ds = create_dataset(vars,cases)
       time_optimized= Benchmark.realtime do
         reps.times { 
         Statsample::Bivariate.correlation_matrix_optimized(ds) 
@@ -40,36 +40,33 @@ ds_sizes.each do |cases|
       end
       
       time_pairwise= Benchmark.realtime do
-        reps.times { 
-        Statsample::Bivariate.correlation_matrix_pairwise(ds)
-        }
+        reps.times { Statsample::Bivariate.correlation_matrix_pairwise(ds) }
       end
       
       puts "Cases:#{cases}, vars:#{vars} -> opt:%0.3f (%0.3f) | pair: %0.3f (%0.3f)" % [time_optimized, prediction_optimized(vars,cases), time_pairwise, prediction_pairwise(vars,cases)]
       
-      rs.add_case({'cases'=>cases,'vars'=>vars,'time_optimized'=>Math.sqrt(time_optimized*1000),'time_pairwise'=>Math.sqrt(time_pairwise*1000)})
+      rs.add_row(Daru::Vector.new({
+        :cases          => cases,
+        :vars           => vars,
+        :time_optimized => Math.sqrt(time_optimized*1000),
+        :time_pairwise  =>Math.sqrt(time_pairwise*1000)
+        })
+      )
     end
-  end
-  
+  end 
 else
   rs=Statsample.load("correlation_matrix.ds")
 end
 
+rs[:c_v] = rs.collect {|row| row[:cases]*row[:vars]}
 
-rs.fields.each {|f| rs[f].type=:numeric}
-
-rs['c_v']=rs.collect {|row| row['cases']*row['vars']}
-
-rs.update_valid_data
+rs.update
 rs.save("correlation_matrix.ds")
 Statsample::Excel.write(rs,"correlation_matrix.xls")
 
+rb = ReportBuilder.new(:name=>"Correlation matrix analysis")
 
-
-rb=ReportBuilder.new(:name=>"Correlation matrix analysis")
-
-rb.add(Statsample::Regression.multiple(rs[['cases','vars','time_optimized','c_v']],'time_optimized', :digits=>6))
-rb.add(Statsample::Regression.multiple(rs[['cases','vars','time_pairwise','c_v']],'time_pairwise', :digits=>6))
-
+rb.add(Statsample::Regression.multiple(rs[:cases,:vars,:time_optimized,:c_v],:time_optimized, :digits=>6))
+rb.add(Statsample::Regression.multiple(rs[:cases,:vars,:time_pairwise,:c_v],:time_pairwise, :digits=>6))
 
 rb.save_html("correlation_matrix.html")

data/examples/boxplot.rb

@@ -1,14 +1,26 @@
 #!/usr/bin/ruby
+# == Description
+# 
+# This example illustrates how daru, combined with Statsample::Graph::Boxplot
+# can be used for generating box plots of a normally distributed set of data.
+# 
+# The 'rnorm' function, defined in statsample/shorthands generates a Daru::Vector
+# object which contains the specified number of random variables in a normal distribution.
+# It uses the 'distribution' gem for this purpose.
+# 
+# Create a boxplot of the data by specifying the vectors a, b and c and providing 
+# necessary options to Statsample::Graph::Boxplot. The 'boxplot' function is shorthand
+# for calling Statsample::Graph::Boxplot.
 $:.unshift(File.dirname(__FILE__)+'/../lib/')
 require 'statsample'
 Statsample::Analysis.store(Statsample::Graph::Boxplot) do 
-  n=30
-  a=rnorm(n-1,50,10)
-  b=rnorm(n, 30,5)
-  c=rnorm(n,5,1)
+  n = 30
+  a = rnorm(n-1,50,10)
+  b = rnorm(n, 30,5)
+  c = rnorm(n,5,1)
   a.push(2)
+
   boxplot(:vectors=>[a,b,c],:width=>300, :height=>300, :groups=>%w{first first second}, :minimum=>0)
-  
 end
 
 if __FILE__==$0

data/examples/correlation_matrix.rb

@@ -1,16 +1,45 @@
 #!/usr/bin/ruby
+
+# == Description
+# 
+# Creating and summarizing a correlation matrix with daru and statsample
 $:.unshift(File.dirname(__FILE__)+'/../lib/')
-require 'statsample'
 
+require 'statsample'
 Statsample::Analysis.store("Statsample::Bivariate.correlation_matrix") do
+  # It so happens that Daru::Vector and Daru::DataFrame must update metadata
+  # like positions of missing values every time they are created. 
+  #
+  # Since we dont have any missing values in the data that we are creating, 
+  # we set Daru.lazy_update = true so that missing data is not updated every
+  # time and things happen much faster.
+  #
+  # In case you do have missing data and lazy_update has been set to *true*, 
+  # you _SHOULD_ called `#update` on the concerned Vector or DataFrame object
+  # everytime an assingment or deletion cycle is complete.
+  Daru.lazy_update = true
+  
+  # Create a Daru::DataFrame containing 4 vectors a, b, c and d.
+  #
+  # Notice that the `clone` option has been set to *false*. This tells Daru
+  # to not clone the Daru::Vectors being supplied by `rnorm`, since it would
+  # be unnecessarily counter productive to clone the vectors once they have
+  # been assigned to the dataframe.
   samples=1000
-  ds=data_frame(
-    'a'=>rnorm(samples),
-    'b'=>rnorm(samples),
-    'c'=>rnorm(samples),
-    'd'=>rnorm(samples))
-  cm=cor(ds)
+  ds = Daru::DataFrame.new({
+    :a => rnorm(samples),
+    :b => rnorm(samples),
+    :c => rnorm(samples),
+    :d => rnorm(samples)
+  }, clone: false)
+
+  # Calculate correlation matrix by calling the `cor` shorthand.
+  cm = cor(ds)
   summary(cm)
+
+  # Set lazy_update to *false* once our job is done so that this analysis does
+  # not accidentally affect code elsewhere.
+  Daru.lazy_update = false
 end
 
 if __FILE__==$0