statsample 2.0.2 → 2.1.0

data/lib/statsample/matrix.rb

@@ -21,7 +21,6 @@ class ::Matrix
     row_size.times {|i|
       ds.add_row(self.row(i).to_a)
     }
-    ds.update
     ds.rename(self.name) if self.respond_to? :name
     ds
   end
@@ -95,7 +94,6 @@ module GSL
       row_size.times {|i|
         ds.add_row(self.row(i).to_a)
       }
-      ds.update
       ds.rename(self.name) if self.respond_to? :name
       ds
     end

data/lib/statsample/regression.rb

@@ -15,8 +15,6 @@ module Statsample
     # 
     # * Simple Regression :  Statsample::Regression::Simple
     # * Multiple Regression: Statsample::Regression::Multiple
-    # * Logit Regression:    Statsample::Regression::Binomial::Logit
-    # * Probit Regression:    Statsample::Regression::Binomial::Probit
     module Regression
       
       LinearDependency=Class.new(Exception)
@@ -58,7 +56,7 @@ module Statsample
           if Statsample.has_gsl? and false
             Statsample::Regression::Multiple::GslEngine.new(ds, y_var, opts)
           else
-            ds2=ds.dup_only_valid
+            ds2=ds.reject_values(*Daru::MISSING_VALUES)
             Statsample::Regression::Multiple::RubyEngine.new(ds2,y_var, opts)
           end
         end

data/lib/statsample/regression/multiple/alglibengine.rb

@@ -19,7 +19,7 @@ module Multiple
 class AlglibEngine < BaseEngine
   def initialize(ds,y_var, opts=Hash.new)
     super    
-    @ds       = ds.dup_only_valid
+    @ds       = ds.reject_values(*Daru::MISSING_VALUES)
     @ds_valid = @ds
     @dy       = @ds[@y_var]
     @ds_indep = ds.dup(ds.vectors.to_a - [y_var])

data/lib/statsample/regression/multiple/gslengine.rb

@@ -19,7 +19,7 @@ if Statsample.has_gsl?
         class GslEngine < BaseEngine
           def initialize(ds,y_var, opts=Hash.new)
             super
-            @ds          = ds.dup_only_valid
+            @ds          = ds.reject_values(*Daru::MISSING_VALUES)
             @ds_valid    = @ds
             @valid_cases = @ds_valid.nrows
             @dy          = @ds[@y_var]

data/lib/statsample/regression/multiple/rubyengine.rb

@@ -30,7 +30,7 @@ class RubyEngine < MatrixEngine
     super(matrix, y_var, opts)
     @ds = ds
     @dy = ds[@y_var]
-    @ds_valid = ds.dup_only_valid
+    @ds_valid = ds.reject_values(*Daru::MISSING_VALUES)
     @total_cases = @ds.nrows
     @valid_cases = @ds_valid.nrows
     @ds_indep    = ds.dup(ds.vectors.to_a - [y_var])
@@ -55,7 +55,6 @@ class RubyEngine < MatrixEngine
       end
       i += 1
     end
-    @ds_indep.update
     set_dep_columns
   end
   def fix_with_regression
@@ -75,7 +74,6 @@ class RubyEngine < MatrixEngine
       end
       i+=1
     end
-    @ds_indep.update
     set_dep_columns
   end
   # Standard error for constant

data/lib/statsample/reliability.rb

@@ -4,10 +4,10 @@ module Statsample
       # Calculate Chonbach's alpha for a given dataset.
       # only uses tuples without missing data
       def cronbach_alpha(ods)
-        ds = ods.dup_only_valid
+        ds = ods.reject_values(*Daru::MISSING_VALUES)
         n_items = ds.ncols
         return nil if n_items <= 1
-        s2_items = ds.to_hash.values.inject(0) { |ac,v| 
+        s2_items = ds.to_h.values.inject(0) { |ac,v| 
           ac + v.variance }
         total    = ds.vector_sum
         
@@ -18,7 +18,7 @@ module Statsample
       # Only uses tuples without missing data
       # Return nil if one or more vectors has 0 variance
       def cronbach_alpha_standarized(ods)
-        ds = ods.dup_only_valid
+        ds = ods.reject_values(*Daru::MISSING_VALUES)
         return nil if ds.any? { |v| v.variance==0}
         
         ds = Daru::DataFrame.new(

data/lib/statsample/reliability/icc.rb

@@ -96,8 +96,7 @@ module Statsample
       attr_accessor :alpha
       attr_accessor :name
       def initialize(ds, opts=Hash.new)
-        ds.update
-        @ds=ds.dup_only_valid
+        @ds=ds.reject_values(*Daru::MISSING_VALUES)
         @vectors=@ds.map { |e| e }
         @n=@ds.nrows
         @k=@ds.ncols

data/lib/statsample/reliability/multiscaleanalysis.rb

@@ -128,7 +128,6 @@ module Statsample
           ds[code.to_sym] = scale.ds.vector_sum
         end
         
-        ds.update
         ds
       end
 

data/lib/statsample/reliability/scaleanalysis.rb

@@ -21,7 +21,7 @@ module Statsample
         }
         
         @ods = ds
-        @ds  = ds.dup_only_valid(ds.vectors.to_a - @dumped)
+        @ds  = ds.reject_values(*Daru::MISSING_VALUES).dup(ds.vectors.to_a - @dumped)
         @ds.rename ds.name
         
         @k     = @ds.ncols
@@ -117,11 +117,10 @@ module Statsample
         ds_new = Daru::DataFrame.new({}, order: ([:case,:score] + dif_sort.collect{|a,b| a.to_sym}))
         scores_sort.each do |i,score|
           row = [i, score]
-          case_row = @ds.row[i].to_hash
+          case_row = @ds.row[i].to_h
           dif_sort.each{ |variable,dif_value| row.push(case_row[variable]) }
           ds_new.add_row(row)
         end
-        ds_new.update
         ds_new
       end
       

data/lib/statsample/reliability/skillscaleanalysis.rb

@@ -63,7 +63,7 @@ module Statsample
             out = {}
             row.each_with_index do |v, k|
               if @key.has_key? k
-                if @ds[k].exists? v
+                if @ds[k].reject_values(*Daru::MISSING_VALUES).include_values? v
                   out[k]= @key[k] == v ? 1 : 0
                 else
                   out[k] = nil

data/lib/statsample/test/levene.rb

@@ -30,7 +30,7 @@ module Statsample
       # Input could be an array of vectors or a dataset
       def initialize(input, opts=Hash.new())
         if input.is_a? Daru::DataFrame
-          @vectors = input.to_hash.values
+          @vectors = input.to_h.values
         else
           @vectors = input
         end
@@ -48,7 +48,7 @@ module Statsample
         builder.text "%s : F(%d, %d) = %0.4f , p = %0.4f" % [@name, @d1, @d2, f, probability]
       end
       def compute
-        n=@vectors.inject(0) { |ac,v| ac + v.n_valid}
+        n=@vectors.inject(0) { |ac,v| ac + v.reject_values(*Daru::MISSING_VALUES).size }
         
         zi=@vectors.collect do |vector|
           mean=vector.mean
@@ -57,7 +57,7 @@ module Statsample
         
         total_mean = Daru::Vector.new(
           zi.inject([]) do |ac,vector|
-            ac + vector.only_valid(:array)
+            ac + vector.reject_values(*Daru::MISSING_VALUES).to_a
           end
         ).mean
       
@@ -68,7 +68,7 @@ module Statsample
         
         sum_den = zi.inject(0) do |ac,vector|
           z_mean = vector.mean
-          ac + vector.only_valid(:array).inject(0) do |acp,zij|
+          ac + vector.reject_values(*Daru::MISSING_VALUES).to_a.inject(0) do |acp,zij|
             acp + (zij - z_mean)**2
           end
         end

data/lib/statsample/test/t.rb

@@ -163,11 +163,11 @@ module Statsample
           @u=@opts[:u]
           @tails=@opts[:tails]
           @confidence_level=@opts[:confidence_level] || 0.95
-          @df= @vector.n_valid-1
+          @df= @vector.reject_values(*Daru::MISSING_VALUES).size-1
           @t=nil
         end        
         def t_object
-          T.new(@vector.mean-u, @vector.se, @vector.n_valid-1, opts)
+          T.new(@vector.mean-u, @vector.se, @vector.reject_values(*Daru::MISSING_VALUES).size-1, opts)
         end
         def t
           t_object.t
@@ -264,12 +264,12 @@ module Statsample
        
         # Set t and probability for given u
         def compute
-          @t_equal_variance= T.two_sample_independent(@v1.mean, @v2.mean, @v1.sd, @v2.sd, @v1.n_valid, @v2.n_valid,true)
+          @t_equal_variance= T.two_sample_independent(@v1.mean, @v2.mean, @v1.sd, @v2.sd, @v1.reject_values(*Daru::MISSING_VALUES).size, @v2.reject_values(*Daru::MISSING_VALUES).size,true)
           
-          @t_not_equal_variance= T.two_sample_independent(@v1.mean, @v2.mean, @v1.sd, @v2.sd, @v1.n_valid, @v2.n_valid, false)
+          @t_not_equal_variance= T.two_sample_independent(@v1.mean, @v2.mean, @v1.sd, @v2.sd, @v1.reject_values(*Daru::MISSING_VALUES).size, @v2.reject_values(*Daru::MISSING_VALUES).size, false)
 
-          @df_equal_variance=T.df_equal_variance(@v1.n_valid, @v2.n_valid)
-          @df_not_equal_variance=T.df_not_equal_variance(@v1.sd, @v2.sd, @v1.n_valid, @v2.n_valid)
+          @df_equal_variance=T.df_equal_variance(@v1.reject_values(*Daru::MISSING_VALUES).size, @v2.reject_values(*Daru::MISSING_VALUES).size)
+          @df_not_equal_variance=T.df_not_equal_variance(@v1.sd, @v2.sd, @v1.reject_values(*Daru::MISSING_VALUES).size, @v2.reject_values(*Daru::MISSING_VALUES).size)
           
           @probability_equal_variance = p_using_cdf(Distribution::T.cdf(@t_equal_variance, @df_equal_variance), tails)
           
@@ -278,8 +278,8 @@ module Statsample
         end
         # Cohen's d is a measure of effect size. Its defined as the difference between two means divided by a standard deviation for the data
         def d
-          n1=@v1.n_valid
-          n2=@v2.n_valid
+          n1=@v1.reject_values(*Daru::MISSING_VALUES).size
+          n2=@v2.reject_values(*Daru::MISSING_VALUES).size
           num=@v1.mean-@v2.mean
           den=Math::sqrt( ((n1-1)*@v1.sd+(n2-1)*@v2.sd).quo(n1+n2))
           num.quo(den)
@@ -288,8 +288,8 @@ module Statsample
         def report_building(b) # :nodoc:
           b.section(:name=>@name) {|g|
             g.table(:name=>_("Mean and standard deviation"), :header=>[_("Variable"), _("mean"), _("sd"),_("n")]) {|t|
-              t.row([@v1.name,"%0.4f" % @v1.mean,"%0.4f" % @v1.sd, @v1.n_valid])
-              t.row([@v2.name,"%0.4f" % @v2.mean,"%0.4f" % @v2.sd, @v2.n_valid])
+              t.row([@v1.name,"%0.4f" % @v1.mean,"%0.4f" % @v1.sd, @v1.reject_values(*Daru::MISSING_VALUES).size])
+              t.row([@v2.name,"%0.4f" % @v2.mean,"%0.4f" % @v2.sd, @v2.reject_values(*Daru::MISSING_VALUES).size])
             }
             g.parse_element(Statsample::Test.levene([@v1,@v2],:name=>_("Levene test for equality of variances")))
             

data/lib/statsample/test/umannwhitney.rb

@@ -118,8 +118,8 @@ module Statsample
       def initialize(v1,v2, opts=Hash.new)
         @v1      = v1
         @v2      = v2
-        v1_valid = v1.only_valid.reset_index!
-        v2_valid = v2.only_valid.reset_index!
+        v1_valid = v1.reject_values(*Daru::MISSING_VALUES).reset_index!
+        v2_valid = v2.reject_values(*Daru::MISSING_VALUES).reset_index!
         @n1      = v1_valid.size
         @n2      = v2_valid.size
         data     = Daru::Vector.new(v1_valid.to_a + v2_valid.to_a)
@@ -172,7 +172,7 @@ module Statsample
       # == Reference: 
       # * http://europe.isixsigma.com/library/content/c080806a.asp
       def adjust_for_ties(data)
-        @t = data.frequencies.find_all { |k,v| v > 1 }.inject(0) { |a,v|
+        @t = data.frequencies.to_h.find_all { |k,v| v > 1 }.inject(0) { |a,v|
           a + (v[1]**3 - v[1]).quo(12)
         }        
       end

data/lib/statsample/version.rb

@@ -1,3 +1,3 @@
 module Statsample
-  VERSION = '2.0.2'
+  VERSION = '2.1.0'
 end

data/statsample.gemspec

@@ -60,7 +60,7 @@ Gem::Specification.new do |s|
   s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
 
-  s.add_runtime_dependency 'daru', '~> 0.1'
+  s.add_runtime_dependency 'daru', '~> 0.1.6'
   s.add_runtime_dependency 'spreadsheet', '~> 1.1'
   s.add_runtime_dependency 'reportbuilder', '~> 1.4'
   s.add_runtime_dependency 'minimization', '~> 0.2'
@@ -81,4 +81,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'mocha', '~> 1.1'
   s.add_development_dependency 'nmatrix', '~> 0.2.1'
   s.add_development_dependency 'gsl', '~> 2.1'
+  s.add_development_dependency 'pry'
+  s.add_development_dependency 'rubocop'
+  s.add_development_dependency 'activesupport', '~> 4.2'
 end

data/test/fixtures/df.csv

@@ -0,0 +1,15 @@
+y,a,b,c,d,e
+0,6,62.1,no,female,A
+1,18,34.7,yes,male,B
+1,6,29.7,no,female,C
+0,4,71,no,male,C
+1,5,36.9,yes,male,B
+0,11,58.7,no,female,B
+0,8,63.3,no,male,B
+1,21,20.4,yes,male,A
+1,2,20.5,yes,male,C
+0,11,59.2,no,male,B
+0,1,76.4,yes,female,A
+0,8,71.7,no,female,B
+1,2,77.5,no,male,C
+1,3,31.1,no,male,B

data/test/helpers_tests.rb

@@ -30,6 +30,13 @@ module Minitest
   end
 
   module Assertions
+    def assert_vectors_from_formula(formula, names)
+      model = Statsample::FitModel.new formula, @df
+  
+      model.df_for_regression.vectors.to_a.sort
+        .must_equal names.sort    
+    end
+
     def assert_similar_vector(exp, obs, delta = 1e-10, msg = nil)
       msg ||= "Different vectors #{exp} - #{obs}"
       assert_equal(exp.size, obs.size)

data/test/test_factor.rb

@@ -7,19 +7,14 @@ class StatsampleFactorTestCase < Minitest::Test
   # Based on Hardle and Simar
   def setup
     @fixtures_dir = File.expand_path(File.dirname(__FILE__) + '/fixtures')
-    Daru.lazy_update = true
   end
 
-  def teardown
-    Daru.lazy_update = false
-  end
   # Based on Hurdle example
   def test_covariance_matrix
     ds = Daru::DataFrame.from_plaintext(@fixtures_dir + '/bank2.dat', [:v1,:v2,:v3,:v4,:v5,:v6])
     ds.vectors.each {|f|
       ds[f] = ds[f].center
     }
-    ds.update
     cm = Statsample::Bivariate.covariance_matrix ds
     pca = Statsample::Factor::PCA.new(cm, m: 6)
     # puts pca.summary

data/test/test_factor_pa.rb

@@ -7,11 +7,6 @@ class StatsampleFactorTestCase < Minitest::Test
   # Based on Hardle and Simar
   def setup
     @fixtures_dir = File.expand_path(File.dirname(__FILE__) + '/fixtures')
-    Daru.lazy_update = true
-  end
-
-  def teardown
-    Daru.lazy_update = false
   end
 
   def test_parallelanalysis_with_data
@@ -44,7 +39,7 @@ class StatsampleFactorTestCase < Minitest::Test
       pa2 = Statsample::Factor::ParallelAnalysis.with_random_data(samples, variables, iterations: iterations, percentil: 95)
       3.times do |n|
         var = "ev_0000#{n + 1}".to_sym
-        assert_in_delta(pa1.ds_eigenvalues[var].mean, pa2.ds_eigenvalues[var].mean, 0.05)
+        assert_in_delta(pa1.ds_eigenvalues[var].mean, pa2.ds_eigenvalues[var].mean, 0.07)
       end
     else
       skip('Too slow without GSL')

data/test/test_fit_model.rb

@@ -0,0 +1,88 @@
+require(File.expand_path(File.dirname(__FILE__) + '/helpers_tests.rb'))
+require 'minitest/autorun'
+
+describe Statsample::FitModel do
+  before do
+    @df = Daru::DataFrame.from_csv 'test/fixtures/df.csv'
+    @df.to_category 'c', 'd', 'e'
+  end
+  context '#df_for_regression' do
+    context 'no interaction' do
+      it { assert_vectors_from_formula 'y~a+e', %w[a e_B e_C y] }
+    end
+
+    context '2-way interaction' do
+      context 'interaction of numerical with numerical' do
+        context 'none reoccur' do
+          it { assert_vectors_from_formula 'y~a:b', %w[a:b y] }
+        end
+        
+        context 'one reoccur' do
+          it { assert_vectors_from_formula 'y~a+a:b', %w[a a:b y] }
+        end
+  
+        context 'both reoccur' do
+          it { assert_vectors_from_formula 'y~a+b+a:b', %w[a a:b b y] }
+        end        
+      end
+  
+      context 'interaction of category with numerical' do
+        context 'none reoccur' do
+          it { assert_vectors_from_formula 'y~a:e', %w[e_A:a e_B:a e_C:a y] }
+        end
+  
+        context 'one reoccur' do
+          context 'numeric occur' do
+            it { assert_vectors_from_formula 'y~a+a:e', %w[a e_B:a e_C:a y] }
+          end
+  
+          context 'category occur' do
+            it { assert_vectors_from_formula 'y~e+a:e',
+              %w[e_B e_C e_A:a e_B:a e_C:a y] }
+          end  
+        end        
+        
+        context 'both reoccur' do
+          it { assert_vectors_from_formula 'y~a+e+a:e',
+            %w[a e_B e_C e_B:a e_C:a y] }
+        end
+      end
+  
+      context 'interaction of category with category' do
+        context 'none reoccur' do
+          it { assert_vectors_from_formula 'y~c:e',
+            %w[e_B e_C c_yes:e_A c_yes:e_B c_yes:e_C y] }
+        end
+  
+        context 'one reoccur' do
+          it { assert_vectors_from_formula 'y~e+c:e',
+            %w[e_B e_C c_yes:e_A c_yes:e_B c_yes:e_C y] }
+        end
+  
+        context 'both reoccur' do
+          it { assert_vectors_from_formula 'y~c+e+c:e',
+            %w[c_yes e_B e_C c_yes:e_B c_yes:e_C y] }
+        end        
+      end
+    end
+
+    context 'corner case' do
+      context 'example 1' do
+        it { assert_vectors_from_formula 'y~d:a+d:e',
+          %w[e_B e_C d_male:e_A d_male:e_B d_male:e_C d_female:a d_male:a y] }
+      end
+    end
+    
+    context 'complex examples' do
+      context 'random example 1' do
+        it { assert_vectors_from_formula 'y~a+e+c:d+e:d',
+          %w[e_B e_C d_male c_yes:d_female c_yes:d_male e_B:d_male e_C:d_male a y] }
+      end
+      
+      context 'random example 2' do
+        it { assert_vectors_from_formula 'y~e+b+c+d:e+b:e+a:e+0',
+          %w[e_A e_B e_C c_yes d_male:e_A d_male:e_B d_male:e_C b e_B:b e_C:b e_A:a e_B:a e_C:a y] }
+      end
+    end
+  end
+end

data/test/test_reliability.rb

@@ -1,14 +1,6 @@
 require(File.expand_path(File.dirname(__FILE__) + '/helpers_tests.rb'))
 class StatsampleReliabilityTestCase < Minitest::Test
   context Statsample::Reliability do
-    setup do
-      Daru.lazy_update = true
-    end
-
-    teardown do
-      Daru.lazy_update = false
-    end
-
     should 'return correct r according to Spearman-Brown prophecy' do
       r = 0.6849
       n = 62.quo(15)
@@ -29,14 +21,12 @@ class StatsampleReliabilityTestCase < Minitest::Test
           @ds[i] = Daru::Vector.new(base.collect { |v| v + rand })
         end
 
-        @ds.update
         @k = @ds.ncols
         @cm = Statsample::Bivariate.covariance_matrix(@ds)
         @dse = @ds.dup
         @dse.vectors.each do |f|
           @dse[f] = @dse[f].standardize
         end
-        @dse.update
         @cme = Statsample::Bivariate.covariance_matrix(@dse)
         @a = Statsample::Reliability.cronbach_alpha(@ds)
         @as = Statsample::Reliability.cronbach_alpha_standarized(@ds)