statsample-ekatena 2.0.2

data/test/test_reliability_icc.rb

@@ -0,0 +1,198 @@
+require(File.expand_path(File.dirname(__FILE__) + '/helpers_tests.rb'))
+
+$reliability_icc = nil
+
+class StatsampleReliabilityIccTestCase < Minitest::Test
+  context Statsample::Reliability::ICC do
+    setup do
+      a = Daru::Vector.new([9, 6, 8, 7, 10, 6])
+      b = Daru::Vector.new([2, 1, 4, 1, 5, 2])
+      c = Daru::Vector.new([5, 3, 6, 2, 6, 4])
+      d = Daru::Vector.new([8, 2, 8, 6, 9, 7])
+      @ds = Daru::DataFrame.new({ :a => a, :b => b, :c => c, :d => d })
+      @icc = Statsample::Reliability::ICC.new(@ds)
+    end
+    should 'basic method be correct' do
+      assert_equal(6, @icc.n)
+      assert_equal(4, @icc.k)
+    end
+    should 'total mean be correct' do
+      assert_in_delta(5.291, @icc.total_mean, 0.001)
+    end
+    should 'df methods be correct' do
+      assert_equal(5,  @icc.df_bt)
+      assert_equal(18, @icc.df_wt)
+      assert_equal(3,  @icc.df_bj)
+      assert_equal(15, @icc.df_residual)
+    end
+    should 'ms between targets be correct' do
+      assert_in_delta(11.24, @icc.ms_bt, 0.01)
+    end
+    should 'ms within targets be correct' do
+      assert_in_delta(6.26,  @icc.ms_wt, 0.01)
+    end
+    should 'ms between judges be correct' do
+      assert_in_delta(32.49, @icc.ms_bj, 0.01)
+    end
+    should 'ms residual be correct' do
+      assert_in_delta(1.02,  @icc.ms_residual, 0.01)
+    end
+    context 'with McGraw and Wong denominations,' do
+    end
+    context 'with Shrout & Fleiss denominations, ' do
+      should 'icc(1,1) method be correct' do
+        assert_in_delta(0.17, @icc.icc_1_1, 0.01)
+      end
+      # Verified on SPSS and R
+      should 'icc(2,1) method be correct' do
+        assert_in_delta(0.29, @icc.icc_2_1, 0.01)
+      end
+      should 'icc(3,1) method be correct' do
+        assert_in_delta(0.71, @icc.icc_3_1, 0.01)
+      end
+      should 'icc(1,k) method be correct' do
+        assert_in_delta(0.44, @icc.icc_1_k, 0.01)
+      end
+      # Verified on SPSS and R
+      should 'icc(2,k) method be correct' do
+        assert_in_delta(0.62, @icc.icc_2_k, 0.01)
+      end
+      should 'icc(3,k) method be correct' do
+        assert_in_delta(0.91, @icc.icc_3_k, 0.01)
+      end
+
+      should 'icc(1,1) F be correct' do
+        assert_in_delta(1.795, @icc.icc_1_f.f)
+      end
+      should 'icc(1,1) confidence interval should be correct' do
+        assert_in_delta(-0.133, @icc.icc_1_1_ci[0], 0.001)
+        assert_in_delta(0.723, @icc.icc_1_1_ci[1], 0.001)
+      end
+      should 'icc(1,k) confidence interval should be correct' do
+        assert_in_delta(-0.884, @icc.icc_1_k_ci[0], 0.001)
+        assert_in_delta(0.912, @icc.icc_1_k_ci[1], 0.001)
+      end
+
+      should 'icc(2,1) F be correct' do
+        assert_in_delta(11.027, @icc.icc_2_f.f)
+      end
+      should 'icc(2,1) confidence interval should be correct' do
+        # skip("Not yet operational")
+        assert_in_delta(0.019, @icc.icc_2_1_ci[0], 0.001)
+        assert_in_delta(0.761, @icc.icc_2_1_ci[1], 0.001)
+      end
+
+      # Verified on SPSS and R
+      should 'icc(2,k) confidence interval should be correct' do
+        # skip("Not yet operational")
+        # p @icc.icc_2_k_ci
+        assert_in_delta(0.039, @icc.icc_2_k_ci[0], 0.001)
+        assert_in_delta(0.929, @icc.icc_2_k_ci[1], 0.001)
+      end
+      # should "Shrout icc(2,k) and McGraw icc(a,k) ci be equal" do
+      #  assert_in_delta(@icc.icc_2_k_ci_shrout[0], @icc.icc_2_k_ci_mcgraw[0], 10e-5)
+      # end
+
+      should 'icc(3,1) F be correct' do
+        assert_in_delta(11.027, @icc.icc_3_f.f)
+      end
+
+      should 'icc(3,1) confidence interval should be correct' do
+        assert_in_delta(0.342, @icc.icc_3_1_ci[0], 0.001)
+        assert_in_delta(0.946, @icc.icc_3_1_ci[1], 0.001)
+      end
+      should 'icc(3,k) confidence interval should be correct' do
+        assert_in_delta(0.676, @icc.icc_3_k_ci[0], 0.001)
+        assert_in_delta(0.986, @icc.icc_3_k_ci[1], 0.001)
+      end
+      should 'incorrect type raises an error' do
+        assert_raise(::RuntimeError) do
+          @icc.type = :nonexistant_type
+        end
+      end
+    end
+
+    begin
+      require 'rserve'
+      require 'daru/extensions/rserve'
+      context 'McGraw and Wong' do
+        teardown do
+          @r = $reliability_icc[:r].close unless $reliability_icc[:r].nil?
+        end
+        setup do
+          if $reliability_icc.nil?
+            size = 100
+            a = Daru::Vector.new(size.times.map { rand(10) })
+            b = a.recode { |i| i + rand(4) - 2 }
+            c = a.recode { |i| i + rand(4) - 2 }
+            d = a.recode { |i| i + rand(4) - 2 }
+            @ds = Daru::DataFrame.new({ :a => a, :b => b, :c => c, :d => d })
+
+            @icc = Statsample::Reliability::ICC.new(@ds)
+            @r = Rserve::Connection.new
+
+            @r.assign('ds', @ds)
+
+            @r.void_eval("library(irr);
+              iccs=list(
+              icc_1=icc(ds,'o','c','s'),
+              icc_k=icc(ds,'o','c','a'),
+              icc_c_1=icc(ds,'t','c','s'),
+              icc_c_k=icc(ds,'t','c','a'),
+              icc_a_1=icc(ds,'t','a','s'),
+              icc_a_k=icc(ds,'t','a','a'))
+              ")
+            @iccs = @r.eval('iccs').to_ruby
+            $reliability_icc = { icc: @icc, iccs: @iccs, r: @r
+            }
+
+          end
+          @icc = $reliability_icc[:icc]
+          @iccs = $reliability_icc[:iccs]
+          @r = $reliability_icc[:r]
+        end
+        [:icc_1, :icc_k, :icc_c_1, :icc_c_k, :icc_a_1, :icc_a_k].each do |t|
+          context "ICC Type #{t} " do
+            should 'value be correct' do
+              @icc.type = t
+              @r_icc = @iccs[t.to_s]
+              assert_in_delta(@r_icc['value'], @icc.r)
+            end
+            should 'fvalue be correct' do
+              @icc.type = t
+              @r_icc = @iccs[t.to_s]
+              assert_in_delta(@r_icc['Fvalue'], @icc.f.f)
+            end
+            should 'num df be correct' do
+              @icc.type = t
+              @r_icc = @iccs[t.to_s]
+              assert_in_delta(@r_icc['df1'], @icc.f.df_num)
+            end
+            should 'den df be correct' do
+              @icc.type = t
+              @r_icc = @iccs[t.to_s]
+              assert_in_delta(@r_icc['df2'], @icc.f.df_den)
+            end
+
+            should 'f probability be correct' do
+              @icc.type = t
+              @r_icc = @iccs[t.to_s]
+              assert_in_delta(@r_icc['p.value'], @icc.f.probability)
+            end
+            should 'bounds be equal' do
+              @icc.type = t
+              @r_icc = @iccs[t.to_s]
+              assert_in_delta(@r_icc['lbound'], @icc.lbound, 0.1)
+              assert_in_delta(@r_icc['ubound'], @icc.ubound, 0.1)
+            end
+            should 'summary generated' do
+              assert(@icc.summary.size > 0)
+            end
+          end
+        end
+      end
+    rescue
+      puts 'requires rserve'
+    end
+  end
+end

data/test/test_reliability_skillscale.rb

@@ -0,0 +1,57 @@
+require(File.expand_path(File.dirname(__FILE__) + '/helpers_tests.rb'))
+
+class StatsampleReliabilitySkillScaleTestCase < Minitest::Test
+  context Statsample::Reliability::SkillScaleAnalysis do
+    setup do
+      options = %w(a b c d e)
+      cases = 20
+      @id = Daru::Vector.new(cases.times.map { |v| v })
+      @a = Daru::Vector.new(cases.times.map { options[rand(5)] })
+      @b = Daru::Vector.new(cases.times.map { options[rand(5)] })
+      @c = Daru::Vector.new(cases.times.map { options[rand(5)] })
+      @d = Daru::Vector.new(cases.times.map { options[rand(5)] })
+      @e = Daru::Vector.new(
+        cases.times.map do |i|
+          i == 0 ? options[rand(0)] :
+          rand > 0.8 ? nil : options[rand(5)]
+        end
+      )
+      @ds = Daru::DataFrame.new({ :id => @id, :a => @a, :b => @b, :c => @c, :d => @d, :e => @e })
+      @key = { :a => 'a', :b => options[rand(5)], :c => options[rand(5)], :d => options[rand(5)], :e => options[rand(5)] }
+      @ssa = Statsample::Reliability::SkillScaleAnalysis.new(@ds, @key)
+      @ac = Daru::Vector.new(@a.map { |v| v == @key[:a] ? 1 : 0 })
+      @bc = Daru::Vector.new(@b.map { |v| v == @key[:b] ? 1 : 0 })
+      @cc = Daru::Vector.new(@c.map { |v| v == @key[:c] ? 1 : 0 })
+      @dc = Daru::Vector.new(@d.map { |v| v == @key[:d] ? 1 : 0 })
+      @ec = Daru::Vector.new(@e.map { |v| v.nil? ? nil : (v == @key[:e] ? 1 : 0) })
+    end
+    should 'return proper corrected dataset' do
+      cds = Daru::DataFrame.new({ :id => @id, :a => @ac, :b => @bc, :c => @cc, :d => @dc, :e => @ec })
+      assert_equal(cds, @ssa.corrected_dataset)
+    end
+    should 'return proper corrected minimal dataset' do
+      cdsm = Daru::DataFrame.new({ :a => @ac, :b => @bc, :c => @cc, :d => @dc, :e => @ec })
+      assert_equal(cdsm, @ssa.corrected_dataset_minimal)
+    end
+    should 'return correct vector_sum and vector_sum' do
+      cdsm = @ssa.corrected_dataset_minimal
+      assert_equal(cdsm.vector_sum, @ssa.vector_sum)
+      assert_equal(cdsm.vector_mean, @ssa.vector_mean)
+    end
+    should 'not crash on rare case' do
+      a = Daru::Vector.new(['c', 'c', 'a', 'a', 'c', 'a', 'b', 'c', 'c', 'b', 'a', 'd', 'a', 'd', 'a', 'a', 'd', 'e', 'c', 'd'])
+      b = Daru::Vector.new(['e', 'b', 'e', 'b', 'c', 'd', 'a', 'e', 'e', 'c', 'b', 'e', 'e', 'b', 'd', 'c', 'e', 'b', 'b', 'd'])
+      c = Daru::Vector.new(['e', 'b', 'e', 'c', 'e', 'c', 'b', 'd', 'e', 'c', 'a', 'a', 'b', 'd', 'e', 'c', 'b', 'a', 'a', 'e'])
+      d = Daru::Vector.new(['a', 'b', 'd', 'd', 'e', 'b', 'e', 'b', 'd', 'c', 'e', 'a', 'c', 'd', 'c', 'c', 'e', 'd', 'd', 'b'])
+      e = Daru::Vector.new(['a', 'b', nil, 'd', 'c', 'c', 'd', nil, 'd', 'd', 'e', 'e', nil, nil, nil, 'd', 'c', nil, 'e', 'd'])
+      key = { :a => 'a', :b => 'e', :c => 'd', :d => 'c', :e => 'd' }
+      ds = Daru::DataFrame.new({:a => a, :b => b, :c => c, :d => d, :e => e})
+      ssa = Statsample::Reliability::SkillScaleAnalysis.new(ds, key)
+      assert(ssa.summary)
+    end
+
+    should 'return valid summary' do
+      assert(@ssa.summary.size > 0)
+    end
+  end
+end

data/test/test_resample.rb

@@ -0,0 +1,24 @@
+require(File.expand_path(File.dirname(__FILE__) + '/helpers_tests.rb'))
+
+class StatsampleResampleTestCase < Minitest::Test
+  def initialize(*args)
+    super
+  end
+
+  def test_basic
+    r = Statsample::Resample.generate(20, 1, 10)
+    assert_equal(20, r.size)
+    assert(r.min >= 1)
+    assert(r.max <= 10)
+  end
+
+  def test_repeat_and_save
+    r = Statsample::Resample.repeat_and_save(400) {
+      Statsample::Resample.generate(20, 1, 10).count(1)
+    }
+    assert_equal(400, r.size)
+    v = Daru::Vector.new(r)
+    a = v.count { |x|  x > 3 }
+    assert(a >= 30 && a <= 70)
+  end
+end

data/test/test_srs.rb

@@ -0,0 +1,9 @@
+require(File.expand_path(File.dirname(__FILE__) + '/helpers_tests.rb'))
+
+class StatsampleSrsTestCase < Minitest::Test
+  def test_std_error
+    assert_equal(384, Statsample::SRS.estimation_n0(0.05, 0.5, 0.95).to_i)
+    assert_equal(108, Statsample::SRS.estimation_n(0.05, 0.5, 150, 0.95).to_i)
+    assert_in_delta(0.0289, Statsample::SRS.proportion_sd_kp_wor(0.5, 100, 150), 0.001)
+  end
+end

data/test/test_statistics.rb

@@ -0,0 +1,69 @@
+require(File.expand_path(File.dirname(__FILE__) + '/helpers_tests.rb'))
+class StatsampleStatisicsTestCase < Minitest::Test
+  def initialize(*args)
+    super
+  end
+
+  def test_p_using_cdf
+    assert_equal(0.25, Statsample::Test.p_using_cdf(0.25, tails = :left))
+    assert_equal(0.75, Statsample::Test.p_using_cdf(0.25, tails = :right))
+    assert_equal(0.50, Statsample::Test.p_using_cdf(0.25, tails = :both))
+    assert_equal(1, Statsample::Test.p_using_cdf(0.50, tails = :both))
+    assert_equal(0.05, Statsample::Test.p_using_cdf(0.025, tails = :both))
+    assert_in_delta(0.05, Statsample::Test.p_using_cdf(0.975, tails = :both), 0.0001)
+  end
+
+  def test_recode_repeated
+    a = %w(a b c c d d d e)
+    exp = %w(a b c_1 c_2 d_1 d_2 d_3 e)
+    assert_equal(exp, Daru::ArrayHelper.recode_repeated(a))
+  end
+
+  def test_is_number
+    assert('10'.is_number?)
+    assert('-10'.is_number?)
+    assert('0.1'.is_number?)
+    assert('-0.1'.is_number?)
+    assert('10e3'.is_number?)
+    assert('10e-3'.is_number?)
+    assert(!'1212-1212-1'.is_number?)
+    assert(!'a10'.is_number?)
+    assert(!''.is_number?)
+  end
+
+  def test_estimation_mean
+    v = Daru::Vector.new([42] * 23 + [41] * 4 + [36] * 1 + [32] * 1 + [29] * 1 + [27] * 2 + [23] * 1 + [19] * 1 + [16] * 2 + [15] * 2 + [14, 11, 10, 9, 7] + [6] * 3 + [5] * 2 + [4, 3])
+    assert_equal(50, v.size)
+    assert_equal(1471, v.sum)
+    # limits=Statsample::SRS.mean_confidence_interval_z(v.mean(), v.sds(), v.size,676,0.80)
+  end
+
+  def test_estimation_proportion
+    # total
+    pop = 3042
+    sam = 200
+    prop = 0.19
+    assert_in_delta(81.8, Statsample::SRS.proportion_total_sd_ep_wor(prop, sam, pop), 0.1)
+
+    # confidence limits
+    pop = 500
+    sam = 100
+    prop = 0.37
+    a = 0.95
+    l = Statsample::SRS.proportion_confidence_interval_z(prop, sam, pop, a)
+    assert_in_delta(0.28, l[0], 0.01)
+    assert_in_delta(0.46, l[1], 0.01)
+  end
+
+  def test_simple_linear_regression
+    a = Daru::Vector.new([1, 2, 3, 4, 5, 6])
+    b = Daru::Vector.new([6, 2, 4, 10, 12, 8])
+    reg = Statsample::Regression::Simple.new_from_vectors(a, b)
+    assert_in_delta((reg.ssr + reg.sse).to_f, reg.sst, 0.001)
+    assert_in_delta(Statsample::Bivariate.pearson(a, b), reg.r, 0.001)
+    assert_in_delta(2.4, reg.a, 0.01)
+    assert_in_delta(1.314, reg.b, 0.001)
+    assert_in_delta(0.657, reg.r, 0.001)
+    assert_in_delta(0.432, reg.r2, 0.001)
+  end
+end

data/test/test_stest.rb

@@ -0,0 +1,69 @@
+require(File.expand_path(File.dirname(__FILE__) + '/helpers_tests.rb'))
+
+class StatsampleTestTestCase < Minitest::Test
+  def test_chi_square_matrix_with_expected
+    real = Matrix[[95, 95], [45, 155]]
+    expected = Matrix[[68, 122], [72, 128]]
+    assert_nothing_raised do
+      Statsample::Test.chi_square(real, expected)
+    end
+    chi = Statsample::Test.chi_square(real, expected).chi_square
+    assert_in_delta(32.53, chi, 0.1)
+  end
+
+  def test_chi_square_matrix_only_observed
+    observed = Matrix[[20, 30, 40], [30, 40, 50], [60, 70, 80], [10, 20, 40]]
+    assert_nothing_raised do
+      Statsample::Test.chi_square(observed)
+    end
+    chi = Statsample::Test.chi_square(observed)
+    assert_in_delta(9.5602, chi.chi_square, 0.0001)
+    assert_in_delta(0.1444, chi.probability, 0.0001)
+
+    assert_equal(6, chi.df)
+  end
+
+  def test_chi_square_vector
+    observed = Vector[20,30,15]
+    expected = Vector[20,20,20]
+    assert_nothing_raised do
+      Statsample::Test.chi_square(observed, expected)
+    end
+    chi = Statsample::Test.chi_square(observed, expected)
+
+    assert_in_delta(6.25, chi.chi_square, 0.0001)
+    assert_in_delta(0.04393, chi.probability, 0.00001)
+
+    assert_equal(2, chi.df)
+  end
+
+  def test_u_mannwhitney
+    a = Daru::Vector.new([1, 2, 3, 4, 5, 6])
+    b = Daru::Vector.new([0, 5, 7, 9, 10, 11])
+    assert_equal(7.5, Statsample::Test.u_mannwhitney(a, b).u)
+    assert_equal(7.5, Statsample::Test.u_mannwhitney(b, a).u)
+    a = Daru::Vector.new([1, 7, 8, 9, 10, 11])
+    b = Daru::Vector.new([2, 3, 4, 5, 6, 12])
+    assert_equal(11, Statsample::Test.u_mannwhitney(a, b).u)
+  end
+
+  def test_levene
+    a = Daru::Vector.new([1, 2, 3, 4, 5, 6, 7, 8, 100, 10])
+    b = Daru::Vector.new([30, 40, 50, 60, 70, 80, 90, 100, 110, 120])
+    levene = Statsample::Test::Levene.new([a, b])
+    assert_levene(levene)
+  end
+
+  def test_levene_dataset
+    a = Daru::Vector.new([1, 2, 3, 4, 5, 6, 7, 8, 100, 10])
+    b = Daru::Vector.new([30, 40, 50, 60, 70, 80, 90, 100, 110, 120])
+    ds = Daru::DataFrame.new({ :a => a, :b => b })
+    levene = Statsample::Test::Levene.new(ds)
+    assert_levene(levene)
+  end
+
+  def assert_levene(levene)
+    assert_in_delta(0.778, levene.f, 0.001)
+    assert_in_delta(0.389, levene.probability, 0.001)
+  end
+end

data/test/test_stratified.rb

@@ -0,0 +1,17 @@
+require(File.expand_path(File.dirname(__FILE__) + '/helpers_tests.rb'))
+
+class StatsampleStratifiedTestCase < Minitest::Test
+  def initialize(*args)
+    super
+  end
+
+  def test_mean
+    a = [10, 20, 30, 40, 50]
+    b = [110, 120, 130, 140]
+    pop = a + b
+    av   = Daru::Vector.new(a)
+    bv   = Daru::Vector.new(b)
+    popv = Daru::Vector.new(pop)
+    assert_equal(popv.mean, Statsample::StratifiedSample.mean(av, bv))
+  end
+end