statsample-ekatena 2.0.2

data/lib/statsample-ekatena/test/levene.rb

@@ -0,0 +1,88 @@
+module Statsample
+  module Test
+    # = Levene Test for Equality of Variances
+    # From NIST/SEMATECH:
+    # <blockquote>Levene's test ( Levene, 1960) is used to test if k samples have equal variances. Equal variances across samples is called homogeneity of variance. Some statistical tests, for example the analysis of variance, assume that variances are equal across groups or samples. The Levene test can be used to verify that assumption.</blockquote>
+    # Use:
+    #   require 'statsample'
+    #   a = Daru::Vector.new([1,2,3,4,5,6,7,8,100,10])
+    #   b = Daru::Vector.new([30,40,50,60,70,80,90,100,110,120])
+    # 
+    #   levene=Statsample::Test::Levene.new([a,b])
+    #   puts levene.summary
+    #   
+    # Output:
+    #   Levene Test
+    #   F: 0.778121319848449
+    #   p: 0.389344552595791
+    #
+    # Reference:
+    # * NIST/SEMATECH e-Handbook of Statistical Methods. Available on http://www.itl.nist.gov/div898/handbook/eda/section3/eda35a.htm
+    class Levene
+      include Statsample::Test
+      include Summarizable
+      # Degrees of freedom 1 (k-1)
+      attr_reader :d1
+      # Degrees of freedom 2 (n-k)
+      attr_reader :d2
+      # Name of test
+      attr_accessor :name
+      # Input could be an array of vectors or a dataset
+      def initialize(input, opts=Hash.new())
+        if input.is_a? Daru::DataFrame
+          @vectors = input.to_hash.values
+        else
+          @vectors = input
+        end
+        @name=_("Levene Test")
+        opts.each{|k,v|
+          self.send("#{k}=",v) if self.respond_to? k
+        }
+        compute
+      end
+      # Value of the test
+      def f
+        @w
+      end
+      def report_building(builder) # :nodoc:
+        builder.text "%s : F(%d, %d) = %0.4f , p = %0.4f" % [@name, @d1, @d2, f, probability]
+      end
+      def compute
+        n=@vectors.inject(0) { |ac,v| ac + v.reject_values(*Daru::MISSING_VALUES).size }
+        
+        zi=@vectors.collect do |vector|
+          mean=vector.mean
+          Daru::Vector.new(vector.collect { |v| (v - mean).abs })
+        end
+        
+        total_mean = Daru::Vector.new(
+          zi.inject([]) do |ac,vector|
+            ac + vector.reject_values(*Daru::MISSING_VALUES).to_a
+          end
+        ).mean
+      
+        k = @vectors.size
+        sum_num = zi.inject(0) do |ac,vector|
+          ac + (vector.size * (vector.mean - total_mean)**2)
+        end
+        
+        sum_den = zi.inject(0) do |ac,vector|
+          z_mean = vector.mean
+          ac + vector.reject_values(*Daru::MISSING_VALUES).to_a.inject(0) do |acp,zij|
+            acp + (zij - z_mean)**2
+          end
+        end
+
+        @w  = ((n - k) * sum_num).quo((k - 1) * sum_den)
+        @d1 = k - 1
+        @d2 = n - k
+      end
+      private :compute
+      # Probability.
+      # With H_0 = Sum(s2)=0, probability of getting a value of the test upper or equal to the obtained on the sample
+      def probability
+        p_using_cdf(Distribution::F.cdf(f, @d1, @d2), :right)
+      end
+    end
+  end
+end

data/lib/statsample-ekatena/test/t.rb

@@ -0,0 +1,309 @@
+module Statsample
+  module Test
+    # A t-test is any statistical hypothesis test in which the test 
+    # statistic follows a Student's t distribution, if the null 
+    # hypothesis is supported
+    class T
+      
+      class << self
+        include Math
+        # Test the null hypothesis that the population mean is equal to a specified value u, one uses the statistic.
+        # Is the same formula used on t-test for paired sample.
+        # * <tt>x</tt>: sample/differences mean
+        # * <tt>u</tt>: population mean
+        # * <tt>s</tt>: sample/differences standard deviation
+        # * <tt>n</tt>: sample size
+        def one_sample(x,u,s,n)
+          (x-u)*Math::sqrt(n).quo(s)
+        end
+        # Test if means of two samples are different.
+        # * <tt>x1</tt>: sample 1 mean
+        # * <tt>x2</tt>: sample 2 mean
+        # * <tt>s1</tt>: sample 1 standard deviation
+        # * <tt>s2</tt>: sample 2 standard deviation
+        # * <tt>n1</tt>: sample 1 size
+        # * <tt>n2</tt>: sample 2 size
+        # * <tt>equal_variance</tt>: true if equal_variance assumed
+        #
+        def two_sample_independent(x1, x2, s1, s2, n1, n2, equal_variance = false)
+          num=x1-x2
+          if equal_variance
+            sx1x2 = sqrt(((n1-1)*s1**2 + (n2-1)*s2**2).quo(n1+n2-2))
+            den   = sx1x2*sqrt(1.quo(n1)+1.quo(n2))
+          else
+            den=sqrt((s1**2).quo(n1) + (s2**2).quo(n2))
+          end
+          num.quo(den)
+        end
+        # Degrees of freedom for equal variance on t test
+        def df_equal_variance(n1,n2)
+          n1+n2-2
+        end
+        # Degrees of freedom for unequal variance
+        # * <tt>s1</tt>: sample 1 standard deviation
+        # * <tt>s2</tt>: sample 2 standard deviation
+        # * <tt>n1</tt>: sample 1 size
+        # * <tt>n2</tt>: sample 2 size
+        # == Reference
+        # * http://en.wikipedia.org/wiki/Welch-Satterthwaite_equation
+        def df_not_equal_variance(s1,s2,n1,n2)
+          s2_1=s1**2
+          s2_2=s2**2
+          num=(s2_1.quo(n1)+s2_2.quo(n2))**2
+          den=(s2_1.quo(n1)**2).quo(n1-1) + (s2_2.quo(n2)**2).quo(n2-1)
+          num.quo(den)
+        end        
+      end
+      
+      include Statsample::Test
+      include Summarizable
+      attr_reader :standard_error, :estimate, :df
+      # Tails for p-value (:both, :left or :right). Default :both
+      attr_accessor :tails
+      # Name of F analysis
+      attr_accessor :name
+      attr_accessor :confidence_level
+      attr_reader :t
+      attr_accessor :estimate_name, :standard_error_name
+      # Creates a generic t test. Use OneSample or TwoSamplesIndependent
+      # classes for better summaries.
+      # Parameters:
+      # * estimate: estimate
+      # * standard_error: standard error of estimate
+      # * df: degrees of freedom
+      def initialize(estimate, standard_error, df, opts=Hash.new)
+        @estimate=estimate
+        @standard_error=standard_error
+        @df=df
+        @t = @estimate / @standard_error.to_f
+        opts_default={  :tails=>:both,
+                        :name=>_("T Test"),
+                        :estimate_name=>_("Estimate"),
+                        :standard_error_name=>_("Std.Err.of Estimate"),
+        :confidence_level=>0.95}
+        @opts = opts_default.merge(opts)
+        
+        @opts.keys.each {|k|
+          send("#{k}=", @opts[k]) if respond_to? k
+        }
+      end
+      
+      alias :se :standard_error
+      
+      def to_f
+        t
+      end
+      
+      # probability
+      def probability
+        p_using_cdf(Distribution::T.cdf(t, df),  tails)
+      end
+      
+      def confidence_interval(cl=nil)
+          cl||=confidence_level
+          t_crit = t_critical(cl, df)
+          [estimate - se*t_crit, estimate + se*t_crit]
+      end
+      alias :ci :confidence_interval
+      
+      
+      def report_building(builder) #:nodoc:
+        builder.section(:name=>@name) do |section|
+          section.text _("%s: %0.4f | %s: %0.4f") % [@estimate_name, @estimate, @standard_error_name, se]
+          report_building_t(section)
+        end
+      end
+      def report_building_t(s)
+        df_f=@df.is_a?(Integer) ? "%d" : "%0.4f"
+        s.text _("t(%d) = %0.4f, p=%0.4f (%s tails)") % [df, t,probability, tails]
+        s.text _("CI(%d%%): %0.4f - %0.4f") % [confidence_level*100, ci[0],ci[1]]
+        
+      end
+      
+      
+      # One Sample t-test
+      # == Usage
+      #   a = Daru::Vector.new(1000.times.map {rand(100)})
+      #   t_1=Statsample::Test::T::OneSample.new(a, {:u=>50})
+      #   t_1.summary
+      #
+      # === Output
+      #
+      #  = One Sample T Test
+      #  Sample mean: 48.954
+      #  Population mean:50
+      #  Tails: both
+      #  t = -1.1573, p=0.2474, d.f=999
+
+      class OneSample
+        include Math
+        include Statsample::Test
+        include Summarizable
+        # Options
+        attr_accessor :opts
+        # Name of test
+        attr_accessor :name
+        # Population mean to contrast
+        attr_accessor :u
+        # Degress of freedom
+        attr_reader :df
+        # Tails for probability (:both, :left or :right)
+        attr_accessor :tails 
+        
+        # Create a One Sample T Test
+        # Options:
+        # * :u = Mean to compare. Default= 0
+        # * :name = Name of the analysis
+        # * :tails = Tail for probability. Could be :both, :left, :right
+        def initialize(vector, opts=Hash.new)
+          @vector=vector
+          default={:u=>0, :name=>"One Sample T Test", :tails=>:both}
+          @opts=default.merge(opts)
+          @name=@opts[:name]
+          @u=@opts[:u]
+          @tails=@opts[:tails]
+          @confidence_level=@opts[:confidence_level] || 0.95
+          @df= @vector.reject_values(*Daru::MISSING_VALUES).size-1
+          @t=nil
+        end        
+        def t_object
+          T.new(@vector.mean-u, @vector.se, @vector.reject_values(*Daru::MISSING_VALUES).size-1, opts)
+        end
+        def t
+          t_object.t
+        end
+        def probability
+          t_object.probability
+        end
+        def standard_error
+          t_object.standard_error
+        end
+        alias :se :standard_error
+        def confidence_interval(cl=nil)
+          t_object.confidence_interval(cl)
+        end
+        alias :ci :confidence_interval
+        def report_building(b) # :nodoc:
+          b.section(:name=>@name) {|s|
+            s.text _("Sample mean: %0.4f | Sample sd: %0.4f | se : %0.4f") % [@vector.mean, @vector.sd, se]
+            s.text _("Population mean: %0.4f") % u if u!=0
+            t_object.report_building_t(s)
+          }
+        end
+      end
+      # Two Sample t-test.
+      #
+      # == Usage
+      #   a = Daru::Vector.new(1000.times.map {rand(100)})
+      #   b = Daru::Vector.new(1000.times.map {rand(100)})
+      #   t_2=Statsample::Test::T::TwoSamplesIndependent.new(a,b)
+      #   t_2.summary
+      # === Output
+      #  = Two Sample T Test
+      #  Mean and standard deviation
+      #  +----------+---------+---------+------+
+      #  | Variable |    m    |   sd    |  n   |
+      #  +----------+---------+---------+------+
+      #  | 1        | 49.3310 | 29.3042 | 1000 |
+      #  | 2        | 47.8180 | 28.8640 | 1000 |
+      #  +----------+---------+---------+------+
+      #  
+      #  == Levene Test
+      #   Levene Test
+      #   F: 0.3596
+      #   p: 0.5488
+      #   T statistics
+      #   +--------------------+--------+-----------+----------------+
+      #   |        Type        |   t    |    df     | p (both tails) |
+      #   +--------------------+--------+-----------+----------------+
+      #   | Equal variance     | 1.1632 | 1998      | 0.2449         |
+      #   | Non equal variance | 1.1632 | 1997.5424 | 0.1362         |
+      #   +--------------------+--------+-----------+----------------+
+
+      class TwoSamplesIndependent
+        include Math
+        include Statsample::Test
+        
+        include DirtyMemoize
+        include Summarizable
+        # Options
+        attr_accessor :opts
+        # Name of test
+        attr_accessor :name
+        # Degress of freedom (equal variance)
+        attr_reader :df_equal_variance
+        # Degress of freedom (not equal variance)
+        attr_reader :df_not_equal_variance
+        # Value of t for equal_variance
+        attr_reader :t_equal_variance
+        # Value of t for non-equal_variance
+        attr_reader :t_not_equal_variance
+        # Probability(equal variance)
+        attr_reader :probability_equal_variance
+        # Probability(unequal variance)
+        attr_reader :probability_not_equal_variance        
+        # Tails for probability (:both, :left or :right)
+        attr_accessor :tails 
+        # Create the object
+        
+        dirty_writer :tails
+        dirty_memoize :t_equal_variance, :t_not_equal_variance, :probability_equal_variance, :probability_not_equal_variance, :df_equal_variance, :df_not_equal_variance
+
+        # Create a Two Independent T Test
+        # Options:
+        # * :name = Name of the analysis
+        # * :tails = Tail for probability. Could be :both, :left, :right        
+        def initialize(v1, v2, opts=Hash.new)
+          @v1=v1
+          @v2=v2
+          default={:u=>0, :name=>"Two Sample T Test",  :tails=>:both}
+          @opts=default.merge(opts)
+          @name=@opts[:name]
+          @tails=@opts[:tails]          
+        end        
+       
+        # Set t and probability for given u
+        def compute
+          @t_equal_variance= T.two_sample_independent(@v1.mean, @v2.mean, @v1.sd, @v2.sd, @v1.reject_values(*Daru::MISSING_VALUES).size, @v2.reject_values(*Daru::MISSING_VALUES).size,true)
+          
+          @t_not_equal_variance= T.two_sample_independent(@v1.mean, @v2.mean, @v1.sd, @v2.sd, @v1.reject_values(*Daru::MISSING_VALUES).size, @v2.reject_values(*Daru::MISSING_VALUES).size, false)
+
+          @df_equal_variance=T.df_equal_variance(@v1.reject_values(*Daru::MISSING_VALUES).size, @v2.reject_values(*Daru::MISSING_VALUES).size)
+          @df_not_equal_variance=T.df_not_equal_variance(@v1.sd, @v2.sd, @v1.reject_values(*Daru::MISSING_VALUES).size, @v2.reject_values(*Daru::MISSING_VALUES).size)
+          
+          @probability_equal_variance = p_using_cdf(Distribution::T.cdf(@t_equal_variance, @df_equal_variance), tails)
+          
+          @probability_not_equal_variance = p_using_cdf(Distribution::T.cdf(@t_not_equal_variance, @df_not_equal_variance), tails)
+
+        end
+        # Cohen's d is a measure of effect size. Its defined as the difference between two means divided by a standard deviation for the data
+        def d
+          n1=@v1.reject_values(*Daru::MISSING_VALUES).size
+          n2=@v2.reject_values(*Daru::MISSING_VALUES).size
+          num=@v1.mean-@v2.mean
+          den=Math::sqrt( ((n1-1)*@v1.sd+(n2-1)*@v2.sd).quo(n1+n2))
+          num.quo(den)
+        end
+        
+        def report_building(b) # :nodoc:
+          b.section(:name=>@name) {|g|
+            g.table(:name=>_("Mean and standard deviation"), :header=>[_("Variable"), _("mean"), _("sd"),_("n")]) {|t|
+              t.row([@v1.name,"%0.4f" % @v1.mean,"%0.4f" % @v1.sd, @v1.reject_values(*Daru::MISSING_VALUES).size])
+              t.row([@v2.name,"%0.4f" % @v2.mean,"%0.4f" % @v2.sd, @v2.reject_values(*Daru::MISSING_VALUES).size])
+            }
+            g.parse_element(Statsample::Test.levene([@v1,@v2],:name=>_("Levene test for equality of variances")))
+            
+            g.table(:name=>_("T statistics"),:header=>["Type","t","df", "p (#{tails} tails)"].map{|v| _(v)}) {|t|
+              t.row([_("Equal variance"), "%0.4f" % t_equal_variance, df_equal_variance, "%0.4f" % probability_equal_variance])
+              t.row([_("Non equal variance"), "%0.4f" % t_not_equal_variance, "%0.4f" % df_not_equal_variance, "%0.4f" % probability_not_equal_variance])
+            }
+            g.table(:name=>_("Effect size")) do |t|
+              t.row ['x1-x2', "%0.4f" % (@v1.mean-@v2.mean)]
+              t.row ['d', "%0.4f" % d]
+            end
+          }
+        end
+      end      
+    end
+  end
+end

data/lib/statsample-ekatena/test/umannwhitney.rb

@@ -0,0 +1,208 @@
+module Statsample
+  module Test
+    #
+    # = U Mann-Whitney test
+    #
+    # Non-parametric test for assessing whether two independent samples
+    # of observations come from the same distribution.
+    # 
+    # == Assumptions
+    #
+    # * The two samples under investigation in the test are independent of each other and the observations within each sample are independent.
+    # * The observations are comparable (i.e., for any two observations, one can assess whether they are equal or, if not, which one is greater).
+    # * The variances in the two groups are approximately equal.
+    #
+    # Higher differences of distributions correspond to 
+    # to lower values of U.
+    #
+    class UMannWhitney
+      # Max for m*n allowed for exact calculation of probability
+      MAX_MN_EXACT=10000
+      
+      # U sampling distribution, based on Dinneen & Blakesley (1973) algorithm.
+      # This is the algorithm used on SPSS.
+      # 
+      # Parameters:
+      # * <tt>n1</tt>: group 1 size
+      # * <tt>n2</tt>: group 2 size 
+      # == Reference: 
+      # * Dinneen, L., & Blakesley, B. (1973). Algorithm AS 62: A Generator for the Sampling Distribution of the Mann- Whitney U Statistic. <em>Journal of the Royal Statistical Society, 22</em>(2), 269-273
+      # 
+      def self.u_sampling_distribution_as62(n1,n2)
+
+        freq=[]
+        work=[]
+        mn1=n1*n2+1
+        max_u=n1*n2
+        minmn=n1<n2 ? n1 : n2
+        maxmn=n1>n2 ? n1 : n2
+        n1=maxmn+1
+        (1..n1).each{|i| freq[i]=1}
+        n1+=1
+        (n1..mn1).each{|i| freq[i]=0}
+        work[1]=0
+        xin=maxmn
+        (2..minmn).each do |i|
+          work[i]=0
+          xin=xin+maxmn
+          n1=xin+2
+          l=1+xin.quo(2)
+          k=i
+          (1..l).each do |j|
+            k=k+1
+            n1=n1-1
+            sum=freq[j]+work[j]
+            freq[j]=sum
+            work[k]=sum-freq[n1]
+            freq[n1]=sum
+          end
+        end
+        
+        # Generate percentages for normal U
+        dist=(1+max_u/2).to_i
+        freq.shift
+        total=freq.inject(0) {|a,v| a+v }
+        (0...dist).collect {|i|
+          if i!=max_u-i
+            ues=freq[i]*2
+          else
+            ues=freq[i]
+          end
+          ues.quo(total)
+        }
+      end
+      
+      # Generate distribution for permutations. 
+      # Very expensive, but useful for demostrations
+      
+      def self.distribution_permutations(n1,n2)
+        base=[0]*n1+[1]*n2
+        po=Statsample::Permutation.new(base)
+        
+        total=n1*n2
+        req={}
+        po.each do |perm|
+          r0,s0=0,0
+          perm.each_index {|c_i|
+            if perm[c_i]==0
+              r0+=c_i+1
+              s0+=1
+            end
+          }
+          u1=r0-((s0*(s0+1)).quo(2))
+          u2=total-u1
+          temp_u= (u1 <= u2) ? u1 : u2
+          req[perm]=temp_u
+        end
+        req
+      end
+      # Sample 1 Rank sum
+      attr_reader :r1
+      # Sample 2 Rank sum
+      attr_reader :r2
+      # Sample 1 U (useful for demostration)
+      attr_reader :u1
+      # Sample 2 U (useful for demostration)
+      attr_reader :u2
+      # U Value 
+      attr_reader :u
+      # Value of compensation for ties (useful for demostration)
+      attr_reader :t
+      # Name of test
+      attr_accessor :name
+      include Summarizable
+      #
+      # Create a new U Mann-Whitney test
+      # Params: Two Daru::Vectors
+      # 
+      def initialize(v1,v2, opts=Hash.new)
+        @v1      = v1
+        @v2      = v2
+        v1_valid = v1.reject_values(*Daru::MISSING_VALUES).reset_index!
+        v2_valid = v2.reject_values(*Daru::MISSING_VALUES).reset_index!
+        @n1      = v1_valid.size
+        @n2      = v2_valid.size
+        data     = Daru::Vector.new(v1_valid.to_a + v2_valid.to_a)
+        groups   = Daru::Vector.new(([0] * @n1) + ([1] * @n2))
+        ds       = Daru::DataFrame.new({:g => groups, :data => data})
+        @t       = nil
+        @ties    = data.to_a.size != data.to_a.uniq.size        
+        if @ties
+          adjust_for_ties(ds[:data])
+        end
+        ds[:ranked] = ds[:data].ranked      
+        @n = ds.nrows
+          
+        @r1 = ds.filter_rows { |r| r[:g] == 0}[:ranked].sum
+        @r2 = ((ds.nrows * (ds.nrows + 1)).quo(2)) - r1
+        @u1 = r1 - ((@n1 * (@n1 + 1)).quo(2))
+        @u2 = r2 - ((@n2 * (@n2 + 1)).quo(2))
+        @u  = (u1 < u2) ? u1 : u2
+        opts_default = { :name=>_("Mann-Whitney's U") }
+        @opts = opts_default.merge(opts)
+        opts_default.keys.each {|k|
+          send("#{k}=", @opts[k])
+        }       
+      end
+      def report_building(generator) # :nodoc:
+        generator.section(:name=>@name) do |s|
+          s.table(:name=>_("%s results") % @name) do |t|
+            t.row([_("Sum of ranks %s") % @v1.name, "%0.3f" % @r1])
+            t.row([_("Sum of ranks %s") % @v2.name, "%0.3f" % @r2])
+            t.row([_("U Value"), "%0.3f" % @u])
+            t.row([_("Z"), "%0.3f (p: %0.3f)" % [z, probability_z]])
+            if @n1*@n2<MAX_MN_EXACT
+              t.row([_("Exact p (Dinneen & Blakesley, 1973):"), "%0.3f" % probability_exact])
+            end
+          end
+        end
+      end
+      # Exact probability of finding values of U lower or equal to sample on U distribution. Use with caution with m*n>100000.
+      # Uses u_sampling_distribution_as62
+      def probability_exact
+        dist = UMannWhitney.u_sampling_distribution_as62(@n1,@n2)
+        sum = 0
+        (0..@u.to_i).each {|i|
+          sum+=dist[i]
+        }
+        sum
+      end
+      # Adjunt for ties.
+      # 
+      # == Reference: 
+      # * http://europe.isixsigma.com/library/content/c080806a.asp
+      def adjust_for_ties(data)
+        @t = data.frequencies.find_all { |k,v| v > 1 }.inject(0) { |a,v|
+          a + (v[1]**3 - v[1]).quo(12)
+        }        
+      end
+      
+      private :adjust_for_ties
+      
+      # Z value for U, with adjust for ties.
+      # For large samples, U is approximately normally distributed. 
+      # In that case, you can use z to obtain probabily for U.
+      # == Reference: 
+      # * SPSS Manual
+      def z
+        mu=(@n1*@n2).quo(2)
+        if(!@ties)
+          ou=Math::sqrt(((@n1*@n2)*(@n1+@n2+1)).quo(12))
+        else
+          n=@n1+@n2
+          first=(@n1*@n2).quo(n*(n-1))
+          second=((n**3-n).quo(12))-@t
+          ou=Math::sqrt(first*second)
+        end
+        (@u-mu).quo(ou)
+      end
+      # Assuming H_0, the proportion of cdf with values of U lower
+      # than the sample, using normal approximation.
+      # Use with more than 30 cases per group.
+      def probability_z
+        (1-Distribution::Normal.cdf(z.abs()))*2
+      end
+    end
+      
+  end
+end