statsample 0.3.0

data/lib/statsample/anova.rb

@@ -0,0 +1,74 @@
+module Statsample
+    module Anova
+        # One Way Anova
+        # Example:
+        #   v1=[2,3,4,5,6].to_vector(:scale)
+        #   v2=[3,3,4,5,6].to_vector(:scale)
+        #   v3=[5,3,1,5,6].to_vector(:scale)
+        #   anova=Statsample::Anova::OneWay.new([v1,v2,v3])
+        #   puts anova.f
+        #   puts anova.significance
+        class OneWay
+        def initialize(vectors)
+            @vectors=vectors
+        end
+        # Total sum
+        def sum
+            @vectors.inject(0){|a,v| a+v.sum}
+        end
+	# Total mean
+	def mean
+            sum.quo(n)
+        end
+        # Total sum of squares
+        def sst
+            m=mean.to_f
+            @vectors.inject(0) {|total,vector|
+                total+vector.sum_of_squares(m)
+            }
+        end
+        # Sum of squares within groups
+        def sswg
+            @vectors.inject(0) {|total,vector|
+                total+vector.sum_of_squares
+            }
+        end
+        # Sum of squares between groups
+        def ssbg
+            m=mean
+            @vectors.inject(0) {|total,vector|
+                total+(vector.mean-m).square*vector.size
+            }
+        end
+        # Degrees of freedom within groups
+        def df_wg
+            @vectors.inject(0) {|a,v| a+(v.size-1)}
+        end
+        # Degrees of freedom between groups 
+        def df_bg
+            @vectors.size-1
+        end
+        # Total Degrees of freedom
+        def df_total
+            n-1
+        end
+        # Total number of cases
+        def n
+            @vectors.inject(0){|a,v| a+v.size}
+        end
+        # Fisher
+        def f
+            k=@vectors.size
+            (ssbg*(n-k)) / (sswg*(k-1))
+        end
+        # Significance of Fisher
+        def significance
+            if HAS_GSL
+                GSL::Cdf.fdist_Q(f,df_bg,df_wg)
+            else
+                raise "Need Ruby/GSL"
+            end
+        end
+    end
+    end
+end

data/lib/statsample/bivariate.rb

@@ -0,0 +1,255 @@
+module Statsample
+    # Diverse correlation methods 
+    module Bivariate
+        class << self
+            # Covariance between two vectors
+			def covariance(v1,v2)
+				v1a,v2a=Statsample.only_valid(v1,v2)
+                return nil if v1a.size==0
+				if HAS_GSL
+					GSL::Stats::covariance(v1a.gsl, v2a.gsl)
+				else
+					covariance_slow(v1a,v2a)
+				end
+			end
+			# Covariance. The denominator is n-1
+			def covariance_slow(v1a,v2a)
+				t=0
+				m1=v1a.mean
+				m2=v1a.mean
+				(0...v1a.size).each {|i|
+					t+=((v1a[i]-m1)*(v2a[i]-m2))
+				}
+				t.to_f / (v1a.size-1)
+			end
+            # Calculate Pearson correlation coefficient between 2 vectors
+            def pearson(v1,v2)
+				v1a,v2a=Statsample.only_valid(v1,v2)
+                return nil if v1a.size ==0
+				if HAS_GSL
+					GSL::Stats::correlation(v1a.gsl, v2a.gsl)
+				else
+					pearson_slow(v1a,v2a)
+				end
+            end
+            #:nodoc:
+            def pearson_slow(v1a,v2a)
+                v1s,v2s=v1a.vector_standarized_pop,v2a.vector_standarized_pop
+                t=0
+                siz=v1s.size
+                (0...v1s.size).each {|i| t+=(v1s[i]*v2s[i]) }
+                t.to_f/v2s.size
+            end
+            # Retrieves the value for t test for a pearson correlation
+            # between two vectors to test the null hipothesis of r=0
+            def t_pearson(v1,v2)
+				v1a,v2a=Statsample.only_valid(v1,v2)
+                r=pearson(v1a,v2a)
+                if(r==1.0) 
+                    0
+                else
+                    t_r(r,v1a.size)
+                end
+            end
+            # Retrieves the value for t test for a pearson correlation
+            # giving r and vector size
+            def t_r(r,size)
+                r*Math::sqrt(((size)-2).to_f / (1 - r**2))
+            end
+            # Retrieves the probability value (a la SPSS)
+            # for a given t, size and number of tails
+            def prop_pearson(t,size, tails=2)
+		if HAS_GSL
+                t=-t if t>0
+                cdf=GSL::Cdf::tdist_P(t,(size)-2)
+                cdf*tails
+		else
+			raise "Needs ruby-gsl"
+		end
+            end
+            # Returns residual score after delete variance
+            # from another variable
+            # 
+            def residuals(from,del)
+                r=Statsample::Bivariate.pearson(from,del)
+                froms, dels = from.vector_standarized, del.vector_standarized
+                nv=[]
+                froms.data_with_nils.each_index{|i|
+                    if froms[i].nil? or dels[i].nil?
+                        nv.push(nil)
+                    else
+                        nv.push(froms[i]-r*dels[i])
+                    end
+                }
+                nv.to_vector(:scale)
+            end
+            def partial_correlation(v1,v2,control)
+                v1a,v2a,cona=Statsample.only_valid(v1,v2,control)
+                rv1v2=pearson(v1a,v2a)
+                rv1con=pearson(v1a,cona)
+                rv2con=pearson(v2a,cona)
+                
+                (rv1v2-(rv1con*rv2con)).quo(Math::sqrt(1-rv1con**2) * Math::sqrt(1-rv2con**2))
+                
+            end
+            # Covariance matrix
+            def covariance_matrix(ds)
+                ds.collect_matrix do |row,col|
+                    if (ds[row].type!=:scale or ds[col].type!=:scale)
+                        nil
+                    else
+                        covariance(ds[row],ds[col])
+                    end
+                end
+            end
+            
+            # The classic correlation matrix for all fields of a dataset
+            
+            def correlation_matrix(ds)
+                ds.collect_matrix {|row,col|
+                        if row==col
+                            1.0
+                        elsif (ds[row].type!=:scale or ds[col].type!=:scale)
+                            nil
+                        else
+                            pearson(ds[row],ds[col])
+                        end
+                }
+            end
+            # Retrieves the n valid pairwise
+            def n_valid_matrix(ds)
+                ds.collect_matrix {|row,col|
+                        if row==col
+                            ds[row].valid_data.size
+                        else
+                            rowa,rowb=Statsample.only_valid(ds[row],ds[col])
+                            rowa.size
+                        end
+                }
+            end
+            def correlation_probability_matrix(ds)
+                rows=ds.fields.collect{|row|
+                    ds.fields.collect{|col|
+                        v1a,v2a=Statsample.only_valid(ds[row],ds[col])
+                        (row==col or ds[row].type!=:scale or ds[col].type!=:scale) ? nil : prop_pearson(t_pearson(ds[row],ds[col]), v1a.size)
+                    }
+                }
+                Matrix.rows(rows)
+            end
+			# Calculate Spearman correlation coefficient between 2 vectors
+			def spearman(v1,v2)
+				v1a,v2a=Statsample.only_valid(v1,v2)
+				v1r,v2r=v1a.ranked(:scale),v2a.ranked(:scale)
+                pearson(v1r,v2r)
+			end
+			# Calculate Point biserial correlation.
+			# Equal to Pearson correlation, with one dichotomous value replaced
+			# by "0" and the other by "1"
+			def point_biserial(dichotomous,continous)
+				ds={'d'=>dichotomous,'c'=>continous}.to_dataset.dup_only_valid
+				raise(TypeError, "First vector should be dichotomous") if ds['d'].factors.size!=2
+				raise(TypeError, "Second vector should be continous") if ds['c'].type!=:scale
+				f0=ds['d'].factors.sort[0]
+				m0=ds.filter_field('c') {|c| c['d']==f0}
+				m1=ds.filter_field('c') {|c| c['d']!=f0}
+				((m1.mean-m0.mean).to_f / ds['c'].sdp) * Math::sqrt(m0.size*m1.size.to_f / ds.cases**2)
+			end
+			# Kendall Rank Correlation Coefficient.
+			#
+			# Based on Hervé Adbi article
+			def tau_a(v1,v2)
+				v1a,v2a=Statsample.only_valid(v1,v2)
+				n=v1.size
+				v1r,v2r=v1a.ranked(:scale),v2a.ranked(:scale)
+				o1=ordered_pairs(v1r)
+				o2=ordered_pairs(v2r)
+				delta= o1.size*2-(o2  & o1).size*2
+				1-(delta * 2 / (n*(n-1)).to_f)				
+			end
+			# Calculates Tau b correlation.
+			#
+			# Tau-b defines perfect association as strict monotonicity.
+			# Although it requires strict monotonicity to reach 1.0, 
+			# it does not penalize ties as much as some other measures. 
+			#
+			# Source: http://faculty.chass.ncsu.edu/garson/PA765/assocordinal.htm
+			def tau_b(matrix)
+				v=pairs(matrix)
+				((v['P']-v['Q']).to_f / Math::sqrt((v['P']+v['Q']+v['Y'])*(v['P']+v['Q']+v['X'])).to_f)
+			end
+			# Calculates Goodman and Kruskal's gamma.
+			# 
+			# Gamma is the surplus of concordant pairs over discordant pairs, 
+			# as a percentage of all pairs ignoring ties.
+			# 
+			# Source: http://faculty.chass.ncsu.edu/garson/PA765/assocordinal.htm			
+			def gamma(matrix)
+				v=pairs(matrix)
+				(v['P']-v['Q']).to_f / (v['P']+v['Q']).to_f
+			end
+			# Calculate indexes for a matrix
+			# the rows and cols has to be ordered
+			def pairs(matrix)
+				# calculate concordant
+				#p matrix
+				rs=matrix.row_size
+				cs=matrix.column_size
+				conc=disc=ties_x=ties_y=0
+				(0...(rs-1)).each {|x|
+					(0...(cs-1)).each{|y|
+						((x+1)...rs).each{|x2|
+							((y+1)...cs).each{|y2|
+								#p sprintf("%d:%d,%d:%d",x,y,x2,y2)
+								conc+=matrix[x,y]*matrix[x2,y2]
+							}
+						}
+					}
+				}
+				(0...(rs-1)).each {|x|
+					(1...(cs)).each{|y|
+						((x+1)...rs).each{|x2|
+							(0...y).each{|y2|
+								#p sprintf("%d:%d,%d:%d",x,y,x2,y2)
+								disc+=matrix[x,y]*matrix[x2,y2]
+							}
+						}
+					}
+				}
+				(0...(rs-1)).each {|x|
+					(0...(cs)).each{|y|
+						((x+1)...(rs)).each{|x2|
+							ties_x+=matrix[x,y]*matrix[x2,y]
+						}
+					}
+				}
+				(0...rs).each {|x|
+					(0...(cs-1)).each{|y|
+						((y+1)...(cs)).each{|y2|
+							ties_y+=matrix[x,y]*matrix[x,y2]
+						}
+					}
+				}
+				{'P'=>conc,'Q'=>disc,'Y'=>ties_y,'X'=>ties_x}
+			end
+			def ordered_pairs(vector)
+			d=vector.data
+			a=[]
+			(0...(d.size-1)).each{|i|
+				((i+1)...(d.size)).each {|j|
+					a.push([d[i],d[j]])
+				}
+			}
+			a
+			end
+			def sum_of_codeviated(v1,v2)
+				v1a,v2a=Statsample.only_valid(v1,v2)
+				sum=0
+				(0...v1a.size).each{|i|
+					sum+=v1a[i]*v2a[i]
+				}
+				sum-((v1a.sum*v2a.sum) / v1a.size.to_f)
+			end
+        end
+    end
+end
+

data/lib/statsample/chidistribution.rb

@@ -0,0 +1,39 @@
+module Statsample
+    # Based on Babatunde, Iyiola & Eni () : 
+    # "A Numerical Procedure for Computing Chi-Square Percentage Points"
+    # 
+    module ChiDistribution
+        class << self
+            def steps(av, bv, itv)
+                steps = ((bv.to_f - av.to_f) / itv.to_f).to_i
+            end
+            def loggamma(k)
+            c1 = 76.18009173
+            c2 = -86.50532033
+            c3 = 24.01409822
+            c4 = -1.231739516
+            c5 = 0.00120858
+            c6 = -0.000005364
+            c7 = 2.506628275
+            x1 = k - 1
+            ws = x1 + 5.5
+            ws = (x1 + 0.5) * Math::log(ws) - ws
+            s = 1 + c1 / (x1 + 1) + c2 / (x1 + 2) + c3 / (x1 + 3) + c4 / (x1 + 4) + c5 / (x1 + 5) + c6 / (x1 + 6)
+            ws + Math::log(c7 * s)
+            end
+            def f(x, k)
+                Math::exp(0.5 * k * Math::log(0.5 * x) - Math::log(x) - loggamma(0.5 * k) - 0.5 * x)
+            end
+            def cdf(b,k)
+                a = 0.001
+                b=b.to_f
+                if k==2
+                    1 - Math::exp( -b.to_f / 2)
+                else
+                    w = (b - a) / 28.to_f
+                    2 * w / 45 * (7 * (f(a, k) + f(a + 28 * w, k)) + 12 * (f(a + 2 * w, k) +  f(a + 6 * w, k) + f(a + 10 * w, k) + f(a + 14 * w, k) + f(a + 18 * w, k) + f(a + 22 * w, k) + f(a + 26 * w, k)) + 14 * (f(a + 4 * w, k) + f(a + 8 * w, k) + f(a + 12 * w, k) + f(a + 16 * w, k) + f(a + 20 * w, k) + f(a + 24 * w, k)) + 32 * (f(a + w, k) + f(a + 3 * w, k) + f(a + 5 * w, k) + f(a + 7 * w, k) + f(a + 9 * w, k) + f(a + 11 * w, k) + f(a + 13 * w, k) + f(a + 15 * w, k) + f(a + 17 * w, k) + f(a + 19 * w, k) + f(a + 21 * w, k) + f(a + 23 * w, k) + f(a + 25 * w, k) + f(a + 27 * w, k)))
+                end
+            end
+        end
+    end
+end

data/lib/statsample/codification.rb

@@ -0,0 +1,120 @@
+require 'yaml'
+
+module Statsample
+# Codification
+#
+# This tool aids to code open questions
+# * Load one or more vectors on the workflow, to create a file on yaml of values. If data have Statsample::SEPARATOR_TOKEN, the value will be splitted on two or more values 
+# * Edit the yaml and replace the values with your codes. If you need to create two or mores codes for an answer, use the separator (default Statsample::SEPARATOR_TOKEN)
+# * Recode the vectors, loading the yaml file:
+#   * The new vectors have the same name of the original plus "_recoded"
+#   * Instead of load new recoded vectors, create many vectors as values, as add_vectors_by_split
+#
+# Usage:
+#   recode_file="recodification.yaml"
+#   phase=:first # flag
+#   if phase==:first
+#       File.open(recode_file,"w") {|fp|
+#   Statsample::Codification.create_yaml(ds,%w{vector1 vector2}, ",",fp)
+#   } # Edit the file recodification.yaml
+#   elsif phase==:second 
+#       File.open(recode_file,"r") {|fp|
+#       Statsample::Codification.verify(fp,['vector1'])
+#       }
+#   elsif phase==:third
+#       File.open(recode_file,"r") {|fp|
+#   Statsample::Codification.recode_dataset_split!(ds,fp,"*")
+#   }
+#   end
+#     
+    module Codification
+        class << self
+            # Create a yaml dump for a hash, based on vectors
+            # The keys will be vectors name on dataset and the values
+            # will be hashes, with keys = values, for recodification
+            # 
+            #   v1=%w{a,b b,c d}.to_vector
+            #   ds={"v1"=>v1}.to_dataset
+            #   Statsample::Codification.create_yaml(ds,['v1'])
+            #   => "--- \nv1: \n  a: a\n  b: b\n  c: c\n  d: d\n"
+            def create_yaml(dataset,vectors,sep=Statsample::SPLIT_TOKEN,io=nil)
+                raise ArgumentError,"Array should't be empty" if vectors.size==0
+                pro_hash=vectors.inject({}){|h,v_name|
+                    raise Exception, "Vector #{v_name} doesn't exists on Dataset" if !dataset.fields.include? v_name
+                    v=dataset[v_name]
+                    split_data=v.splitted(sep)
+                    factors=split_data.flatten.uniq.compact.sort.inject({}) {|a,v| a[v]=v;a}
+                    h[v_name]=factors
+                    h
+                }
+                YAML.dump(pro_hash,io)
+            end
+            def inverse_hash(h,sep=Statsample::SPLIT_TOKEN)
+                h.inject({}) {|a,v|
+                    v[1].split(sep).each {|val|
+                        a[val]||=[]
+                        a[val].push(v[0])
+                    }
+                    a
+                }
+            end
+            def dictionary(h,sep=Statsample::SPLIT_TOKEN)
+                h.inject({}) {|a,v|
+                    a[v[0]]=v[1].split(sep)
+                    a
+                }
+            end
+            def recode_vector(v,h,sep=Statsample::SPLIT_TOKEN)
+                dict=dictionary(h,sep)
+                new_data=v.splitted(sep)
+                recoded=new_data.collect{|c|
+                    if c.nil?
+                        nil
+                    else
+                    c.collect{|value|
+                        dict[value]
+                    }.flatten.uniq
+                end
+                }
+            end
+            def recode_dataset_simple!(dataset,yaml,sep=Statsample::SPLIT_TOKEN)
+                _recode_dataset(dataset,yaml,sep,false)
+            end
+            def recode_dataset_split!(dataset,yaml,sep=Statsample::SPLIT_TOKEN)
+                _recode_dataset(dataset,yaml,sep,true)
+            end
+            
+            def _recode_dataset(dataset,yaml,sep=Statsample::SPLIT_TOKEN,split=false)
+                h=YAML::load(yaml)
+                v_names||=h.keys
+                v_names.each do |v_name|
+                    raise Exception, "Vector #{v_name} doesn't exists on Dataset" if !dataset.fields.include? v_name
+                    recoded=recode_vector(dataset[v_name],h[v_name],sep).collect { |c|
+                        if c.nil?
+                            nil
+                        else
+                            c.join(sep)
+                        end
+                    }.to_vector
+                    if(split)
+                    recoded.split_by_separator(sep).each {|k,v|
+                        dataset[v_name+"_"+k]=v
+                    }
+                    else
+                        dataset[v_name+"_recoded"]=recoded
+                    end
+                end
+            end
+            def verify(yaml,v_names=nil,sep=Statsample::SPLIT_TOKEN,io=$>)
+                require 'pp'
+                h=YAML::load(yaml)
+                v_names||=h.keys
+                v_names.each{|v_name|
+                    inverse=inverse_hash(h[v_name],sep)
+                    io.puts "Vector: #{v_name}"
+                    YAML.dump(inverse.sort,io)
+                }
+            end
+        end
+    end
+end