statsample 1.5.0 → 2.0.0

data/examples/velicer_map_test.rb

@@ -1,5 +1,8 @@
 #!/usr/bin/ruby
 $:.unshift(File.dirname(__FILE__)+'/../lib/')
+# == Description
+#
+# Velicer MAP test.
 
 require 'statsample'
 
@@ -15,17 +18,18 @@ Statsample::Analysis.store(Statsample::Factor::MAP) do
   vectors={}
   
   variables.times do |i|
-  vectors["v#{i}"]=samples.times.collect {|nv|    
-  if i<5
-    f1[nv]*5 + f2[nv] *2 +rng.call
-  else
-    f1[nv]*2 + f2[nv] *3 +rng.call
-  end
-  }.to_numeric
+  vectors["v#{i}".to_sym]= Daru::Vector.new(
+    samples.times.collect do |nv|    
+      if i<5
+        f1[nv]*5 + f2[nv] *2 +rng.call
+      else
+        f1[nv]*2 + f2[nv] *3 +rng.call
+      end
+    end)
   end
   
   
-  ds=vectors.to_dataset
+  ds = Daru::DataFrame.new(vectors)
   cor=cor(ds)
   pca=pca(cor)
   

data/lib/statsample.rb

@@ -22,6 +22,8 @@ require 'extendmatrix'
 require 'distribution'
 require 'dirty-memoize'
 require 'reportbuilder'
+require 'daru'
+require 'statsample/daru'
 
 class Numeric
   def square
@@ -52,42 +54,6 @@ class Module
 end
 
 class Array
-  # Recode repeated values on an array, adding the number of repetition
-  # at the end
-  # Example:
-  #   a=%w{a b c c d d d e}
-  #   a.recode_repeated
-  #   => ["a","b","c_1","c_2","d_1","d_2","d_3","e"]
-  def recode_repeated
-    if size != uniq.size
-      # Find repeated
-      repeated = inject({}) do |acc, v|
-        if acc[v].nil?
-          acc[v] = 1
-        else
-          acc[v] += 1
-        end
-        acc
-      end.select { |_k, v| v > 1 }.keys
-
-      ns = repeated.inject({}) do |acc, v|
-        acc[v] = 0
-        acc
-      end
-
-      collect do |f|
-        if repeated.include? f
-          ns[f] += 1
-          sprintf('%s_%d', f, ns[f])
-        else
-          f
-        end
-      end
-    else
-      self
-    end
-  end
-
   def sum
     inject(:+)
   end
@@ -218,7 +184,7 @@ module Statsample
       size = vs[0].size
 
       vs.each do |v|
-        fail ArgumentError, 'Arguments should be Vector' unless v.instance_of? Statsample::Vector
+        fail ArgumentError, 'Arguments should be Vector' unless v.instance_of? Daru::Vector
         fail ArgumentError, 'Vectors size should be the same' if v.size != size
       end
 
@@ -228,26 +194,26 @@ module Statsample
     # Returns a duplicate of the input vectors, without missing data
     # for any of the vectors.
     #
-    #  a=[1,2,3,6,7,nil,3,5].to_numeric
-    #  b=[nil,nil,5,6,4,5,10,2].to_numeric
-    #  c=[2,4,6,7,4,5,6,7].to_numeric
+    #  a = Daru::Vector.new([1,2,3,6,7,nil,3,5])
+    #  b = Daru::Vector.new([nil,nil,5,6,4,5,10,2])
+    #  c = Daru::Vector.new([2,4,6,7,4,5,6,7])
     #  a2,b2,c2=Statsample.only_valid(a,b,c)
-    #  => [#<Statsample::Scale:0xb748c8c8 @data=[3, 6, 7, 3, 5]>,
-    #        #<Statsample::Scale:0xb748c814 @data=[5, 6, 4, 10, 2]>,
-    #        #<Statsample::Scale:0xb748c760 @data=[6, 7, 4, 6, 7]>]
+    #  => [#<Daru::Vector:0xb748c8c8 @data=[3, 6, 7, 3, 5]>,
+    #        #<Daru::Vector:0xb748c814 @data=[5, 6, 4, 10, 2]>,
+    #        #<Daru::Vector:0xb748c760 @data=[6, 7, 4, 6, 7]>]
     #
     def only_valid(*vs)
       i = 1
-      h = vs.inject({}) { |acc, v| acc["v#{i}"] = v; i += 1; acc }
-      ds = Statsample::Dataset.new(h).dup_only_valid
-      ds.vectors.values
+      h = vs.inject({}) { |acc, v| acc["v#{i}".to_sym] = v; i += 1; acc }
+      df = Daru::DataFrame.new(h).dup_only_valid
+      df.map { |v| v }
     end
 
     # Cheap version of #only_valid.
     # If any vectors have missing_values, return only valid.
     # If not, return the vectors itself
     def only_valid_clone(*vs)
-      if vs.any?(&:flawed?)
+      if vs.any?(&:has_missing_data?)
         only_valid(*vs)
       else
         vs

data/lib/statsample/analysis/suite.rb

@@ -80,7 +80,7 @@ module Statsample
       
       def method_missing(name, *args,&block)
         @attached.reverse.each do |ds|
-          return ds[name.to_s] if ds.fields.include? (name.to_s)
+          return ds[name] if ds.vectors.to_a.include? (name)
         end
         raise "Method #{name} doesn't exists"
       end

data/lib/statsample/anova/oneway.rb

@@ -67,9 +67,9 @@ module Statsample
     
     # One Way Anova with vectors
     # Example:
-    #   v1=[2,3,4,5,6].to_numeric
-    #   v2=[3,3,4,5,6].to_numeric
-    #   v3=[5,3,1,5,6].to_numeric
+    #   v1 = Daru::Vector.new([2,3,4,5,6])
+    #   v2 = Daru::Vector.new([3,3,4,5,6])
+    #   v3 = Daru::Vector.new([5,3,1,5,6])
     #   anova=Statsample::Anova::OneWayWithVectors.new([v1,v2,v3])
     #   anova.f
     #   => 0.0243902439024391
@@ -90,10 +90,10 @@ module Statsample
       
       def initialize(*args)
         if args[0].is_a? Array
-          @vectors=args.shift
+          @vectors = args.shift
         else
-          @vectors=args.find_all {|v| v.is_a? Statsample::Vector}
-          opts=args.find {|v| v.is_a? Hash}
+          @vectors = args.find_all {|v| v.is_a? Daru::Vector}
+          opts     = args.find {|v| v.is_a? Hash}
         end
         opts||=Hash.new
         opts_default={:name=>_("Anova One-Way"), 

data/lib/statsample/anova/twoway.rb

@@ -107,9 +107,9 @@ module Statsample
     
     # Two Way Anova with vectors
     # Example:
-    #   v1=[1,1,2,2].to_numeric
-    #   v2=[1,2,1,2].to_numeric
-    #   v3=[5,3,1,5].to_numeric
+    #   v1 = Daru::Vector.new([1,1,2,2])
+    #   v2 = Daru::Vector.new([1,2,1,2])
+    #   v3 = Daru::Vector.new([5,3,1,5])
     #   anova=Statsample::Anova::TwoWayWithVectors.new(:a=>v1,:b=>v2, :dependent=>v3)
     #
     class TwoWayWithVectors < TwoWay
@@ -121,25 +121,26 @@ module Statsample
       # For now, only equal sample cells allowed
       def initialize(opts=Hash.new)
         raise "You should insert at least :a, :b and :dependent" unless  [:a, :b, :dependent].all? {|v| opts.has_key? v}
-        @a_var='a'
-        @b_var='b'
-        @dep_var='dependent'
-        @a_vector, @b_vector, @dep_vector=Statsample.only_valid_clone opts[:a], opts[:b], opts[:dependent]
+        @a_var   = :a
+        @b_var   = :b
+        @dep_var = :dependent
+        @a_vector, @b_vector, @dep_vector = 
+          Statsample.only_valid_clone opts[:a], opts[:b], opts[:dependent]
         
-        ds={@a_var=>@a_vector, @b_var=>@b_vector, @dep_var=>@dep_vector}.to_dataset
-        @ds=ds.clone_only_valid
-        _p=@a_vector.factors.size
-        _q=@b_vector.factors.size
-        @x_general=@dep_vector.mean
-        @axb_means={}
-        @axb_sd={}
-        @vectors=[]
+        ds  = Daru::DataFrame.new({@a_var=>@a_vector, @b_var=>@b_vector, @dep_var=>@dep_vector})
+        @ds = ds.clone_only_valid
+        _p  = @a_vector.factors.size
+        _q  = @b_vector.factors.size
+        @x_general = @dep_vector.mean
+        @axb_means = {}
+        @axb_sd    = {}
+        @vectors   = []
         n=nil
         @ds.to_multiset_by_split(a_var,b_var).each_vector(dep_var) {|k,v|
-          @axb_means[k]=v.mean
-          @axb_sd[k]=v.sd
+          @axb_means[k] = v.mean
+          @axb_sd[k]    = v.sd
           @vectors << v
-          n||=v.size
+          n ||= v.size
           raise "All cell sizes should be equal" if n!=v.size
         }
 
@@ -151,20 +152,21 @@ module Statsample
         @ds.to_multiset_by_split(b_var).each_vector(dep_var) {|k,v|
           @b_means[k]=v.mean
         }
-        ss_a=n*_q*@ds[a_var].factors.inject(0) {|ac,v|
-          ac+(@a_means[v]-@x_general)**2
+        ss_a = n*_q*@ds[a_var].factors.inject(0) {|ac,v|
+          ac + (@a_means[v]-@x_general)**2
         }
         ss_b=n*_p*@ds[b_var].factors.inject(0) {|ac,v|
           ac+(@b_means[v]-@x_general)**2
         }
-        ss_within=@ds.collect {|row|
+        ss_within = @ds.collect(:row) { |row|
           (row[dep_var]-@axb_means[[row[a_var],row[b_var]]])**2
         }.sum
-        ss_axb=n*@axb_means.inject(0) {|ac,v|
+        ss_axb = n*@axb_means.inject(0) {|ac,v|
           j,k=v[0]
           xjk=v[1]
           ac+(xjk-@a_means[j]-@b_means[k]+@x_general)**2
         }
+
         df_a=_p-1
         df_b=_q-1
         df_within=(_p*_q)*(n-1)
@@ -186,9 +188,9 @@ module Statsample
       def report_building(builder) #:nodoc:#
         builder.section(:name=>@name) do |s|
           if summary_descriptives
-            s.table(:header =>['']+@ds[a_var].factors.map {|a| @ds[a_var].labeling(a)}+[_("%s Mean") % @name_b]) do |t|
+            s.table(:header =>['']+@ds[a_var].factors.map {|a| @ds[a_var].index_of(a)}+[_("%s Mean") % @name_b]) do |t|
               @ds[b_var].factors.each do |b|
-                t.row([@ds[b_var].labeling(b)]+@ds[a_var].factors.map {|a| "%0.3f" % @axb_means[[a,b]] } + ["%0.3f" % @b_means[b]])
+                t.row([@ds[b_var].index_of(b)]+@ds[a_var].factors.map {|a| "%0.3f" % @axb_means[[a,b]] } + ["%0.3f" % @b_means[b]])
               end
               t.row([_("%s Mean") % @name_a]+@ds[a_var].factors.map {|a| "%0.3f" % @a_means[a]}+ ["%0.3f" % @x_general])
             end

data/lib/statsample/bivariate.rb

@@ -12,9 +12,10 @@ module Statsample
       # Covariance between two vectors
       def covariance(v1,v2)
         v1a,v2a=Statsample.only_valid_clone(v1,v2)
+
         return nil if v1a.size==0
         if Statsample.has_gsl?
-          GSL::Stats::covariance(v1a.gsl, v2a.gsl)
+          GSL::Stats::covariance(v1a.to_gsl, v2a.to_gsl)
         else
           covariance_slow(v1a,v2a)
         end
@@ -34,7 +35,9 @@ module Statsample
         sum_of_squares(v1a,v2a) / (v1a.size-1)
       end
       def sum_of_squares(v1,v2)
-        v1a,v2a=Statsample.only_valid_clone(v1,v2)        
+        v1a,v2a=Statsample.only_valid_clone(v1,v2)
+        v1a.reset_index!
+        v2a.reset_index!        
         m1=v1a.mean
         m2=v2a.mean
         (v1a.size).times.inject(0) {|ac,i| ac+(v1a[i]-m1)*(v2a[i]-m2)}
@@ -44,13 +47,14 @@ module Statsample
         v1a,v2a=Statsample.only_valid_clone(v1,v2)
         return nil if v1a.size ==0
         if Statsample.has_gsl?
-          GSL::Stats::correlation(v1a.gsl, v2a.gsl)
+          GSL::Stats::correlation(v1a.to_gsl, v2a.to_gsl)
         else
           pearson_slow(v1a,v2a)
         end
       end
       def pearson_slow(v1,v2) # :nodoc:
         v1a,v2a=Statsample.only_valid_clone(v1,v2)
+
         # Calculate sum of squares
         ss=sum_of_squares(v1a,v2a)
         ss.quo(Math::sqrt(v1a.sum_of_squares) * Math::sqrt(v2a.sum_of_squares))
@@ -118,14 +122,16 @@ module Statsample
         r=Statsample::Bivariate.pearson(from,del)
         froms, dels = from.vector_standarized, del.vector_standarized
         nv=[]
-        froms.data_with_nils.each_index do |i|
+        froms.reset_index!
+        dels.reset_index!
+        froms.each_index do |i|
           if froms[i].nil? or dels[i].nil?
             nv.push(nil)
           else
             nv.push(froms[i]-r*dels[i])
           end
         end
-        nv.to_vector(:numeric)
+        Daru::Vector.new(nv)
       end
       # Correlation between v1 and v2, controling the effect of
       # control on both.
@@ -135,7 +141,6 @@ module Statsample
         rv1con=pearson(v1a,cona)
         rv2con=pearson(v2a,cona)        
         (rv1v2-(rv1con*rv2con)).quo(Math::sqrt(1-rv1con**2) * Math::sqrt(1-rv2con**2))
-        
       end
       
       def covariance_matrix_optimized(ds)
@@ -153,50 +158,53 @@ module Statsample
       # Order of rows and columns depends on Dataset#fields order
       
       def covariance_matrix(ds)
-        vars,cases=ds.fields.size,ds.cases
+        vars,cases = ds.ncols, ds.nrows
         if !ds.has_missing_data? and Statsample.has_gsl? and prediction_optimized(vars,cases) < prediction_pairwise(vars,cases)
           cm=covariance_matrix_optimized(ds)
         else
           cm=covariance_matrix_pairwise(ds)
-          
         end
         cm.extend(Statsample::CovariateMatrix)
-        cm.fields=ds.fields
+        cm.fields = ds.vectors.to_a
         cm
       end
       
       
       def covariance_matrix_pairwise(ds)
         cache={}
-        matrix=ds.collect_matrix do |row,col|
-          if (ds[row].type!=:numeric or ds[col].type!=:numeric)
-            nil
-          elsif row==col
-            ds[row].variance
-          else
-            if cache[[col,row]].nil?
-              cov=covariance(ds[row],ds[col])
-              cache[[row,col]]=cov
-              cov
+        vectors = ds.vectors.to_a
+        mat_rows = vectors.collect do |row|
+          vectors.collect do |col|
+            if (ds[row].type!=:numeric or ds[col].type!=:numeric)
+              nil
+            elsif row==col
+              ds[row].variance
             else
-               cache[[col,row]]
+              if cache[[col,row]].nil?
+                cov=covariance(ds[row],ds[col])
+                cache[[row,col]]=cov
+                cov
+              else
+                cache[[col,row]]
+              end
             end
           end
         end
-        matrix
+        
+        Matrix.rows mat_rows
       end
       
       # Correlation matrix.
       # Order of rows and columns depends on Dataset#fields order
       def correlation_matrix(ds)
-        vars,cases=ds.fields.size,ds.cases
+        vars, cases = ds.ncols, ds.nrows
         if !ds.has_missing_data? and Statsample.has_gsl? and prediction_optimized(vars,cases) < prediction_pairwise(vars,cases)
           cm=correlation_matrix_optimized(ds)
         else
           cm=correlation_matrix_pairwise(ds)
         end
         cm.extend(Statsample::CovariateMatrix)
-        cm.fields=ds.fields
+        cm.fields = ds.vectors.to_a
         cm
       end
 
@@ -212,33 +220,43 @@ module Statsample
       end
       def correlation_matrix_pairwise(ds)
         cache={}
-        cm=ds.collect_matrix do |row,col|
-          if row==col
-            1.0
-          elsif (ds[row].type!=:numeric or ds[col].type!=:numeric)
-            nil
-          else
-            if cache[[col,row]].nil?
-              r=pearson(ds[row],ds[col])
-              cache[[row,col]]=r
-              r
+        vectors = ds.vectors.to_a
+        cm = vectors.collect do |row|
+          vectors.collect do |col|
+            if row==col
+              1.0
+            elsif (ds[row].type!=:numeric or ds[col].type!=:numeric)
+              nil
             else
-              cache[[col,row]]
-            end 
+              if cache[[col,row]].nil?
+                r=pearson(ds[row],ds[col])
+                cache[[row,col]]=r
+                r
+              else
+                cache[[col,row]]
+              end 
+            end
           end
         end
+
+        Matrix.rows cm
       end
       
       # Retrieves the n valid pairwise.
       def n_valid_matrix(ds)
-        ds.collect_matrix do |row,col|
-          if row==col
-            ds[row].valid_data.size
-          else
-            rowa,rowb=Statsample.only_valid_clone(ds[row],ds[col])
-            rowa.size
+        vectors = ds.vectors.to_a
+        m = vectors.collect do |row|
+          vectors.collect do |col|
+            if row==col
+              ds[row].only_valid.size
+            else
+              rowa,rowb = Statsample.only_valid_clone(ds[row],ds[col])
+              rowa.size
+            end
           end
         end
+
+        Matrix.rows m
       end
       
       # Matrix of correlation probabilities.
@@ -256,27 +274,27 @@ module Statsample
       
       # Spearman ranked correlation coefficient (rho) between 2 vectors
       def spearman(v1,v2)
-        v1a,v2a=Statsample.only_valid_clone(v1,v2)
-        v1r,v2r=v1a.ranked(:numeric),v2a.ranked(:numeric)
+        v1a,v2a = Statsample.only_valid_clone(v1,v2)
+        v1r,v2r = v1a.ranked, v2a.ranked
         pearson(v1r,v2r)
       end
       # Calculate Point biserial correlation. Equal to Pearson correlation, with
       # one dichotomous value replaced by "0" and the other by "1"
       def point_biserial(dichotomous,continous)
-        ds={'d'=>dichotomous,'c'=>continous}.to_dataset.dup_only_valid
-        raise(TypeError, "First vector should be dichotomous") if ds['d'].factors.size!=2
-        raise(TypeError, "Second vector should be continous") if ds['c'].type!=:numeric
-        f0=ds['d'].factors.sort[0]
-        m0=ds.filter_field('c') {|c| c['d']==f0}
-        m1=ds.filter_field('c') {|c| c['d']!=f0}
-        ((m1.mean-m0.mean).to_f / ds['c'].sdp) * Math::sqrt(m0.size*m1.size.to_f / ds.cases**2)
+        ds = Daru::DataFrame.new({:d=>dichotomous,:c=>continous}).dup_only_valid
+        raise(TypeError, "First vector should be dichotomous") if ds[:d].factors.size != 2
+        raise(TypeError, "Second vector should be continous") if ds[:c].type != :numeric
+        f0=ds[:d].factors.sort.to_a[0]
+        m0=ds.filter_vector(:c) {|c| c[:d] == f0}
+        m1=ds.filter_vector(:c) {|c| c[:d] != f0}
+        ((m1.mean-m0.mean).to_f / ds[:c].sdp) * Math::sqrt(m0.size*m1.size.to_f / ds.nrows**2)
       end
       # Kendall Rank Correlation Coefficient (Tau a)
       # Based on Hervé Adbi article
       def tau_a(v1,v2)
         v1a,v2a=Statsample.only_valid_clone(v1,v2)
         n=v1.size
-        v1r,v2r=v1a.ranked(:numeric),v2a.ranked(:numeric)
+        v1r,v2r=v1a.ranked,v2a.ranked
         o1=ordered_pairs(v1r)
         o2=ordered_pairs(v2r)
         delta= o1.size*2-(o2  & o1).size*2
@@ -348,14 +366,15 @@ module Statsample
         }
         {'P'=>conc,'Q'=>disc,'Y'=>ties_y,'X'=>ties_x}
       end
+
       def ordered_pairs(vector)
-        d=vector.data
-        a=[]
-        (0...(d.size-1)).each{|i|
-          ((i+1)...(d.size)).each {|j|
+        d = vector.to_a
+        a = []
+        (0...(d.size-1)).each do |i|
+          ((i+1)...(d.size)).each do |j|
             a.push([d[i],d[j]])
-          }
-        }
+          end
+        end
         a
       end
 =begin      
@@ -371,8 +390,8 @@ module Statsample
       # Report the minimum number of cases valid of a covariate matrix
       # based on a dataset
       def min_n_valid(ds)
-        min=ds.cases
-        m=n_valid_matrix(ds)
+        min = ds.nrows
+        m   = n_valid_matrix(ds)
         for x in 0...m.row_size
           for y in 0...m.column_size
             min=m[x,y] if m[x,y] < min
@@ -380,8 +399,6 @@ module Statsample
         end
         min
       end
-      
-      
     end
   end
 end