statsample 1.5.0 → 2.0.0

data/lib/statsample/reliability/skillscaleanalysis.rb

@@ -4,11 +4,11 @@ module Statsample
     # Given a dataset with results and a correct answers hash,
     # generates a ScaleAnalysis 
     # == Usage
-    #  x1=%{a b b c}.to_vector
-    #  x2=%{b a b c}.to_vector
-    #  x3=%{a c b a}.to_vector
-    #  ds={'x1'=>@x1,'x2'=>@x2,'x3'=>@x3}.to_dataset
-    #  key={'x1'=>'a','x2'=>'b','x3'=>'a'}    
+    #  x1 = Daru::Vector.new(%{a b b c})
+    #  x2 = Daru::Vector.new(%{b a b c})
+    #  x3 = Daru::Vector.new(%{a c b a})
+    #  ds = Daru::DataFrame.new({:x1 => @x1, :x2 => @x2, :x3 => @x3})
+    #  key={ :x1 => 'a',:x2 => 'b', :x3 => 'a'}    
     #  ssa=Statsample::Reliability::SkillScaleAnalysis.new(ds,key)
     #  puts ssa.summary
     class SkillScaleAnalysis
@@ -30,53 +30,59 @@ module Statsample
       end
       # Dataset only corrected vectors
       def corrected_dataset_minimal
-        cds=corrected_dataset
-        dsm=@key.keys.inject({}) {|ac,v| ac[v]=cds[v];ac}.to_dataset
-        @key.keys.each do |k|
-          dsm[k].name=_("%s(corrected)") % @ds[k].name
-          dsm[k].labels=@ds[k].labels
-        end
+        cds = corrected_dataset
+        dsm = Daru::DataFrame.new(
+          @key.keys.inject({}) do |ac,v| 
+            ac[v] = cds[v]
+            ac
+          end
+        )
         
-        dsm.name=_("Corrected dataset from %s") % @ds.name
+        dsm.rename _("Corrected dataset from %s") % @ds.name
         dsm
       end
+
       def vector_sum
         corrected_dataset_minimal.vector_sum
       end
+
       def vector_mean
         corrected_dataset_minimal.vector_mean
       end
+
       def scale_analysis
-        sa=ScaleAnalysis.new(corrected_dataset_minimal)
+        sa = ScaleAnalysis.new(corrected_dataset_minimal)
         sa.name=_("%s (Scale Analysis)") % @name
         sa
       end
+
       def corrected_dataset
         if @cds.nil?
-          @cds=@ds.dup_empty
-          @key.keys.each {|k| @cds[k].type=:numeric; @cds[k].name=@ds[k].name}
-          @ds.each do |row|
-            out={}
-            row.each do |k,v|
-              if @key.keys.include? k
-                if @ds[k].is_valid? v
-                  out[k]= @key[k]==v ? 1 : 0
+          @cds = Daru::DataFrame.new({}, order: @ds.vectors, name: @ds.name)
+          @ds.each_row do |row|
+            out = {}
+            row.each_with_index do |v, k|
+              if @key.has_key? k
+                if @ds[k].exists? v
+                  out[k]= @key[k] == v ? 1 : 0
                 else
-                  out[k]=nil
+                  out[k] = nil
                 end
               else
-                out[k]=v
+                out[k] = v
               end
             end
-            @cds.add_case(out,false)
+
+            @cds.add_row(Daru::Vector.new(out))
           end
-          @cds.update_valid_data
+          @cds.update
         end
         @cds
       end
+
       def report_building(builder)
         builder.section(:name=>@name) do |s|
-          sa=scale_analysis
+          sa = scale_analysis
           s.parse_element(sa)
           if summary_show_problematic_items
             s.section(:name=>_("Problematic Items")) do |spi|
@@ -91,17 +97,16 @@ module Statsample
                     
                     spi.table(:name=>"Proportions",:header=>[_("Value"), _("%")]) do |table|
                       props.each do |k1,v|
-                        table.row [ @ds[k].labeling(k1), "%0.3f" % v]
+                        table.row [ @ds[k].index_of(k1), "%0.3f" % v]
                       end
                     end
-                    
                   end
                 end
               end
+
               spi.text _("No problematic items") if count==0
             end
           end
-          
         end
       end
     end    

data/lib/statsample/resample.rb

@@ -7,7 +7,7 @@ module Statsample
       
       def generate (size,low,upper)
         range=upper-low+1
-        Vector.new((0...size).collect {|x| rand(range)+low },:numeric)
+        Daru::Vector.new((0...size).collect {|x| rand(range)+low })
       end
               
     end

data/lib/statsample/shorthand.rb

@@ -11,30 +11,20 @@ module Statsample
     ###
     # :section: R like methods
     ###
-    def read_with_cache(klass, filename,opts=Hash.new, cache=true)
-      file_ds=filename+".ds"
-      if cache and (File.exists? file_ds and File.mtime(file_ds)>File.mtime(filename))
-        ds=Statsample.load(file_ds)
-      else
-        ds=klass.read(filename)
-        ds.save(file_ds) if cache
-      end
-      ds
-    end
-    # Import an Excel file. Cache result by default
-    def read_excel(filename, opts=Hash.new, cache=true)
-      read_with_cache(Statsample::Excel, filename, opts, cache)
 
+    # Import an Excel file. Cache result by default
+    def read_excel(filename, opts=Hash.new)
+      Daru::DataFrame.from_excel filename, opts
     end
-    # Import an CSV file. Cache result by default
 
-    def read_csv
-      read_with_cache(Statsample::CSV, filename, opts, cache)
+    # Import an CSV file. Cache result by default
+    def read_csv(filename, opts=Hash.new)
+      Daru::DataFrame.from_csv filename, opts
     end
     
     # Retrieve names (fields) from dataset
     def names(ds)
-      ds.fields
+      ds.vectors.to_a
     end
     # Create a correlation matrix from a dataset
     def cor(ds)
@@ -44,21 +34,25 @@ module Statsample
     def cov(ds)
       Statsample::Bivariate.covariate_matrix(ds)
     end
-    # Create a Statsample::Vector
+    # Create a Daru::Vector
     # Analog to R's c
     def vector(*args)
-      Statsample::Vector[*args]
+      Daru::Vector[*args]
     end
     # Random generation for the normal distribution
     def rnorm(n,mean=0,sd=1)
       rng=Distribution::Normal.rng(mean,sd)
-      Statsample::Vector.new_numeric(n) { rng.call}
+      Daru::Vector.new_with_size(n) { rng.call}
     end
-    # Creates a new Statsample::Dataset
-    # Each key is transformed into string
+    # Creates a new Daru::DataFrame
+    # Each key is transformed into a Symbol wherever possible.
     def dataset(vectors=Hash.new)
-      vectors=vectors.inject({}) {|ac,v| ac[v[0].to_s]=v[1];ac}
-      Statsample::Dataset.new(vectors)
+      vectors = vectors.inject({}) do |ac,v| 
+        n     = v[0].respond_to?(:to_sym) ? v[0].to_sym : v[0] 
+        ac[n] = v[1]
+        ac
+      end
+      Daru::DataFrame.new(vectors)
     end
     alias :data_frame :dataset
     # Returns a Statsample::Graph::Boxplot
@@ -78,13 +72,15 @@ module Statsample
     def levene(*args)
       Statsample::Test::Levene.new(*args)
     end
+
     def principal_axis(*args)
       Statsample::Factor::PrincipalAxis.new(*args)
-      
     end
+
     def polychoric(*args)
       Statsample::Bivariate::Polychoric.new(*args)
     end
+
     def tetrachoric(*args)
       Statsample::Bivariate::Tetrachoric.new(*args)
     end
@@ -95,27 +91,35 @@ module Statsample
     def lr(*args)
       Statsample::Regression.multiple(*args)
     end
+
     def pca(ds,opts=Hash.new)
       Statsample::Factor::PCA.new(ds,opts)
     end
+
     def dominance_analysis(*args)
       Statsample::DominanceAnalysis.new(*args)
     end
+
     def dominance_analysis_bootstrap(*args)
       Statsample::DominanceAnalysis::Bootstrap.new(*args)
     end
+
     def scale_analysis(*args)
       Statsample::Reliability::ScaleAnalysis.new(*args)
     end
+
     def skill_scale_analysis(*args)
       Statsample::Reliability::SkillScaleAnalysis.new(*args)
     end
+
     def multiscale_analysis(*args,&block)
       Statsample::Reliability::MultiScaleAnalysis.new(*args,&block)
     end
+
     def test_u(*args)
       Statsample::Test::UMannWhitney.new(*args)
     end
+    
     module_function :test_u, :rnorm
   end
 end

data/lib/statsample/test/kolmogorovsmirnov.rb

@@ -22,6 +22,7 @@ module Statsample
          end
          calculate
       end
+      
       def calculate
          d=0
         @d1.each {|x|
@@ -31,12 +32,13 @@ module Statsample
         }
         @d=d
       end
+
       # Make a wrapper EmpiricDistribution to any method which implements
-      # each
-      # On Statsample::Vector, only uses #valid_data
+      # each on Statsample::Vector, only uses non-missing data.
       def make_cdf(v)
-        v.is_a?(Statsample::Vector) ? EmpiricDistribution.new(v.valid_data) : EmpiricDistribution.new(v)
+        v.is_a?(Daru::Vector) ? EmpiricDistribution.new(v.only_valid.to_a) : EmpiricDistribution.new(v)
       end
+
       class EmpiricDistribution
         def initialize(data)
           @min=data.min

data/lib/statsample/test/levene.rb

@@ -5,8 +5,8 @@ module Statsample
     # <blockquote>Levene's test ( Levene, 1960) is used to test if k samples have equal variances. Equal variances across samples is called homogeneity of variance. Some statistical tests, for example the analysis of variance, assume that variances are equal across groups or samples. The Levene test can be used to verify that assumption.</blockquote>
     # Use:
     #   require 'statsample'
-    #   a=[1,2,3,4,5,6,7,8,100,10].to_numeric
-    #   b=[30,40,50,60,70,80,90,100,110,120].to_numeric
+    #   a = Daru::Vector.new([1,2,3,4,5,6,7,8,100,10])
+    #   b = Daru::Vector.new([30,40,50,60,70,80,90,100,110,120])
     # 
     #   levene=Statsample::Test::Levene.new([a,b])
     #   puts levene.summary
@@ -29,10 +29,10 @@ module Statsample
       attr_accessor :name
       # Input could be an array of vectors or a dataset
       def initialize(input, opts=Hash.new())
-        if input.is_a? Statsample::Dataset
-          @vectors=input.vectors.values
+        if input.is_a? Daru::DataFrame
+          @vectors = input.to_hash.values
         else
-          @vectors=input
+          @vectors = input
         end
         @name=_("Levene Test")
         opts.each{|k,v|
@@ -48,32 +48,34 @@ module Statsample
         builder.text "%s : F(%d, %d) = %0.4f , p = %0.4f" % [@name, @d1, @d2, f, probability]
       end
       def compute
-        n=@vectors.inject(0) {|ac,v| ac+v.n_valid}
+        n=@vectors.inject(0) { |ac,v| ac + v.n_valid}
         
-        zi=@vectors.collect {|vector|
+        zi=@vectors.collect do |vector|
           mean=vector.mean
-          vector.collect {|v| (v-mean).abs }.to_numeric
-        }
+          Daru::Vector.new(vector.collect { |v| (v - mean).abs })
+        end
         
-        total_mean=zi.inject([]) {|ac,vector|
-          ac+vector.valid_data
-        }.to_numeric.mean
+        total_mean = Daru::Vector.new(
+          zi.inject([]) do |ac,vector|
+            ac + vector.only_valid(:array)
+          end
+        ).mean
       
-        k=@vectors.size
-        
-        sum_num=zi.inject(0) {|ac,vector|
-          ac+(vector.size*(vector.mean-total_mean)**2)
-        }
+        k = @vectors.size
+        sum_num = zi.inject(0) do |ac,vector|
+          ac + (vector.size * (vector.mean - total_mean)**2)
+        end
         
-        sum_den=zi.inject(0) {|ac,vector|
-          z_mean=vector.mean
-          ac+vector.valid_data.inject(0) {|acp,zij|
-            acp+(zij-z_mean)**2
-          }
-        }
-        @w=((n-k)*sum_num).quo((k-1)*sum_den)
-        @d1=k-1
-        @d2=n-k
+        sum_den = zi.inject(0) do |ac,vector|
+          z_mean = vector.mean
+          ac + vector.only_valid(:array).inject(0) do |acp,zij|
+            acp + (zij - z_mean)**2
+          end
+        end
+
+        @w  = ((n - k) * sum_num).quo((k - 1) * sum_den)
+        @d1 = k - 1
+        @d2 = n - k
       end
       private :compute
       # Probability.
@@ -81,7 +83,6 @@ module Statsample
       def probability
         p_using_cdf(Distribution::F.cdf(f, @d1, @d2), :right)
       end
-      
     end
   end
 end

data/lib/statsample/test/t.rb

@@ -1,10 +1,8 @@
 module Statsample
   module Test
-    
-    
-    
-    
-    # A t-test is any statistical hypothesis test in which the test statistic follows a Student's t distribution, if the null hypothesis is supported
+    # A t-test is any statistical hypothesis test in which the test 
+    # statistic follows a Student's t distribution, if the null 
+    # hypothesis is supported
     class T
       
       class << self
@@ -125,7 +123,7 @@ module Statsample
       
       # One Sample t-test
       # == Usage
-      #   a=1000.times.map {rand(100)}.to_numeric
+      #   a = Daru::Vector.new(1000.times.map {rand(100)})
       #   t_1=Statsample::Test::T::OneSample.new(a, {:u=>50})
       #   t_1.summary
       #
@@ -196,8 +194,8 @@ module Statsample
       # Two Sample t-test.
       #
       # == Usage
-      #   a=1000.times.map {rand(100)}.to_numeric
-      #   b=1000.times.map {rand(100)}.to_numeric
+      #   a = Daru::Vector.new(1000.times.map {rand(100)})
+      #   b = Daru::Vector.new(1000.times.map {rand(100)})
       #   t_2=Statsample::Test::T::TwoSamplesIndependent.new(a,b)
       #   t_2.summary
       # === Output
@@ -290,7 +288,7 @@ module Statsample
         def report_building(b) # :nodoc:
           b.section(:name=>@name) {|g|
             g.table(:name=>_("Mean and standard deviation"), :header=>[_("Variable"), _("mean"), _("sd"),_("n")]) {|t|
-              t.row([@v1.name,"%0.4f" % @v1.mean,"%0.4f" % @v1.sd,@v1.n_valid])
+              t.row([@v1.name,"%0.4f" % @v1.mean,"%0.4f" % @v1.sd, @v1.n_valid])
               t.row([@v2.name,"%0.4f" % @v2.mean,"%0.4f" % @v2.sd, @v2.n_valid])
             }
             g.parse_element(Statsample::Test.levene([@v1,@v2],:name=>_("Levene test for equality of variances")))

data/lib/statsample/test/umannwhitney.rb

@@ -113,36 +113,36 @@ module Statsample
       include Summarizable
       #
       # Create a new U Mann-Whitney test
-      # Params: Two Statsample::Vectors
+      # Params: Two Daru::Vectors
       # 
       def initialize(v1,v2, opts=Hash.new)
-        @v1=v1
-        @v2=v2
-        @n1=v1.valid_data.size
-        @n2=v2.valid_data.size
-        data=(v1.valid_data+v2.valid_data).to_numeric
-        groups=(([0]*@n1)+([1]*@n2)).to_vector
-        ds={'g'=>groups, 'data'=>data}.to_dataset
-        @t=nil
-        @ties=data.data.size!=data.data.uniq.size        
-        if(@ties)
-          adjust_for_ties(ds['data'])
+        @v1      = v1
+        @v2      = v2
+        v1_valid = v1.only_valid.reset_index!
+        v2_valid = v2.only_valid.reset_index!
+        @n1      = v1_valid.size
+        @n2      = v2_valid.size
+        data     = Daru::Vector.new(v1_valid.to_a + v2_valid.to_a)
+        groups   = Daru::Vector.new(([0] * @n1) + ([1] * @n2))
+        ds       = Daru::DataFrame.new({:g => groups, :data => data})
+        @t       = nil
+        @ties    = data.to_a.size != data.to_a.uniq.size        
+        if @ties
+          adjust_for_ties(ds[:data])
         end
-        ds['ranked']=ds['data'].ranked(:numeric)
-        
-        @n=ds.cases
+        ds[:ranked] = ds[:data].ranked      
+        @n = ds.nrows
           
-        @r1=ds.filter{|r| r['g']==0}['ranked'].sum
-        @r2=((ds.cases*(ds.cases+1)).quo(2))-r1
-        @u1=r1-((@n1*(@n1+1)).quo(2))
-        @u2=r2-((@n2*(@n2+1)).quo(2))
-        @u=(u1<u2) ? u1 : u2
-        opts_default={:name=>_("Mann-Whitney's U")}
-        @opts=opts_default.merge(opts)
+        @r1 = ds.filter_rows { |r| r[:g] == 0}[:ranked].sum
+        @r2 = ((ds.nrows * (ds.nrows + 1)).quo(2)) - r1
+        @u1 = r1 - ((@n1 * (@n1 + 1)).quo(2))
+        @u2 = r2 - ((@n2 * (@n2 + 1)).quo(2))
+        @u  = (u1 < u2) ? u1 : u2
+        opts_default = { :name=>_("Mann-Whitney's U") }
+        @opts = opts_default.merge(opts)
         opts_default.keys.each {|k|
           send("#{k}=", @opts[k])
-        }
-          
+        }       
       end
       def report_building(generator) # :nodoc:
         generator.section(:name=>@name) do |s|
@@ -160,8 +160,8 @@ module Statsample
       # Exact probability of finding values of U lower or equal to sample on U distribution. Use with caution with m*n>100000.
       # Uses u_sampling_distribution_as62
       def probability_exact
-        dist=UMannWhitney.u_sampling_distribution_as62(@n1,@n2)
-        sum=0
+        dist = UMannWhitney.u_sampling_distribution_as62(@n1,@n2)
+        sum = 0
         (0..@u.to_i).each {|i|
           sum+=dist[i]
         }
@@ -172,8 +172,8 @@ module Statsample
       # == Reference: 
       # * http://europe.isixsigma.com/library/content/c080806a.asp
       def adjust_for_ties(data)
-        @t=data.frequencies.find_all{|k,v| v>1}.inject(0) {|a,v|
-          a+(v[1]**3-v[1]).quo(12)
+        @t = data.frequencies.find_all { |k,v| v > 1 }.inject(0) { |a,v|
+          a + (v[1]**3 - v[1]).quo(12)
         }        
       end