statsample 0.15.1 → 0.16.0

data/History.txt

@@ -1,3 +1,11 @@
+=== 0.16.0 / 2010-11-13
+* Works on ruby 1.9.2 and HEAD. Updated Rakefile and manifest
+* Removed all graph based on Svg::Graph. 
+* First operative version of Graph with Rubyvis
+* Corrected bug on Distribution::Normal.cdf. 
+* Added reference on references.txt
+* Ruby-based random gaussian distribution generator when gsl not available
+* Added population average deviation [Al Chou]
 === 0.15.1 / 2010-10-20
 * Statsample::Excel and Statsample::PlainText add name to vectors equal to field name 
 * Statsample::Dataset.delete_vector accept multiple fields.

data/Manifest.txt

@@ -25,6 +25,7 @@ examples/parallel_analysis_tetrachoric.rb
 examples/polychoric.rb
 examples/principal_axis.rb
 examples/reliability.rb
+examples/scatterplot.rb
 examples/t_test.rb
 examples/tetrachoric.rb
 examples/u_test.rb
@@ -61,11 +62,7 @@ lib/statsample/factor/pca.rb
 lib/statsample/factor/principalaxis.rb
 lib/statsample/factor/rotation.rb
 lib/statsample/graph.rb
-lib/statsample/graph/gdchart.rb
-lib/statsample/graph/svgboxplot.rb
-lib/statsample/graph/svggraph.rb
-lib/statsample/graph/svghistogram.rb
-lib/statsample/graph/svgscatterplot.rb
+lib/statsample/graph/scatterplot.rb
 lib/statsample/histogram.rb
 lib/statsample/matrix.rb
 lib/statsample/mle.rb
@@ -139,7 +136,6 @@ test/test_srs.rb
 test/test_statistics.rb
 test/test_stest.rb
 test/test_stratified.rb
-test/test_svg_graph.rb
 test/test_test_f.rb
 test/test_test_t.rb
 test/test_umannwhitney.rb

data/Rakefile

@@ -41,9 +41,9 @@ h=Hoe.spec('statsample') do
   #self.testlib=:minitest
 	self.rubyforge_name = "ruby-statsample"
 	self.developer('Claudio Bustos', 'clbustos@gmail.com')
-	self.extra_deps << ["spreadsheet","~>0.6.0"] << ["svg-graph", "~>1.0"] << ["reportbuilder", "~>1.0"] << ["minimization", "~>0.2.0"] << ["fastercsv"] << ["dirty-memoize", "~>0.0"] << ["extendmatrix","~>0.3.1"] << ["statsample-bivariate-extension", "~>0.13.0"]
+	self.extra_deps << ["spreadsheet","~>0.6.0"] <<  ["reportbuilder", "~>1.0"] << ["minimization", "~>0.2.0"] << ["fastercsv"] << ["dirty-memoize", "~>0.0"] << ["extendmatrix","~>0.3.1"] << ["statsample-bivariate-extension", ">0"] << ["rserve-client", "~>0.2.5"] << ["rubyvis", "~>0.2.2"]
   
-	self.extra_dev_deps << ["shoulda"]
+	self.extra_dev_deps << ["shoulda"] << ["minitest", "~>2.0"]
   self.clean_globs << "test/images/*" << "demo/item_analysis/*" << "demo/Regression"
   self.post_install_message = <<-EOF
 ***************************************************

data/examples/dominance_analysis.rb

@@ -9,7 +9,7 @@ c=sample.times.collect {rand}.to_scale
 d=sample.times.collect {rand}.to_scale
 
 ds={'a'=>a,'b'=>b,'c'=>c,'d'=>d}.to_dataset
-ds['y']=ds.collect{|row| row['a']*5+row['b']*3+row['c']*2+row['d']+rand()}
+ds['y']=ds.collect{|row| row['a']*5 + row['b']*3 + row['c']*2 + row['d'] + rand()}
 rb=ReportBuilder.new(:name=>"Dominance Analysis")
 
 cm=Statsample::Bivariate.correlation_matrix(ds)
@@ -23,5 +23,4 @@ rb.add(lr)
 da=Statsample::DominanceAnalysis.new(ds,'y',:name=>"Dominance Analysis using group of predictors", :predictors=>['a', 'b', %w{c d}])
 rb.add(da)
 
-
 puts rb.to_text

data/examples/multivariate_correlation.rb

@@ -16,6 +16,7 @@ require 'mathn'
   [0.02,0.02,0.03,0,0.06,0.22,1,-0.01,-0.05],
   [-0.12,-0.1,-0.06,-0.02,0.18,-0.07,-0.01,1,-0.03],
   [0.08,0.15,0.12,-0.02,0.02,0.36,-0.05,-0.03,1]]
+  
   complete.extend Statsample::CovariateMatrix
   complete.fields=%w{adhd cd odd sex age monly mwork mage poverty}
   

data/examples/parallel_analysis.rb

@@ -14,17 +14,18 @@ vectors={}
 
 variables.times do |i|
   vectors["v#{i}"]=samples.times.collect {|nv| f1[nv]*i+(f2[nv]*(15-i))+((f3[nv]*(30-i))*1.5)*rng.ugaussian()}.to_scale
+  vectors["v#{i}"].name="Vector #{i}"
 end
 ds=vectors.to_dataset
 
 pa=Statsample::Factor::ParallelAnalysis.new(ds, :iterations=>iterations, :debug=>true)
 pca=Statsample::Factor::PCA.new(Statsample::Bivariate.correlation_matrix(ds))
-rb=ReportBuilder.new(:name=>"Parallel Analysis with simulation") do |g|
-  g.text("There are 3 real factors on data")
-  g.parse_element(pca)
-  g.text("Traditional Kaiser criterion (k>1) returns #{pca.m} factors")
-  g.parse_element(pa)
-  g.text("Parallel Analysis returns #{pa.number_of_factors} factors to preserve")
+rb=ReportBuilder.new(:name=>"Parallel Analysis with simulation") do
+  text "There are 3 real factors on data"
+  parse_element pca
+  text "Traditional Kaiser criterion (k>1) returns #{pca.m} factors"
+  parse_element pa
+  text "Parallel Analysis returns #{pa.number_of_factors} factors to preserve"
 end
 
 puts rb.to_text

data/examples/scatterplot.rb

@@ -0,0 +1,12 @@
+#!/usr/bin/ruby
+$:.unshift(File.dirname(__FILE__)+'/../lib/')
+require 'benchmark'
+require 'statsample'
+n=100
+a=n.times.map {|i| rand(10)+i}.to_scale
+b=n.times.map {|i| rand(10)+i}.to_scale
+sp=Statsample::Graph::Scatterplot.new(a,b, :width=>200, :height=>200)
+rb=ReportBuilder.new do |b|
+  b.parse_element(sp)
+end  
+puts rb.to_text

data/lib/distribution/normal.rb

@@ -3,6 +3,34 @@ module Distribution
   # Uses Statistics2 module
   module Normal
     class << self
+      # Return a proc which return a random number within a gaussian distribution -> N(0,1)
+      # == Reference:
+      # * http://www.taygeta.com/random/gaussian.html
+      def rng_ugaussian
+        if Distribution.has_gsl?
+          rng=GSL::Rng.alloc()
+          lambda { rng.ugaussian()}
+        else
+          returned,y1,y2=0,0,0
+          lambda {
+            if returned==0
+              begin
+                x1 = 2.0 * rand - 1.0
+                x2 = 2.0 * rand - 1.0
+                w = x1 * x1 + x2 * x2
+              end while ( w >= 1.0 )
+              w = Math::sqrt( (-2.0 * Math::log( w ) ) / w )
+              y1 = x1 * w
+              y2 = x2 * w
+              returned=1
+              y1
+            else
+              returned=0
+              y2
+            end
+          }
+        end
+      end
       # Return the P-value of the corresponding integral
       def p_value(pr)
           Statistics2.pnormaldist(pr)
@@ -16,7 +44,7 @@ module Distribution
         Statistics2.normaldist(x)
       end
 
-      if false and Distribution.has_gsl?
+      if Distribution.has_gsl?
         alias :cdf_ruby :cdf 
         def cdf(x) # :nodoc:
           GSL::Cdf::gaussian_P(x)

data/lib/statsample.rb

@@ -20,10 +20,13 @@
 
 #$:.unshift(File.dirname(__FILE__))
 require 'matrix'
+require 'extendmatrix'
 require 'distribution'
 require 'dirty-memoize'
 require 'reportbuilder'
 
+
+
 class Numeric
   def square ; self * self ; end
 end
@@ -115,7 +118,7 @@ module Statsample
     @@has_gsl
   end
   
-  VERSION = '0.15.1'
+  VERSION = '0.16.0'
   SPLIT_TOKEN = ","
   autoload(:Database, 'statsample/converters')
   autoload(:Anova, 'statsample/anova')
@@ -253,4 +256,4 @@ end
 require 'statsample/vector'
 require 'statsample/dataset'
 require 'statsample/crosstab'
-require 'statsample/matrix'
+require 'statsample/matrix'

data/lib/statsample/bivariate.rb

@@ -7,8 +7,6 @@ module Statsample
   # * Statsample::Bivariate::Tetrachoric : Tetrachoric correlation
   # * Statsample::Bivariate::Polychoric  : Polychoric correlation (using joint, two-step and polychoric series)
   module Bivariate
-    autoload(:Polychoric, "statsample/bivariate/polychoric")
-    autoload(:Tetrachoric, "statsample/bivariate/tetrachoric")
     
     class << self
       # Covariance between two vectors
@@ -337,3 +335,6 @@ module Statsample
   end
 end
 
+require 'statsample/bivariate/polychoric'
+require 'statsample/bivariate/tetrachoric'
+

data/lib/statsample/converters.rb

@@ -89,7 +89,7 @@ module Statsample
       end
                                          
       def process_row(row,empty)
-        row.to_a.collect do |c|
+        row.to_a.map do |c|
           if empty.include?(c)
               nil
           else

data/lib/statsample/dataset.rb

@@ -89,8 +89,6 @@ module Statsample
     #    b  1   0
     #
     # Useful to process outputs from databases
-    #
-    
     def self.crosstab_by_asignation(rows,columns,values)
       raise "Three vectors should be equal size" if rows.size!=columns.size or rows.size!=values.size
       cols_values=columns.factors
@@ -125,8 +123,7 @@ module Statsample
     # a variable of the Dataset named as its key
     # [fields]  Array of names for vectors. Is only used for set the
     # order of variables. If empty, vectors keys on alfabethic order as
-    # used as fields
-    #
+    # used as fields.
     def initialize(vectors={}, fields=[])
       @@n_dataset||=0
       @@n_dataset+=1
@@ -154,10 +151,12 @@ module Statsample
       end
       matrix
     end
-
+    # 
     # Creates a copy of the given dataset, deleting all the cases with
-    # missing data on one of the vectors
+    # missing data on one of the vectors.
+    # 
     # @param array of fields to include. No value include all fields
+    #
     def dup_only_valid(*fields_to_include)
       if fields_to_include.size==1 and fields_to_include[0].is_a? Array
         fields_to_include=fields_to_include[0]
@@ -177,10 +176,14 @@ module Statsample
       end
       ds
     end
-    
-    # Returns a duplicate of the Database
-    # If fields given, only include those vectors.
+    #
+    # Returns a duplicate of the Dataset. 
+    # All vectors are copied, so any modification on new
+    # dataset doesn't affect original dataset's vectors.
+    # If fields given as parameter, only include those vectors.
+    #
     # @param array of fields to include. No value include all fields    
+    # @return {Statsample::Dataset}
     def dup(*fields_to_include)
       if fields_to_include.size==1 and fields_to_include[0].is_a? Array
         fields_to_include=fields_to_include[0]
@@ -206,9 +209,10 @@ module Statsample
     
     # Returns (when possible) a cheap copy of dataset.
     # If no vector have missing values, returns original vectors.
-    # If missing values presents, uses Dataset.dup_only_valid
+    # If missing values presents, uses Dataset.dup_only_valid.
+    #
     # @param array of fields to include. No value include all fields
-
+    # @return {Statsample::Dataset}
     def clone_only_valid(*fields_to_include)
       if fields_to_include.size==1 and fields_to_include[0].is_a? Array
         fields_to_include=fields_to_include[0]
@@ -223,6 +227,7 @@ module Statsample
     # Returns a shallow copy of Dataset.
     # Object id will be distinct, but @vectors will be the same.
     # @param array of fields to include. No value include all fields
+    # @return {Statsample::Dataset}    
     def clone(*fields_to_include)
       if fields_to_include.size==1 and fields_to_include[0].is_a? Array
         fields_to_include=fields_to_include[0]
@@ -238,6 +243,8 @@ module Statsample
       ds
     end
     # Creates a copy of the given dataset, without data on vectors
+    #
+    # @return {Statsample::Dataset}
     def dup_empty
       vectors=@vectors.inject({}) {|a,v|
         a[v[0]]=v[1].dup_empty
@@ -248,6 +255,8 @@ module Statsample
     # Merge vectors from two datasets
     # In case of name collition, the vectors names are changed to 
     # x_1, x_2 ....
+    #
+    # @return {Statsample::Dataset}
     def merge(other_ds)
       raise "Cases should be equal (this:#{@cases}; other:#{other_ds.cases}" unless @cases==other_ds.cases
       types = @fields.collect{|f| @vectors[f].type} + other_ds.fields.collect{|f| other_ds[f].type}
@@ -264,7 +273,9 @@ module Statsample
       ds_new.update_valid_data
       ds_new
     end
-    # Returns a dataset with standarized data
+    # Returns a dataset with standarized data.
+    #
+    # @return {Statsample::Dataset}
     def standarize
       ds=dup()
       ds.fields.each do |f|
@@ -273,6 +284,9 @@ module Statsample
       ds
     end
     # Generate a matrix, based on fields of dataset
+    #
+    # @return {::Matrix}
+    
     def collect_matrix
       rows=@fields.collect{|row|
         @fields.collect{|col|
@@ -281,27 +295,39 @@ module Statsample
       }
       Matrix.rows(rows)
     end
+    
     # We have the same datasets if +vectors+ and +fields+ are the same
+    #
+    # @return {Boolean}
     def ==(d2)
       @vectors==d2.vectors and @fields==d2.fields
     end
     # Returns vector <tt>c</tt>
+    # 
+    # @return {Statsample::Vector}
     def col(c)
       @vectors[c]
     end
     alias_method :vector, :col
     # Equal to Dataset[<tt>name</tt>]=<tt>vector</tt>
+    #
+    # @return self
     def add_vector(name, vector)
       raise ArgumentError, "Vector have different size" if vector.size!=@cases
       @vectors[name]=vector
       check_order
+      self
     end
-    # Returns true if dataset have vector <tt>v</tt>
+    # Returns true if dataset have vector <tt>v</tt>.
+    #
+    # @return {Boolean}
     def has_vector? (v)
       return @vectors.has_key?(v)
     end
     # Creates a dataset with the random data, of a n size
-    # If n not given, uses original number of cases
+    # If n not given, uses original number of cases.
+    #
+    # @return {Statsample::Dataset}
     def bootstrap(n=nil)
       n||=@cases
       ds_boot=dup_empty
@@ -314,6 +340,8 @@ module Statsample
     # Fast version of #add_case.
     # Can only add one case and no error check if performed
     # You SHOULD use #update_valid_data at the end of insertion cycle
+    #
+    # 
     def add_case_array(v)
       v.each_index {|i| d=@vectors[@fields[i]].data; d.push(v[i])}
     end

data/lib/statsample/factor/parallelanalysis.rb

@@ -10,12 +10,12 @@ module Statsample
     # == Usage
     # *With real dataset*
     #   # ds should be any valid dataset
-    #   pa=Statsample::Factor::ParallelAnalysis.new(ds, :iterations=>100, :bootstrap_method=>:raw_data)
+    #   pa=Statsample::Factor::ParallelAnalysis.new(ds, :iterations=>100, :bootstrap_method=>:data)
     #
     # *With number of cases and variables*
     #   pa=Statsample::Factor::ParallelAnalysis.with_random_data(100,8)
     # 
-    # == Reference:
+    # == Reference
     # * Hayton, J., Allen, D. & Scarpello, V.(2004). Factor Retention Decisions in Exploratory Factor Analysis: a Tutorial on Parallel Analysis. <i>Organizational Research Methods, 7</i> (2), 191-205.
     # * O'Connor, B. (2000). SPSS and SAS programs for determining the number of components using parallel analysis and Velicer's MAP test. Behavior Research Methods, Instruments, & Computers, 32(3), 396-402.
     # * Liu, O., & Rijmen, F. (2008). A modified procedure for parallel analysis of ordered categorical data. Behavior Research Methods, 40(2), 556-562.
@@ -127,7 +127,7 @@ module Statsample
         @ds_eigenvalues=Statsample::Dataset.new((1..@n_variables).map{|v| "ev_%05d" % v})
         @ds_eigenvalues.fields.each {|f| @ds_eigenvalues[f].type=:scale}
         if bootstrap_method==:parameter or bootstrap_method==:random
-          rng = GSL::Rng.alloc(GSL::Rng::MT19937, rand(32000))
+          rng = Distribution::Normal.rng_ugaussian
         end
         
         @iterations.times do |i|
@@ -137,7 +137,7 @@ module Statsample
             ds_bootstrap=Statsample::Dataset.new(@ds.fields)
             @fields.each do |f|
               if bootstrap_method==:random
-                ds_bootstrap[f]=@n_cases.times.map {|c| rng.ugaussian()}.to_scale
+                ds_bootstrap[f]=@n_cases.times.map {|c| rng.call}.to_scale
               elsif bootstrap_method==:data
                 ds_bootstrap[f]=ds[f].sample_with_replacement(@n_cases).to_scale
               else
@@ -153,7 +153,7 @@ module Statsample
             end
             ev=matrix.eigenvalues
             @ds_eigenvalues.add_case_array(ev)
-          rescue Tetrachoric::RequerimentNotMeet => e
+          rescue Statsample::Bivariate::Tetrachoric::RequerimentNotMeet => e
             puts "Error: #{e}" if $DEBUG
             redo
           end

data/lib/statsample/graph.rb

@@ -1,7 +1,4 @@
-require 'statsample/graph/svggraph'
-require 'statsample/graph/svghistogram'
-require 'statsample/graph/svgscatterplot'
-
+require 'statsample/graph/scatterplot'
 module Statsample
   module Graph
   end

data/lib/statsample/graph/scatterplot.rb

@@ -0,0 +1,169 @@
+require 'tmpdir'
+require 'rubyvis'
+module Statsample
+  module Graph
+    # = Scatterplot
+    # 
+    # From Wikipedia:
+    # A scatter plot or scattergraph is a type of mathematical diagram using
+    # Cartesian coordinates to display values for two variables for a set of data.
+    # 
+    # The data is displayed as a collection of points, each having the value of one variable determining the position on the horizontal axis and the value of the other variable determining the position on the vertical axis.[2] This kind of plot is also called a scatter chart, scatter diagram and scatter graph.
+    # == Usage
+    # === Svg output
+    #  a=[1,2,3,4].to_scale
+    #  b=[3,4,5,6].to_scale
+    #  puts Statsample::Graph::Scatterplot.new(a,b).to_svg
+    # === Using ReportBuilder
+    #  a=[1,2,3,4].to_scale
+    #  b=[3,4,5,6].to_scale
+    #  rb=ReportBuilder.new
+    #  rb.add(Statsample::Graph::Scatterplot.new(a,b))
+    #  rb.save_html('scatter.html')
+    
+    class Scatterplot
+      include Summarizable
+      attr_accessor :name
+      # Total width of Scatterplot
+      attr_accessor :width
+      # Total height of Scatterplot
+      attr_accessor :height
+      attr_accessor :dot_alpha
+      # Add a line on median of x and y axis 
+      attr_accessor :line_median
+      # Top margin
+      attr_accessor :margin_top
+      # Bottom margin
+      attr_accessor :margin_bottom
+      # Left margin
+      attr_accessor :margin_left
+      # Right margin
+      attr_accessor :margin_right
+      
+      attr_reader   :data
+      attr_reader :v1,:v2
+      attr_reader :x_scale, :y_scale
+      # Create a new Scatterplot.
+      # Params:
+      # * v1: Vector on X axis
+      # * v2: Vector on Y axis
+      # * opts: Hash of options. See attributes of Scatterplot
+      def initialize(v1,v2,opts=Hash.new)
+        @v1_name,@v2_name = v1.name,v2.name
+        @v1,@v2           = Statsample.only_valid_clone(v1,v2)
+        opts_default={
+          :name=>_("Scatterplot (%s - %s)") % [@v1_name, @v2_name],
+          :width=>400,
+          :height=>300,
+          :dot_alpha=>0.5,
+          :line_median=>false,
+          :margin_top=>10,
+          :margin_bottom=>20,
+          :margin_left=>20,
+          :margin_right=>20
+          
+        }
+        @opts=opts_default.merge(opts)
+        opts_default.keys.each {|k| send("#{k}=", @opts[k]) }
+        @data=[]
+        @v1.each_with_index {|d1,i|
+          @data.push({:x=>d1,:y=>@v2[i]})
+        }
+      end
+      # Add a rule on median of X and Y axis
+      def add_line_median(vis) # :nodoc:
+        that=self
+        x=@x_scale
+        y=@y_scale
+        vis.execute {
+          rule do
+            data [that.v1.median]
+            left x
+            stroke_style Rubyvis.color("#933").alpha(0.5)
+            label(:anchor=>"top") do
+              text x.tick_format
+            end
+          end
+          rule do
+            data [that.v2.median]
+            bottom y
+            stroke_style Rubyvis.color("#933").alpha(0.5)
+            label(:anchor=>"right") do
+              text y.tick_format
+            end
+          end  
+        }
+        
+      end
+      # Returns a Rubyvis panel with scatterplot
+      def rubyvis_panel # :nodoc:
+        that=self
+        #p @v1.map {|v| v}
+        x=Rubyvis::Scale.linear(@v1.to_a).range(0,width)
+        y=Rubyvis::Scale.linear(@v2.to_a).range(0,height)
+        @x_scale=x
+        @y_scale=y
+        vis=Rubyvis::Panel.new do |pan| 
+          pan.width  width  - (margin_left + margin_right)
+          pan.height height - (margin_top  + margin_bottom)
+          pan.bottom margin_bottom
+          pan.left   margin_left
+          pan.right  margin_right
+          pan.top    margin_top
+          # X axis
+          pan.rule do
+            data y.ticks
+            bottom y
+            stroke_style {|d| d!=0 ? "#eee" : "#000"}
+            label(:anchor=>'left') do
+              visible {|d| d>0 and d<that.width}
+              text y.tick_format
+            end
+          end
+          
+          # Y axis
+          pan.rule do
+            data x.ticks
+            left x
+            stroke_style {|d| d!=0 ? "#eee" : "#000"}
+            label(:anchor=>'bottom') do
+              visible {|d| d>0 and d < that.height}
+              text x.tick_format
+            end
+          end
+          # Add lines on median
+          add_line_median(pan) if line_median
+
+          pan.panel do
+            data(that.data)
+            dot do
+              left {|d| x.scale(d[:x])}
+              bottom {|d| y.scale(d[:y])}
+              stroke_style Rubyvis.color("red").alpha(that.dot_alpha)
+              shape_radius 2
+            end
+          end
+        end
+      end
+      # Returns SVG with scatterplot
+      def to_svg
+        rp=rubyvis_panel
+        rp.render
+        rp.to_svg
+      end
+      def report_building(builder) # :nodoc:
+        img_svg=to_svg
+        builder.section(:name=>name) do |b|
+          Dir.mktmpdir {|dir|
+            time=Time.new.to_f
+            File.open("#{dir}/image_#{time}.svg","w") {|fp|
+              fp.write img_svg
+            }
+            b.image("#{dir}/image_#{time}.svg", :width=>width, :height=>height)
+          }
+        end
+        
+      end
+    end
+  end
+end