statsample 0.6.7 → 0.7.0

data/History.txt

@@ -1,5 +1,12 @@
+=== 0.7.0 / 2010-03-25
+* Ported to ReportBuilder 1.x series
+* Implementation of ruby based covariance and correlation changed to a clearer code
+* Statsample::Vector#svggraph_frequencies accepts IO
+* Some test ported to Miniunit
+* CSV on Ruby1.8 uses FasterCSV
+
 === 0.6.7 / 2010-03-23
-* Bug fix: dependency on Reportbuilder should be set to "~>0.2.0", not "0.2"
+* Bug fix: dependency on ReportBuilder should be set to "~>0.2.0", not "0.2"
 === 0.6.6 / 2010-03-22
 * Set ReportBuilder dependency to '0.2.~' version, because future API break
 * Removed Alglib dependency

data/Manifest.txt

@@ -34,8 +34,7 @@ lib/statsample/bivariate/polychoric.rb
 lib/statsample/bivariate/tetrachoric.rb
 lib/statsample/codification.rb
 lib/statsample/combination.rb
-lib/statsample/converter/csv18.rb
-lib/statsample/converter/csv19.rb
+lib/statsample/converter/csv.rb
 lib/statsample/converter/spss.rb
 lib/statsample/converters.rb
 lib/statsample/crosstab.rb

data/Rakefile

@@ -42,7 +42,7 @@ h=Hoe.spec('statsample') do
 	self.version=Statsample::VERSION
 	self.rubyforge_name = "ruby-statsample"
 	self.developer('Claudio Bustos', 'clbustos@gmail.com')
-	self.extra_deps << ["spreadsheet","~>0.6.0"] << ["svg-graph", "~>1.0"] << ["reportbuilder", "~>0.2.0"] << ["minimization", "~>0.1.0"]
+	self.extra_deps << ["spreadsheet","~>0.6.0"] << ["svg-graph", "~>1.0"] << ["reportbuilder", "~>1.0"] << ["minimization", "~>0.1.0"] << ["fastercsv"]
 	self.clean_globs << "test/images/*" << "demo/item_analysis/*" << "demo/Regression"
 	self.need_rdoc=false
 end

data/demo/dominanceanalysis.rb

@@ -10,7 +10,7 @@ d=sample.times.collect {rand}.to_scale
 
 ds={'a'=>a,'b'=>b,'c'=>c,'d'=>d}.to_dataset
 ds['y']=ds.collect{|row| row['a']*5+row['b']*3+row['c']*2+row['d']+rand()}
-rb=ReportBuilder.new("Dominance Analysis")
+rb=ReportBuilder.new(:name=>"Dominance Analysis")
 
 cm=Statsample::Bivariate.correlation_matrix(ds)
 rb.add(cm)
@@ -25,3 +25,4 @@ rb.add(da)
 
 
 puts rb.to_text
+rb.save_rtf("dominance_analysis.rtf")

data/lib/statsample.rb

@@ -23,7 +23,7 @@ $:.unshift(File.expand_path(File.dirname(__FILE__)+"/../ext"))
 
 require 'matrix'
 require 'distribution'
-raise "Install reportbuilder ~>0.2.0" unless gem 'reportbuilder','~>0.2.0'
+raise "Install reportbuilder ~>1.0" unless gem 'reportbuilder','~>1.0'
 require 'reportbuilder'
 class Numeric
   def square ; self * self ; end
@@ -113,7 +113,7 @@ module Statsample
       false
     end
   end
-  VERSION = '0.6.7'
+  VERSION = '0.7.0'
   SPLIT_TOKEN = ","
   autoload(:Database, 'statsample/converters')
   autoload(:Anova, 'statsample/anova')

data/lib/statsample/bivariate.rb

@@ -24,14 +24,16 @@ module Statsample
         sum
       end
       
-      def covariance_slow(v1a,v2a) # :nodoc:
-        t=0
+      def covariance_slow(v1,v2) # :nodoc:
+        v1a,v2a=Statsample.only_valid(v1,v2)
+        sum_of_squares(v1a,v2a) / (v1a.size-1)
+      end
+      def sum_of_squares(v1,v2)
+        v1a,v2a=Statsample.only_valid(v1,v2)        
         m1=v1a.mean
-        m2=v1a.mean
-        (0...v1a.size).each {|i| t+=((v1a[i]-m1)*(v2a[i]-m2)) }
-        t.to_f / (v1a.size-1)
+        m2=v2a.mean
+        (v1a.size).times.inject(0) {|ac,i| ac+(v1a[i]-m1)*(v2a[i]-m2)}
       end
-      
       # Calculate Pearson correlation coefficient (r) between 2 vectors
       def pearson(v1,v2)
         v1a,v2a=Statsample.only_valid(v1,v2)
@@ -42,12 +44,18 @@ module Statsample
           pearson_slow(v1a,v2a)
         end
       end
-      def pearson_slow(v1a,v2a) # :nodoc:
-        v1s,v2s=v1a.vector_standarized_pop,v2a.vector_standarized_pop
+      def pearson_slow(v1,v2) # :nodoc:
+        v1a,v2a=Statsample.only_valid(v1,v2)
+        # Calculate sum of squares
+        ss=sum_of_squares(v1a,v2a)
+        ss.quo(Math::sqrt(v1a.sum_of_squares) * Math::sqrt(v2a.sum_of_squares))
+=begin        
+        v1s,v2s=v1a.vector_standarized,v2a.vector_standarized
         t=0
         siz=v1s.size
         (0...v1s.size).each {|i| t+=(v1s[i]*v2s[i]) }
-        t.to_f/v2s.size
+        t.quo(v2s.size-1)
+=end
       end
       # Retrieves the value for t test for a pearson correlation
       # between two vectors to test the null hipothesis of r=0
@@ -279,6 +287,7 @@ module Statsample
         }
         a
       end
+=begin      
       def sum_of_codeviated(v1,v2)
         v1a,v2a=Statsample.only_valid(v1,v2)
         sum=0
@@ -287,7 +296,7 @@ module Statsample
         }
         sum-((v1a.sum*v2a.sum) / v1a.size.to_f)
       end
-      
+=end
       # Report the minimum number of cases valid of a covariate matrix
       # based on a dataset
       def min_n_valid(ds)

data/lib/statsample/bivariate/polychoric.rb

@@ -744,24 +744,24 @@ module Statsample
         rp.to_text
       end
       
-      def to_reportbuilder(generator) # :nodoc: 
+      def report_building(generator) # :nodoc: 
         compute if @r.nil?
         section=ReportBuilder::Section.new(:name=>@name)
         t=ReportBuilder::Table.new(:name=>_("Contingence Table"),:header=>[""]+(@n.times.collect {|i| "Y=#{i}"})+["Total"])
         @m.times do |i|
-          t.add_row(["X = #{i}"]+(@n.times.collect {|j| @matrix[i,j]}) + [@sumr[i]])
+          t.row(["X = #{i}"]+(@n.times.collect {|j| @matrix[i,j]}) + [@sumr[i]])
         end
-        t.add_hr
-        t.add_row(["T"]+(@n.times.collect {|j| @sumc[j]})+[@total])
+        t.hr
+        t.row(["T"]+(@n.times.collect {|j| @sumc[j]})+[@total])
         section.add(t)
         #generator.parse_element(t)
         section.add(sprintf("r: %0.4f",r))
         t=ReportBuilder::Table.new(:name=>_("Thresholds"), :header=>["","Value"])
         threshold_x.each_with_index {|val,i|
-          t.add_row(["Threshold X #{i}", sprintf("%0.4f", val)])
+          t.row(["Threshold X #{i}", sprintf("%0.4f", val)])
         }
         threshold_y.each_with_index {|val,i|
-          t.add_row(["Threshold Y #{i}", sprintf("%0.4f", val)])
+          t.row(["Threshold Y #{i}", sprintf("%0.4f", val)])
         }
         section.add(t)
         section.add(_("Test of bivariate normality: X2 = %0.3f, df = %d, p= %0.5f" % [ chi_square, chi_square_df, 1-Distribution::ChiSquare.cdf(chi_square, chi_square_df)])) 

data/lib/statsample/bivariate/tetrachoric.rb

@@ -114,18 +114,18 @@ module Statsample
       end
       # Summary of the analysis
       def summary
-        rp=ReportBuilder.new()
+        rp=ReportBuilder.new(:name=>@name)
         rp.add(self)
         rp.to_text
       end
 
-      def to_reportbuilder(generator) # :nodoc:
+      def report_building(generator) # :nodoc:
         section=ReportBuilder::Section.new(:name=>@name)
         t=ReportBuilder::Table.new(:name=>_("Contingence Table"),:header=>["","Y=0","Y=1", "T"])
-        t.add_row(["X=0", @a,@b,@a+@b])
-        t.add_row(["X=1", @c,@d,@c+@d])
-        t.add_hr
-        t.add_row(["T", @a+@c,@b+@d,@a+@b+@c+@d])
+        t.row(["X=0", @a,@b,@a+@b])
+        t.row(["X=1", @c,@d,@c+@d])
+        t.hr
+        t.row(["T", @a+@c,@b+@d,@a+@b+@c+@d])
         section.add(t)
         #generator.parse_element(t)
         section.add(sprintf("r: %0.3f",r))

data/lib/statsample/converter/{csv19.rb → csv.rb}

@@ -1,3 +1,12 @@
+if RUBY_VERSION<"1.9"
+  require 'fastercsv'
+  Statsample::CSV_klass=FasterCSV  
+else
+  require 'csv'
+  Statsample::CSV_klass=CSV  
+  
+end
+
 module Statsample
   class CSV < SpreadsheetBase
     class << self
@@ -6,7 +15,7 @@ module Statsample
       # USE:
       #     ds=Statsample::CSV.read("test_csv.csv")
       def read(filename, empty=[''],ignore_lines=0,fs=nil,rs=nil)
-        require 'csv'
+        
         first_row=true
         fields=[]
         fields_data={}
@@ -15,7 +24,7 @@ module Statsample
         opts={}
         opts[:col_sep]=fs unless fs.nil?
         opts[:row_sep]=rs unless rs.nil?
-        csv=::CSV.open(filename,'r',opts)
+        csv=Statsample::CSV_klass.send(:open, filename,'r',opts)
         csv.each do |row|
           line_number+=1
           if(line_number<=ignore_lines)
@@ -41,8 +50,8 @@ module Statsample
       # USE:
       #     Statsample::CSV.write(ds,"test_csv.csv")
       def write(dataset,filename, convert_comma=false,*opts)
-        require 'csv'
-        writer=::CSV.open(filename,'w',*opts)
+        
+        writer=Statsample::CSV_klass.send(:open, filename,'w',*opts)
         writer << dataset.fields
         dataset.each_array do|row|
           if(convert_comma)

data/lib/statsample/converters.rb

@@ -351,9 +351,5 @@ out
 	end
 end
 
-if RUBY_VERSION<"1.9"
-    require 'statsample/converter/csv18.rb'
-else
-    require 'statsample/converter/csv19.rb'
-end
+require 'statsample/converter/csv.rb'
 

data/lib/statsample/crosstab.rb

@@ -93,17 +93,17 @@ module Statsample
     def cols_empty_hash
       cols_names.inject({}) {|a,x| a[x]=0;a}
     end
-    def to_reportbuilder(generator)
-      anchor=generator.add_toc_entry(_("Crosstab: ")+name)
-      generator.add_html "<div class='crosstab'>"+_("Crosstab")+" #{@name}<a name='#{anchor}'></a>"
+    def report_building(generator)
+      anchor=generator.toc_entry(_("Crosstab: ")+name)
+      generator.html "<div class='crosstab'>"+_("Crosstab")+" #{@name}<a name='#{anchor}'></a>"
       fq=frequencies
       rn=rows_names
       cn=cols_names
       total=0
       total_cols=cols_empty_hash
-      generator.add_text "Chi Square: #{chi_square}"
-      generator.add_text(_("Rows: %s") % @row_label) unless @row_label.nil?
-      generator.add_text(_("Columns: %s") % @column_label) unless @column_label.nil?
+      generator.text "Chi Square: #{chi_square}"
+      generator.text(_("Rows: %s") % @row_label) unless @row_label.nil?
+      generator.text(_("Columns: %s") % @column_label) unless @column_label.nil?
       
       t=ReportBuilder::Table.new(:name=>@name+" - "+_("Raw"), :header=>[""]+cols_names.collect {|c| @v_cols.labeling(c)}+[_("Total")])
       rn.each do |row|
@@ -117,15 +117,15 @@ module Statsample
           t_row.push(data)
         end
         t_row.push(total_row)
-        t.add_row(t_row)
+        t.row(t_row)
       end
-      t.add_horizontal_line
+      t.hr
       t_row=[_("Total")]
       cn.each do |v|
         t_row.push(total_cols[v])
       end
       t_row.push(total)
-      t.add_row(t_row)
+      t.row(t_row)
       generator.parse_element(t)
       
       if(@percentage_row)
@@ -138,7 +138,7 @@ module Statsample
       table_percentage(generator,:total)
       end
       
-      generator.add_html("</div>")
+      generator.html("</div>")
     end
       
     
@@ -174,10 +174,10 @@ module Statsample
             when :total   then  @cases
           end              
           t_row.push(sprintf("%0.2f%%", rt[row]*100.0/total))
-          t.add_row(t_row)
+          t.row(t_row)
         end
         
-        t.add_horizontal_line
+        t.hr
         t_row=[_("Total")]
         cn.each{|col|
           total=case type
@@ -188,7 +188,7 @@ module Statsample
           t_row.push(sprintf("%0.2f%%", ct[col]*100.0/total))
         }
       t_row.push("100%")
-      t.add_row(t_row)
+      t.row(t_row)
       generator.parse_element(t)
     end
   end

data/lib/statsample/dataset.rb

@@ -154,7 +154,7 @@ module Statsample
     # Creates a copy of the given dataset, deleting all the cases with
     # missing data on one of the vectors
     def dup_only_valid
-      if @vectors.find{|field,vector| vector.has_missing_data?}
+      if @vectors.any?{|field,vector| vector.has_missing_data?}
         ds=dup_empty
         each_array { |c|
           ds.add_case_array(c) unless @fields.find{|f| @vectors[f].data_with_nils[@i].nil? }

data/lib/statsample/dominanceanalysis.rb

@@ -313,44 +313,46 @@ module Statsample
       rp.add(self)
       rp.to_text
     end
-    def to_reportbuilder(generator)
+    def report_building(generator)
       compute if @models.nil?
-      anchor=generator.add_toc_entry(_("DA: ")+@name)
-      generator.add_html "<div class='dominance-analysis'>#{@name}<a name='#{anchor}'></a>"
-      t=ReportBuilder::Table.new(:name=>_("Dominance Analysis result"))
       
-      t.header=["","r2",_("sign")]+@predictors.collect {|c| DominanceAnalysis.predictor_name(c) }
+      anchor=generator.toc_entry(_("DA: ")+@name)
+      
+      generator.html "<div class='dominance-analysis'>#{@name}<a name='#{anchor}'></a>"
+      header=["","r2",_("sign")]+@predictors.collect {|c| DominanceAnalysis.predictor_name(c) }
+      t=ReportBuilder::Table.new(:name=>_("Dominance Analysis result"),:header=>header)
+      
       row=[_("Model 0"),"",""]+@predictors.collect{|f|
         sprintf("%0.3f", md([f]).r2)
       }
-      t.add_row(row)
-      t.add_horizontal_line
+      
+      t.row(row)
+      t.hr
       for i in 1..@predictors.size
         mk=md_k(i)
         mk.each{|m|
-          t.add_row(m.add_table_row)
+          t.row(m.add_table_row)
         }
         # Report averages
         a=average_k(i)
         if !a.nil?
-            t.add_horizontal_line
+            t.hr
             row=[_("k=%d Average") % i,"",""] + @predictors.collect{|f|
                 sprintf("%0.3f",a[f])
             }
-            t.add_row(row)
-            t.add_horizontal_line
+            t.row(row)
+            t.hr
             
         end
-        
       end
       
       g=general_averages
-      t.add_horizontal_line
+      t.hr
       
       row=[_("Overall averages"),"",""]+@predictors.collect{|f|
                 sprintf("%0.3f",g[f])
       }
-      t.add_row(row)
+      t.row(row)
       generator.parse_element(t)
       
       td=total_dominance
@@ -360,10 +362,10 @@ module Statsample
       pairs.each{|p|
         name=p.join(" - ")
         row=[name, sprintf("%0.1f",td[p]), sprintf("%0.1f",cd[p]), sprintf("%0.1f",gd[p])]
-        t.add_row(row)
+        t.row(row)
       }
       generator.parse_element(t)
-      generator.add_html("</div>")
+      generator.html("</div>")
     end
     class ModelData # :nodoc:
       attr_reader :contributions

data/lib/statsample/dominanceanalysis/bootstrap.rb

@@ -178,39 +178,39 @@ module Statsample
       def t
         Distribution::T.p_value(1-((1-@alpha) / 2), @n_samples - 1)
       end
-      def to_reportbuilder(generator) # :nodoc:
+      def report_building(generator) # :nodoc:
         raise "You should bootstrap first" if @n_samples==0
-        anchor=generator.add_toc_entry(_("DAB: ")+@name)
-        generator.add_html "<div class='dominance-analysis-bootstrap'>#{@name}<a name='#{anchor}'></a>"
+        anchor=generator.toc_entry(_("DAB: ")+@name)
+        generator.html "<div class='dominance-analysis-bootstrap'>#{@name}<a name='#{anchor}'></a>"
         
-        generator.add_text _("Sample size: %d\n") % @n_samples
-        generator.add_text "t: #{t}\n"
-        generator.add_text _("Linear Regression Engine: %s") % @regression_class.name
+        generator.text _("Sample size: %d\n") % @n_samples
+        generator.text "t: #{t}\n"
+        generator.text _("Linear Regression Engine: %s") % @regression_class.name
         
         table=ReportBuilder::Table.new(:name=>"Bootstrap report", :header => [_("pairs"), "sD","Dij", _("SE(Dij)"), "Pij", "Pji", "Pno", _("Reproducibility")])
-        table.add_row([_("Complete dominance")])
-        table.add_horizontal_line
+        table.row([_("Complete dominance")])
+        table.hr
         @pairs.each{|pair|
           std=@samples_td[pair].to_vector(:scale)
           ttd=da.total_dominance_pairwise(pair[0],pair[1])
-          table.add_row(summary_pairs(pair,std,ttd))
+          table.row(summary_pairs(pair,std,ttd))
         }
-        table.add_horizontal_line
-        table.add_row([_("Conditional dominance")])
-        table.add_horizontal_line
+        table.hr
+        table.row([_("Conditional dominance")])
+        table.hr
         @pairs.each{|pair|
           std=@samples_cd[pair].to_vector(:scale)
           ttd=da.conditional_dominance_pairwise(pair[0],pair[1])
-          table.add_row(summary_pairs(pair,std,ttd))
+          table.row(summary_pairs(pair,std,ttd))
         
         }
-        table.add_horizontal_line
-        table.add_row([_("General Dominance")])
-        table.add_horizontal_line
+        table.hr
+        table.row([_("General Dominance")])
+        table.hr
         @pairs.each{|pair|
           std=@samples_gd[pair].to_vector(:scale)
           ttd=da.general_dominance_pairwise(pair[0],pair[1])
-          table.add_row(summary_pairs(pair,std,ttd))
+          table.row(summary_pairs(pair,std,ttd))
         }
         generator.parse_element(table)
         
@@ -219,12 +219,12 @@ module Statsample
         @fields.each{|f|
           v=@samples_ga[f].to_vector(:scale)
           row=[@ds.label(f), sprintf("%0.3f",v.mean), sprintf("%0.3f",v.sd), sprintf("%0.3f",v.percentil(5)),sprintf("%0.3f",v.percentil(95))]
-          table.add_row(row)
+          table.row(row)
         
         }
         
         generator.parse_element(table)
-        generator.add_html("</div>")
+        generator.html("</div>")
       end
       def summary_pairs(pair,std,ttd)
           freqs=std.proportions