statsample 0.4.1 → 0.5.0

data/lib/statsample/converter/csv19.rb

@@ -0,0 +1,60 @@
+module Statsample
+class CSV < SpreadsheetBase
+		class << self
+        # Returns a Dataset  based on a csv file
+        #
+        # USE:
+        #     ds=Statsample::CSV.read("test_csv.csv")
+        def read(filename, empty=[''],ignore_lines=0,fs=nil,rs=nil)
+        require 'csv'
+                first_row=true
+                fields=[]
+                fields_data={}
+                ds=nil
+                line_number=0
+                opts={}
+                opts[:col_sep]=fs unless fs.nil?
+                opts[:row_sep]=rs unless rs.nil?
+                csv=::CSV.open(filename,'r',opts)
+                
+                csv.each do |row|
+                    line_number+=1
+                    if(line_number<=ignore_lines)
+                        #puts "Skip line"
+                        next
+                    end
+                    row.collect!{|c|
+                        c.to_s
+                    }
+                    if first_row
+                        fields=extract_fields(row)
+                        ds=Statsample::Dataset.new(fields)
+                        first_row=false
+                    else
+                        rowa=process_row(row,empty)
+                        ds.add_case(rowa,false)
+                    end
+                end
+                convert_to_scale(ds,fields)
+                ds.update_valid_data
+                ds
+            end
+        # Save a Dataset on a csv file
+        #
+        # USE:
+        #     Statsample::CSV.write(ds,"test_csv.csv")            
+        def write(dataset,filename, convert_comma=false,*opts)
+            require 'csv'            
+                writer=::CSV.open(filename,'w',*opts)
+                writer << dataset.fields
+                dataset.each_array{|row|
+                    if(convert_comma)
+                        row.collect!{|v| v.to_s.gsub(".",",")}
+                    end
+                    writer << row
+                }
+                writer.close
+            end
+		end
+    end
+end

data/lib/statsample/converters.rb

@@ -1,7 +1,6 @@
 module Statsample
     # Create and dumps Datasets on a database
 	module Database
-        require 'dbi'
 		class << self
         # Read a database query and returns a Dataset
         #
@@ -11,6 +10,7 @@ module Statsample
         #  Statsample.read(dbh, "SELECT * FROM test")
         #
         def read(dbh,query)
+            require 'dbi'
             sth=dbh.execute(query)
             vectors={}
             fields=[]
@@ -35,6 +35,7 @@ module Statsample
         #  Statsample::Database.insert(ds,dbh,"test")
         #
         def insert(ds, dbh,table)
+            require 'dbi'            
             query="INSERT INTO #{table} ("+ds.fields.join(",")+") VALUES ("+((["?"]*ds.fields.size).join(","))+")"
             sth=dbh.prepare(query)
             ds.each_array{|c|
@@ -64,15 +65,11 @@ module Statsample
             def write(dataset,filename)
                 File.open(filename,"wb") do |fp|
                     fp.puts dataset.fields.join("\t")
-                    dataset.each {|row|
-                        values=dataset.fields.collect{|f|
-                            if dataset[f].is_valid? row[f]
-                                row[f]
-                            else
-                                ""
-                            end
+                    dataset.each_array_with_nils{|row|
+                        row2=row.collect{|v|
+                            v.nil? ? "NA" : v.to_s.gsub(/\s+/,"_")
                         }
-                        fp.puts(values.join("\t"))
+                        fp.puts row2.join("\t")
                     }
                 end
         end
@@ -194,7 +191,7 @@ module Statsample
                         first_row=false
                     else
                         rowa=process_row(row,empty)
-                        (fields.size - rowa.size).times {|i|
+                        (fields.size - rowa.size).times {
                             rowa << nil
                         }
                         ds.add_case(rowa,false)
@@ -210,59 +207,6 @@ module Statsample
             end
         end
     end
-    class CSV < SpreadsheetBase
-		class << self
-        # Returns a Dataset  based on a csv file
-        #
-        # USE:
-        #     ds=Statsample::CSV.read("test_csv.csv")
-        def read(filename, empty=[''],ignore_lines=0,fs=nil,rs=nil)
-        require 'csv'
-                first_row=true
-                fields=[]
-                fields_data={}
-                ds=nil
-                line_number=0
-                ::CSV.open(filename,'r',fs,rs) do |row|
-                    line_number+=1
-                    if(line_number<=ignore_lines)
-                        #puts "Skip line"
-                        next
-                    end
-                    row.collect!{|c|
-                        c.to_s
-                    }
-                    if first_row
-                        fields=extract_fields(row)
-                        ds=Statsample::Dataset.new(fields)
-                        first_row=false
-                    else
-                        rowa=process_row(row,empty)
-                        ds.add_case(rowa,false)
-                    end
-                end
-                convert_to_scale(ds,fields)
-                ds.update_valid_data
-                ds
-            end
-        # Save a Dataset on a csv file
-        #
-        # USE:
-        #     Statsample::CSV.write(ds,"test_csv.csv")            
-        def write(dataset,filename, convert_comma=false,*opts)
- require 'csv'            
-                writer=::CSV.open(filename,'w',*opts)
-                writer << dataset.fields
-                dataset.each_array{|row|
-                    if(convert_comma)
-                        row.collect!{|v| v.to_s.gsub(".",",")}
-                    end
-                    writer << row
-                }
-                writer.close
-            end
-		end
-    end
     module Mx
         class << self
             def write(dataset,filename,type=:covariance)
@@ -309,13 +253,13 @@ module Statsample
             end
 			def out(dataset,opt={})
 				require 'ostruct'
-				default_opt = {:dataname => "Default", :description=>""}
+				default_opt = {:dataname => "Default", :description=>"", :missing=>"NA"}
 				default_opt.merge! opt
 				carrier=OpenStruct.new
 				carrier.categorials=[]
 				carrier.conversions={}
-				variables_def=dataset.vectors.collect{|k,v|
-					variable_definition(carrier,v,k)
+				variables_def=dataset.fields.collect{|k|
+					variable_definition(carrier,dataset[k],k)
 				}.join("\n")
 				
 				indexes=carrier.categorials.inject({}) {|s,c|
@@ -327,7 +271,7 @@ module Statsample
 					indexes.each{|ik,iv|
 						c[ik]=carrier.conversions[iv][c[ik]]
 					}
-					records << "<record>#{values_definition(c)}</record>\n"
+					records << "<record>#{values_definition(c, default_opt[:missing])}</record>\n"
 				}
 				
 out=<<EOC
@@ -339,7 +283,7 @@ out=<<EOC
 <variables count="#{dataset.fields.size}">
 #{variables_def}
 </variables>
-<records count="#{dataset.cases}">
+    <records count="#{dataset.cases}" missingValue="#{default_opt[:missing]}">
 #{records}
 </records>
 
@@ -350,14 +294,14 @@ EOC
 out
 
 			end
-			def values_definition(c)
+			def values_definition(c,missing)
 				c.collect{|v|
-					if v.is_a? Float
-						"<real>#{v}</real>"
-					elsif v.is_a? Integer
-						"<int>#{v}</int>"
+                    if v.nil?
+                        "#{missing}"
+                    elsif v.is_a? Numeric
+						"#{v}"
 					else
-						"<string>#{v}</string>"
+						"#{v.gsub(/\s+/,"_")}"
 					end
 				}.join(" ")
 			end
@@ -370,7 +314,7 @@ out
 				if v.type==:nominal or v.data.find {|d|  d.is_a? String }
 					carrier.categorials.push(name)
 					carrier.conversions[name]={}
-					factors=v.data.uniq.sort
+					factors=v.factors
 					out ="<categoricalvariable name=\"#{name}\" #{nickname}>\n"
 					out << "<levels count=\"#{factors.size}\">\n"
 					out << (1..factors.size).to_a.collect{|i|
@@ -389,3 +333,10 @@ out
 		end
 	end
 end
+
+if RUBY_VERSION<"1.9"
+    require 'statsample/converter/csv18.rb'
+else
+    require 'statsample/converter/csv19.rb'
+end
+

data/lib/statsample/dataset.rb

@@ -292,7 +292,7 @@ module Statsample
         end
         def check_length
             size=nil
-            @vectors.each{|k,v|
+            @vectors.each do |k,v|
                 raise Exception, "Data #{v.class} is not a vector on key #{k}" if !v.is_a? Statsample::Vector
                 if size.nil?
                     size=v.size
@@ -302,38 +302,33 @@ module Statsample
                         raise Exception, "Vector #{k} have size #{v.size} and dataset have size #{size}"
                     end
                 end
-            }
-            @cases=size
             end
-            def each_vector
-                @fields.each{|k|
-                    yield k,@vectors[k]
-                }
+            @cases=size
+        end
+        def each_vector
+            @fields.each{|k| yield k,@vectors[k]}
+        end
+        if Statsample::STATSAMPLE__.respond_to?(:case_as_hash)
+            def case_as_hash(c) # :nodoc:
+                Statsample::STATSAMPLE__.case_as_hash(self,c)
             end
-            if Statsample::STATSAMPLE__.respond_to?(:case_as_hash)
-                def case_as_hash(c) # :nodoc:
-                    Statsample::STATSAMPLE__.case_as_hash(self,c)
-                end
-            else
-                def case_as_hash(c)
-                    _case_as_hash(c)
-                end                
+        else
+            def case_as_hash(c)
+                _case_as_hash(c)
+            end                
+        end
+        
+        if Statsample::STATSAMPLE__.respond_to?(:case_as_array)
+            def case_as_array(c) # :nodoc:
+                Statsample::STATSAMPLE__.case_as_array(self,c)
             end
-            
-            if Statsample::STATSAMPLE__.respond_to?(:case_as_array)
-                def case_as_array(c) # :nodoc:
-                    Statsample::STATSAMPLE__.case_as_array(self,c)
-                end
-            else
-                def case_as_array(c)
-                    _case_as_array(c)
-                end
+        else
+            def case_as_array(c)
+                _case_as_array(c)
             end
-            def _case_as_hash(c) # :nodoc:
-            @fields.inject({}) {|a,x|
-                a[x]=@vectors[x][c]
-                a
-            }
+        end
+        def _case_as_hash(c) # :nodoc:
+        @fields.inject({}) {|a,x| a[x]=@vectors[x][c];a }
         end
         def _case_as_array(c) # :nodoc:
             @fields.collect {|x| @vectors[x][c]}
@@ -366,6 +361,20 @@ module Statsample
                 raise DatasetException.new(self,e)
             end
         end
+        # Returns each case as an array, coding missing values as nils
+        def each_array_with_nils
+            m=fields.size
+            @cases.times {|i|
+                @i=i
+                row=Array.new(m)
+                fields.each_index{|j|
+                    f=fields[j]
+                    row[j]=@vectors[f].data_with_nils[i]
+                }
+                yield row
+            }
+            @i=nil
+        end
         # Returns each case as an array
         def each_array
             @cases.times {|i|

data/lib/statsample/dominanceanalysis.rb

@@ -124,8 +124,8 @@ module Statsample
                 }
             }
             out={}
-            averages.each{|k,v|
-                out[k]=v.to_vector(:scale).mean
+            averages.each{|key,val|
+                out[key]=val.to_vector(:scale).mean
             }
             out
         end
@@ -139,8 +139,8 @@ module Statsample
                     }
                 end
                 out={}
-                averages.each{|k,v|
-                    out[k]=v.to_vector(:scale).mean
+                averages.each{|key,val|
+                    out[key]=val.to_vector(:scale).mean
                 }
                 @general_averages=out
             end
@@ -152,8 +152,8 @@ module Statsample
             for i in 1..@fields.size
                 c=Statsample::Combination.new(i,@fields.size)
                 c.each{|data|
-                    convert=data.collect {|i|
-                        @fields[i]
+                    convert=data.collect {|i1|
+                        @fields[i1]
                     }
                     @models.push(convert)
                     ds_prev=@ds.dup(convert+[@y_var])

data/lib/statsample/graph/gdchart.rb

@@ -11,7 +11,8 @@ module Statsample
 				options.each{|k,v|
 					gdc.send(k+"=",v)
 				}
-				f=File.open(file,"w") {|f|
+                
+				File.open(file,"w") {|f|
 					gdc.out_graph(width,height,f,chart_type, data.length/num_datasets,labels,num_datasets,data)
 				}
 		end

data/lib/statsample/graph/svggraph.rb

@@ -9,19 +9,20 @@ module Statsample
 	class Vector
 		# Creates a barchart using ruby-gdchart
 		def svggraph_frequencies(file, width=600, height=300, chart_type=SVG::Graph::BarNoOp, options={})
-			labels,data=[],[]
+			labels, data1=[],[]
 			self.frequencies.sort.each{|k,v|
 				labels.push(k.to_s)
-				data.push(v) 
+				data1.push(v) 
 			}
             options[:height]=height
             options[:width]=width
             options[:fields]=labels  
 			graph = chart_type.new(options)
 			graph.add_data(
-            :data => data,
+            :data => data1,
             :title => "Frequencies"
 			)
+            
 			File.open(file,"w") {|f|
               f.puts(graph.burn)
 			}
@@ -49,7 +50,7 @@ module Statsample
         def svggraph_boxplot(options={})
             check_type :scale
             options={:graph_title=>"Boxplot", :fields=>['vector'], :show_graph_title=>true}.merge! options
-            vx=@data.to_a.to_vector(:scale)
+            vx=@valid_data.to_a.to_vector(:scale)
             graph = Statsample::Graph::SvgBoxplot.new(options)
             graph.add_data(:title=>"vector", :data=>@data.to_a)
             graph
@@ -58,8 +59,8 @@ module Statsample
         def svggraph_lag_plot(options={})
             check_type :scale
             options={:graph_title=>"Lag Plot", :show_graph_title=>true}.merge! options
-            vx=@data[0...(@data.size-1)].to_vector(:scale)
-            vy=@data[1...@data.size].to_vector(:scale)
+            vx=@valid_data[0...(@valid_data.size-1)].to_vector(:scale)
+            vy=@valid_data[1...@valid_data.size].to_vector(:scale)
             ds={'x_minus_1'=>vx,'x'=>vy}.to_dataset            
             graph = Statsample::Graph::SvgScatterplot.new(ds,options)
             graph.set_x('x_minus_1')
@@ -73,11 +74,11 @@ module Statsample
             extend Statsample::Util
             check_type :scale
             options={:graph_title=>"Normal Probability Plot", :show_graph_title=>true}.merge! options
-            n=@data.size
-            vx=(1..@data.size).to_a.collect{|i|
+            n=@valid_data.size
+            vx=(1..@valid_data.size).to_a.collect{|i|
                 Distribution::Normal.p_value(normal_order_statistic_medians(i,n))
             }.to_vector(:scale)
-            vy=@data.sort.to_vector(:scale)
+            vy=@valid_data.sort.to_vector(:scale)
             ds={'normal_order_statistics_medians'=>vx, 'ordered_response'=>vy}.to_dataset            
             graph = Statsample::Graph::SvgScatterplot.new(ds,options)
             graph.set_x('normal_order_statistics_medians')

data/lib/statsample/multiset.rb

@@ -50,13 +50,13 @@ module Statsample
     class StratifiedSample
 		class << self
 			# mean for an array of vectors
-			def mean(*v)
+			def mean(*vectors)
 				n_total=0
-				a=v.inject(0){|a,v|
+				means=vectors.inject(0){|a,v|
 					n_total+=v.size
 					a+v.sum
 				}
-				a.to_f/n_total
+				means.to_f/n_total
 			end
 
             def standard_error_ksd_wr(es)