statsample 0.3.0

data/lib/statsample/converters.rb

@@ -0,0 +1,338 @@
+module Statsample
+    # Create and dumps Datasets on a database
+	module Database
+        require 'dbi'
+		class << self
+        # Read a database query and returns a Dataset
+        #
+        # USE:
+        #
+        #  dbh = DBI.connect("DBI:Mysql:database:localhost", "user", "password")
+        #  Statsample.read(dbh, "SELECT * FROM test")
+        #
+        def read(dbh,query)
+            sth=dbh.execute(query)
+            vectors={}
+            fields=[]
+            sth.column_info.each {|c|
+                vectors[c['name']]=Statsample::Vector.new([])
+                vectors[c['name']].type= (c['type_name']=='INTEGER' or c['type_name']=='DOUBLE') ? :scale : :nominal
+                fields.push(c['name'])
+            }
+            ds=Statsample::Dataset.new(vectors,fields)
+            sth.fetch do |row|
+                ds.add_case(row.to_a, false )
+            end
+            ds.update_valid_data
+            ds
+        end
+        # Insert each case of the Dataset on the selected table
+        #
+        # USE:
+        #        
+        #  ds={'id'=>[1,2,3].to_vector, 'name'=>["a","b","c"].to_vector}.to_dataset
+        #  dbh = DBI.connect("DBI:Mysql:database:localhost", "user", "password")
+        #  Statsample::Database.insert(ds,dbh,"test")
+        #
+        def insert(ds, dbh,table)
+            query="INSERT INTO #{table} ("+ds.fields.join(",")+") VALUES ("+((["?"]*ds.fields.size).join(","))+")"
+            sth=dbh.prepare(query)
+            ds.each_array{|c|
+                sth.execute(*c)
+            }
+        end
+        # Create a sql, basen on a given Dataset
+        #
+        # USE:
+        #        
+        #  ds={'id'=>[1,2,3,4,5].to_vector,'name'=>%w{Alex Peter Susan Mary John}.to_vector}.to_dataset
+        #  Statsample::Database.create_sql(ds,'names')
+        #   ==>"CREATE TABLE names (id INTEGER,\n name VARCHAR (255)) CHARACTER SET=UTF8;"
+	# 
+        def create_sql(ds,table,charset="UTF8")
+            sql="CREATE TABLE #{table} ("
+            fields=ds.fields.collect{|f|
+                v=ds[f]
+                f+" "+v.db_type
+            }
+            sql+fields.join(",\n ")+") CHARACTER SET=#{charset};"
+        end
+		end
+    end
+    module Mondrian
+        class << self
+            def write(dataset,filename)
+                File.open(filename,"wb") do |fp|
+                    fp.puts dataset.fields.join("\t")
+                    dataset.each {|row|
+                        values=dataset.fields.collect{|f|
+                            if dataset[f].is_valid? row[f]
+                                row[f]
+                            else
+                                ""
+                            end
+                        }
+                        fp.puts(values.join("\t"))
+                    }
+                end
+        end
+        end
+    end
+    module Excel
+        class << self
+            def write(dataset,filename)
+                require 'spreadsheet'
+                book = Spreadsheet::Workbook.new
+                sheet = book.create_worksheet
+                format = Spreadsheet::Format.new :color => :blue,
+                                   :weight => :bold
+                sheet.row(0).concat(dataset.fields)
+                sheet.row(0).default_format = format
+                i=1
+                dataset.each_array{|row|
+                    sheet.row(i).concat(row)
+                    i+=1
+                }
+                book.write(filename)
+            end
+            # Returns a dataset based on a xls file
+            # USE:
+            #     ds = Statsample::Excel.read("test.xls")
+            #
+            def read(filename, worksheet_id=0, ignore_lines=0, empty=[''])
+            require 'spreadsheet'
+                
+                first_row=true
+                fields=[]
+                fields_data={}
+                ds=nil
+                line_number=0
+                book = Spreadsheet.open filename
+                sheet= book.worksheet worksheet_id
+                sheet.each do |row|
+                    line_number+=1
+                    if(line_number<=ignore_lines)
+                        #puts "Skip line"
+                        next
+                    end
+                    # This should be fixed.
+                    # If we have a Formula, should be resolver first
+                    row.collect!{|c|
+                        if c.is_a? Spreadsheet::Formula
+                            nil
+                        else
+                            c.to_s
+                        end
+                    }
+                    if first_row
+                        fields=row.to_a.collect{|c| c.downcase}
+                        if fields.size!=fields.uniq.size
+                            repeated=fields.inject({}) {|a,v|
+                                (a[v].nil? ? a[v]=1 : a[v]+=1); a }.find_all{|k,v| v>1}.collect{|k,v|k}.join(",")
+                                raise "There are some repeated fields on the header:#{repeated}. Please, fix" 
+                        end
+                        ds=Statsample::Dataset.new(fields)
+                        first_row=false
+                    else
+                        rowa=row.to_a.collect{|c|
+                            
+                            empty.include?(c) ? nil: c
+                        }
+                        (fields.size - rowa.size).times {|i|
+                            rowa << nil
+                        }
+                        ds.add_case(rowa,false)
+                    end
+                end
+                ds.update_valid_data
+                ds
+            end
+        end
+    end
+    module CSV
+		class << self
+        # Returns a Dataset  based on a csv file
+        #
+        # USE:
+        #     ds=Statsample::CSV.read("test_csv.csv")
+        def read(filename, empty=[''],ignore_lines=0,fs=nil,rs=nil)
+        require 'csv'
+            
+                first_row=true
+                fields=[]
+                fields_data={}
+                ds=nil
+                line_number=0
+                ::CSV.open(filename,'r',fs,rs) do |row|
+                    line_number+=1
+                    if(line_number<=ignore_lines)
+                        #puts "Skip line"
+                        next
+                    end
+                    row.collect!{|c|
+                        c.to_s
+                    }
+                    if first_row
+                        fields=row.to_a.collect{|c| c.downcase}
+                        if fields.size!=fields.uniq.size
+                            repeated=fields.inject({}) {|a,v|
+                                (a[v].nil? ? a[v]=1 : a[v]+=1); a }.find_all{|k,v| v>1}.collect{|k,v|k}.join(",")
+                           
+                                raise "There are some repeated fields on the header:#{repeated}. Please, fix" 
+                        end
+                        ds=Statsample::Dataset.new(fields)
+                        first_row=false
+                    else
+                        rowa=row.to_a.collect{|c|
+                            empty.include?(c) ? nil: c
+                        }
+                        
+                        ds.add_case(rowa,false)
+                    end
+                end
+                ds.update_valid_data
+                ds
+            end
+        # Save a Dataset on a csv file
+        #
+        # USE:
+        #     Statsample::CSV.write(ds,"test_csv.csv")            
+        def write(dataset,filename, convert_comma=false,*opts)
+                writer=::CSV.open(filename,'w',*opts)
+                writer << dataset.fields
+                dataset.each_array{|row|
+                    if(convert_comma)
+                        row.collect!{|v| v.to_s.gsub(".",",")}
+                    end
+                    writer << row
+                }
+                writer.close
+            end
+		end
+    end
+    module Mx
+        class << self
+            def write(dataset,filename,type=:covariance)
+            puts "Writing MX File"
+            File.open(filename,"w") {|fp|
+                fp.puts "! #{filename}"
+                fp.puts "! Output generated by Statsample"
+                fp.puts "Data Ninput=#{dataset.fields.size} Nobservations=#{dataset.cases}"
+                fp.puts "Labels "+dataset.fields.join(" ")
+                case type
+                when :raw
+                    fp.puts "Rectangular"
+                    dataset.each {|row|
+                        out=dataset.fields.collect {|f|
+                            if dataset[f].is_valid? row[f]
+                                row[f]
+                            else
+                                "."
+                            end
+                        }
+                        fp.puts out.join("\t")
+                    }
+                    fp.puts "End Rectangular"
+                when :covariance
+                    fp.puts " CMatrix Full"
+                    cm=Statsample::Bivariate.covariance_matrix(dataset)
+                    d=(0...(cm.row_size)).collect {|row|
+                        (0...(cm.column_size)).collect{|col|
+                            cm[row,col].nil? ? "." : sprintf("%0.3f", cm[row,col])
+                        }.join(" ")
+                    }.join("\n")
+                    fp.puts d
+                end
+            }
+        end
+    end
+    end
+	module GGobi
+		class << self
+            def write(dataset,filename,opt={})
+                File.open(filename,"w") {|fp|
+                    fp.write(self.out(dataset,opt))
+                }
+            end
+			def out(dataset,opt={})
+				require 'ostruct'
+				default_opt = {:dataname => "Default", :description=>""}
+				default_opt.merge! opt
+				carrier=OpenStruct.new
+				carrier.categorials=[]
+				carrier.conversions={}
+				variables_def=dataset.vectors.collect{|k,v|
+					variable_definition(carrier,v,k)
+				}.join("\n")
+				
+				indexes=carrier.categorials.inject({}) {|s,c|
+					s[dataset.fields.index(c)]=c
+					s
+				}
+				records=""
+				dataset.each_array {|c|
+					indexes.each{|ik,iv|
+						c[ik]=carrier.conversions[iv][c[ik]]
+					}
+					records << "<record>#{values_definition(c)}</record>\n"
+				}
+				
+out=<<EOC
+<?xml version="1.0"?>
+<!DOCTYPE ggobidata SYSTEM "ggobi.dtd">
+<ggobidata count="1">
+<data name="#{default_opt[:dataname]}">
+<description>#{default_opt[:description]}</description>
+<variables count="#{dataset.fields.size}">
+#{variables_def}
+</variables>
+<records count="#{dataset.cases}">
+#{records}
+</records>
+
+</data>
+</ggobidata>
+EOC
+
+out
+
+			end
+			def values_definition(c)
+				c.collect{|v|
+					if v.is_a? Float
+						"<real>#{v}</real>"
+					elsif v.is_a? Integer
+						"<int>#{v}</int>"
+					else
+						"<string>#{v}</string>"
+					end
+				}.join(" ")
+			end
+			# Outputs a string for a variable definition
+			# v = vector
+			# name = name of the variable
+			# nickname = nickname
+			def variable_definition(carrier,v,name,nickname=nil)
+				nickname = (nickname.nil? ? "" : "nickname=\"#{nickname}\"" )
+				if v.type==:nominal or v.data.find {|d|  d.is_a? String }
+					carrier.categorials.push(name)
+					carrier.conversions[name]={}
+					factors=v.data.uniq.sort
+					out ="<categoricalvariable name=\"#{name}\" #{nickname}>\n"
+					out << "<levels count=\"#{factors.size}\">\n"
+					out << (1..factors.size).to_a.collect{|i|
+						carrier.conversions[name][factors[i-1]]=i
+						"<level value=\"#{i}\">#{v.labeling(factors[i-1])}</level>"
+					}.join("\n")
+					out << "</levels>\n</categoricalvariable>\n"
+					out
+				elsif v.data.find {|d| d.is_a? Float}
+					"<realvariable name=\"#{name}\" #{nickname} />"
+				else
+					"<integervariable name=\"#{name}\" #{nickname} />"
+				end
+			end
+
+		end
+	end
+end

data/lib/statsample/crosstab.rb

@@ -0,0 +1,122 @@
+module Statsample
+	# Class to create crosstab of data
+	# With this, you can create reports and do chi square test
+	# The first vector will be at rows and the second will the the columns
+	#
+    class Crosstab
+		attr_reader :v_rows, :v_cols
+		def initialize(v1,v2)
+			raise ArgumentError, "Both arguments should be Vectors" unless v1.instance_of? Vector and v2.instance_of? Vector
+			raise ArgumentError, "Vectors should be the same size" unless v1.size==v2.size
+			@v_rows,@v_cols=v1,v2
+		end	
+		def rows_names
+			@v_rows.factors.sort
+		end
+		def cols_names
+			@v_cols.factors.sort
+		end
+		def rows_total
+			@v_rows.frequencies
+		end
+		def cols_total
+			@v_cols.frequencies
+		end
+		def frequencies
+			base=rows_names.inject([]){|s,row| 
+				s+=cols_names.collect{|col| [row,col]}
+			}.inject({}) {|s,par|
+				s[par]=0
+				s
+			}
+			base.update(Statsample::vector_cols_matrix(@v_rows,@v_cols).to_a.to_vector.frequencies)
+		end
+        def to_matrix
+            f=frequencies
+            rn=rows_names
+            cn=cols_names
+            Matrix.rows(rn.collect{|row|
+                cn.collect{|col| f[[row,col]]}
+            })
+        end
+        def frequencies_by_row
+            f=frequencies
+            rows_names.inject({}){|sr,row|
+                sr[row]=cols_names.inject({}) {|sc,col|
+                    sc[col]=f[[row,col]]
+                    sc
+                }
+                sr
+            }
+        end
+        def frequencies_by_col
+            f=frequencies
+            cols_names.inject({}){|sc,col|
+                sc[col]=rows_names.inject({}) {|sr,row|
+                    sr[row]=f[[row,col]]
+                    sr
+                }
+                sc
+            }
+        end
+        # Chi square, based on expected and real matrix
+        def chi_square
+            require 'statsample/test'
+            Statsample::Test.chi_square(self.to_matrix,matrix_expected)
+        end
+        # Useful to obtain chi square
+        def matrix_expected
+            rn=rows_names
+            cn=cols_names
+            rt=rows_total
+            ct=cols_total
+            t=@v_rows.size.to_f
+            m=rn.collect{|row|
+                cn.collect{|col|
+                    (rt[row]*ct[col]) / t 
+                }
+            }
+            Matrix.rows(m)
+        end
+        def to_s
+            fq=frequencies
+            rn=rows_names
+            cn=cols_names
+            total=0
+            total_cols=cn.inject({}) {|a,x| a[x]=0;a}
+            max_row_size = rn.inject(0) {|s,x| sl=@v_rows.labeling(x).size; sl>s ? sl : s}
+            
+            max_row_size=max_row_size<6 ? 6 : max_row_size
+            
+            max_col_size = cn.inject(0) {|s,x| sl=@v_cols.labeling(x).size; sl>s ? sl : s}
+            max_col_size = frequencies.inject(max_col_size) {|s,x| x[1].to_s.size>s ? x[1].to_s.size : s}
+            
+            out=""
+            out << " " * (max_row_size+2) << "|" << cn.collect{|c| name=@v_cols.labeling(c); " "+name+(" "*(max_col_size-name.size))+" "}.join("|") << "| Total\n"
+            linea="-" * (max_row_size+2) << "|" << ("-"*(max_col_size+2) +"|")*cn.size << "-"*7 << "\n"
+            out << linea
+            rn.each{|row|
+                total_row=0;
+                name=@v_rows.labeling(row)
+                out << " " +name  << " "*(max_row_size-name.size) << " | "
+                cn.each{|col|
+                    data=fq[[row,col]].to_s
+                    total_row+=fq[[row,col]]
+                    total+=fq[[row,col]]                    
+                    total_cols[col]+=fq[[row,col]]                    
+                    out << " " << data << " "*(max_col_size-data.size) << "| "
+                }
+                out << " " << total_row.to_s
+            out << "\n"
+            }
+            out << linea
+            out << " Total " << " "*(max_row_size-5) << "| "
+            cn.each{|v|
+                data=total_cols[v].to_s
+                out << " " << data << " "*(max_col_size-data.size) << "| "
+            }
+            out << " " << total.to_s
+            out
+        end
+	end
+end