statsample-ekatena 2.0.2

data/lib/statsample-ekatena/bivariate/pearson.rb

@@ -0,0 +1,54 @@
+module Statsample
+  module Bivariate
+    # = Pearson correlation coefficient (r) 
+    # 
+    # The moment-product Pearson's correlation coefficient, known as 'r'
+    # is a measure of bivariate associate between two continous
+    # variables.
+    # 
+    # == Usage
+    #   a = Daru::Vector.new([1,2,3,4,5,6])
+    #   b = Daru::Vector.new([2,3,4,5,6,7])
+    #   pearson = Statsample::Bivariate::Pearson.new(a,b)
+    #   puts pearson.r
+    #   puts pearson.t
+    #   puts pearson.probability
+    #   puts pearson.summary
+    # 
+    class Pearson
+      
+      include Statsample::Test
+      include Summarizable
+      # Name of correlation
+      attr_accessor :name
+      # Tails for probability (:both, :left or :right)
+      attr_accessor :tails     
+      attr_accessor :n      
+      def initialize(v1,v2,opts=Hash.new)
+        @v1_name,@v2_name = v1.name,v2.name
+        @v1,@v2           = Statsample.only_valid_clone(v1,v2)
+        @n=@v1.size
+        opts_default={
+          :name=>_("Correlation (%s - %s)") % [@v1_name, @v2_name],
+          :tails=>:both
+        }
+        @opts=opts.merge(opts_default)
+        @opts.each{|k,v|
+          self.send("#{k}=",v) if self.respond_to? k
+        }
+      end
+      def r
+        Statsample::Bivariate.pearson(@v1,@v2)
+      end
+      def t
+        Statsample::Bivariate.t_pearson(@v1,@v2)
+      end
+      def probability
+        p_using_cdf(Distribution::T.cdf(t, @v1.size-2), tails)
+      end
+      def report_building(builder)
+        builder.text(_("%s : r=%0.3f (t:%0.3f, g.l.=%d, p:%0.3f / %s tails)") % [@name, r,t, (n-2), probability, tails])
+      end
+    end
+  end
+end

data/lib/statsample-ekatena/codification.rb

@@ -0,0 +1,182 @@
+require 'yaml'
+
+module Statsample
+  # This module aids to code open questions
+  # * Select one or more vectors of a dataset, to create a yaml files, on which each vector is a hash, which keys and values are the vector's factors . If data have Statsample::SPLIT_TOKEN on a value, each value will be separated on two or more hash keys.
+  # * Edit the yaml and replace the values of hashes with your codes. If you need to create two or mores codes for an answer, use the separator (default Statsample::SPLIT_TOKEN)
+  # * Recode the vectors, loading the yaml file:
+  #   * recode_dataset_simple!() : The new vectors have the same name of the original plus "_recoded"
+  #   * recode_dataset_split!() : Create equal number of vectors as values. See Vector.add_vectors_by_split() for arguments
+  #
+  # Usage:
+  #   recode_file="recodification.yaml"
+  #   phase=:first # flag
+  #   if phase==:first
+  #     File.open(recode_file,"w") {|fp|
+  #       Statsample::Codification.create_yaml(ds,%w{vector1 vector2}, ",",fp)
+  #     }
+  #   # Edit the file recodification.yaml and verify changes
+  #   elsif phase==:second
+  #     File.open(recode_file,"r") {|fp|
+  #       Statsample::Codification.verify(fp,['vector1'])
+  #     }
+  #   # Add new vectors to the dataset
+  #   elsif phase==:third
+  #     File.open(recode_file,"r") {|fp|
+  #       Statsample::Codification.recode_dataset_split!(ds,fp,"*")
+  #     }
+  #   end
+  #
+  module Codification
+    class << self
+      # Create a hash, based on vectors, to create the dictionary.
+      # The keys will be vectors name on dataset and the values
+      # will be hashes, with keys = values, for recodification
+      def create_hash(dataset, vectors, sep=Statsample::SPLIT_TOKEN)
+        raise ArgumentError,"Array should't be empty" if vectors.size==0
+        pro_hash = vectors.inject({}) do |h,v_name|
+          v_name = v_name.is_a?(Numeric) ? v_name : v_name.to_sym
+          raise Exception, "Vector #{v_name} doesn't exists on Dataset" if 
+            !dataset.vectors.include?(v_name)
+          v = dataset[v_name]
+          split_data = v.splitted(sep)
+                        .flatten
+                        .collect { |c| c.to_s  }
+                        .find_all{ |c| !c.nil? }
+
+          factors   = split_data.uniq
+                                .compact
+                                .sort
+                                .inject({}) { |ac,val| ac[val] = val; ac }
+          h[v_name] = factors
+          h
+        end
+
+        pro_hash
+      end
+      # Create a yaml to create a dictionary, based on vectors
+      # The keys will be vectors name on dataset and the values
+      # will be hashes, with keys = values, for recodification
+      #
+      #   v1 = Daru::Vector.new(%w{a,b b,c d})
+      #   ds = Daru::DataFrame.new({:v1 => v1})
+      #   Statsample::Codification.create_yaml(ds,[:v1])
+      #   => "--- \nv1: \n  a: a\n  b: b\n  c: c\n  d: d\n"
+      def create_yaml(dataset, vectors, io=nil, sep=Statsample::SPLIT_TOKEN)
+        pro_hash=create_hash(dataset, vectors, sep)
+        YAML.dump(pro_hash,io)
+      end
+      # Create a excel to create a dictionary, based on vectors.
+      # Raises an error if filename exists
+      # The rows will be:
+      # * field: name of vector
+      # * original: original name
+      # * recoded: new code
+
+      def create_excel(dataset, vectors, filename, sep=Statsample::SPLIT_TOKEN)
+        require 'spreadsheet'
+        if File.exist?(filename)
+          raise "Exists a file named #{filename}. Delete ir before overwrite."
+        end
+        book  = Spreadsheet::Workbook.new
+        sheet = book.create_worksheet
+        sheet.row(0).concat(%w(field original recoded))
+        i = 1
+        create_hash(dataset, vectors, sep).sort.each do |field, inner_hash|
+          inner_hash.sort.each do |k,v|
+            sheet.row(i).concat([field.to_s,k.to_s,v.to_s])
+            i += 1
+          end
+        end
+
+        book.write(filename)
+      end
+      # From a excel generates a dictionary hash
+      # to use on recode_dataset_simple!() or recode_dataset_split!().
+      #
+      def excel_to_recoded_hash(filename)
+        require 'spreadsheet'
+        h={}
+        book = Spreadsheet.open filename
+        sheet= book.worksheet 0
+        row_i=0
+        sheet.each do |row|
+          row_i += 1
+          next if row_i == 1 or row[0].nil? or row[1].nil? or row[2].nil?
+          key = row[0].to_sym
+          h[key] ||= {}
+          h[key][row[1]] = row[2]
+        end
+        h
+      end
+
+      def inverse_hash(h, sep=Statsample::SPLIT_TOKEN)
+        h.inject({}) do |a,v|
+          v[1].split(sep).each do |val|
+            a[val]||=[]
+            a[val].push(v[0])
+          end
+          a
+        end
+      end
+
+      def dictionary(h, sep=Statsample::SPLIT_TOKEN)
+        h.inject({}) { |a,v| a[v[0]]=v[1].split(sep); a }
+      end
+
+      def recode_vector(v,h,sep=Statsample::SPLIT_TOKEN)
+        dict     = dictionary(h,sep)
+        new_data = v.splitted(sep)
+        new_data.collect do |c|
+          if c.nil?
+            nil
+          else
+            c.collect{|value| dict[value] }.flatten.uniq
+          end
+        end
+      end
+      def recode_dataset_simple!(dataset, dictionary_hash ,sep=Statsample::SPLIT_TOKEN)
+        _recode_dataset(dataset,dictionary_hash ,sep,false)
+      end
+      def recode_dataset_split!(dataset, dictionary_hash, sep=Statsample::SPLIT_TOKEN)
+        _recode_dataset(dataset, dictionary_hash, sep,true)
+      end
+
+      def _recode_dataset(dataset, h , sep=Statsample::SPLIT_TOKEN, split=false)
+        v_names||=h.keys
+        v_names.each do |v_name|
+          raise Exception, "Vector #{v_name} doesn't exists on Dataset" if !dataset.vectors.include? v_name
+          recoded = Daru::Vector.new(
+            recode_vector(dataset[v_name], h[v_name],sep).collect do |c|
+              if c.nil?
+                nil
+              else
+                c.join(sep)
+              end
+            end
+          )
+          if split
+            recoded.split_by_separator(sep).each {|k,v|
+              dataset[(v_name.to_s + "_" + k).to_sym] = v
+            }
+          else
+            dataset[(v_name.to_s + "_recoded").to_sym] = recoded
+          end
+        end
+      end
+
+
+      def verify(h, v_names=nil,sep=Statsample::SPLIT_TOKEN,io=$>)
+        require 'pp'
+        v_names||=h.keys
+        v_names.each{|v_name|
+          inverse=inverse_hash(h[v_name],sep)
+          io.puts "- Field: #{v_name}"
+          inverse.sort{|a,b| -(a[1].count<=>b[1].count)}.each {|k,v|
+            io.puts "  - \"#{k}\" (#{v.count}) :\n    -'"+v.join("\n    -'")+"'"
+          }
+        }
+      end
+    end
+  end
+end

data/lib/statsample-ekatena/converter/csv.rb

@@ -0,0 +1,28 @@
+# This module will be removed in the next release.
+# Please shift to using Daru::DataFrame.from_csv and #write_csv for CSV
+# related operations.
+module Statsample
+  class CSV
+    class << self
+      # Return a DataFrom created from a csv file.
+      #
+      # == NOTE
+      # 
+      # This method has been DEPRECATED in favour of Daru::DataFrame.from_csv.
+      # Please switch to using that.
+      def read(filename, empty = [''], ignore_lines = 0, opts = {})
+        raise NoMethodError, "Deprecated. Use Daru::DataFrame.from_csv instead."
+      end
+
+      # Save a Dataset on a csv file.
+      #
+      # == NOTE
+      # 
+      # This method has BEEN DEPRECATED in favor of Daru::DataFrame#write_csv.
+      # Please use that instead.
+      def write(dataset, filename, convert_comma = false, opts = {})
+        raise NoMethodError, "Deprecated. Use Daru::DataFrame#write_csv instead."
+      end
+    end
+  end
+end

data/lib/statsample-ekatena/converter/spss.rb

@@ -0,0 +1,48 @@
+module Statsample
+  module SPSS
+    class << self
+      # Export a SPSS Matrix with tetrachoric correlations .
+      #
+      # Use: 
+      #   ds=Daru::DataFrame.from_excel("my_data.xls")
+      #   puts Statsample::SPSS.tetrachoric_correlation_matrix(ds)
+      def tetrachoric_correlation_matrix(ds)
+        dsv=ds.reject_values(*Daru::MISSING_VALUES)
+        # Delete all vectors doesn't have variation
+        dsv.vectors.each { |f|
+          if dsv[f].factors.size==1
+            dsv.delete_vector(f) 
+          else
+            dsv[f]=dsv[f].dichotomize
+          end
+        }
+
+        tcm=Statsample::Bivariate.tetrachoric_correlation_matrix(dsv)
+        n=dsv.vectors.to_a.collect {|f|
+          sprintf("%d",dsv[f].size)
+        }
+        meanlist=dsv.vectors.to_a.collect{|f|
+          sprintf("%0.3f", dsv[f].mean)
+        }
+        stddevlist=dsv.vectors.to_a.collect{|f|
+          sprintf("%0.3f", dsv[f].sd)
+        }
+        out=<<-HEREDOC
+MATRIX DATA VARIABLES=ROWTYPE_ #{dsv.fields.join(",")}.
+BEGIN DATA
+N #{n.join(" ")}
+MEAN	#{meanlist.join(" ")}
+STDDEV #{stddevlist.join(" ")}
+HEREDOC
+tcm.row_size.times {|i|
+  out +="CORR "
+  (i+1).times {|j|
+    out+=sprintf("%0.3f",tcm[i,j])+" "
+  }
+  out +="\n"
+}
+out+="END DATA.\nEXECUTE.\n"
+      end
+    end
+  end
+end

data/lib/statsample-ekatena/converters.rb

@@ -0,0 +1,211 @@
+require 'statsample/converter/spss'
+module Statsample
+  # Create and dumps Datasets on a database
+  # 
+  # == NOTE
+  # 
+  # Deprecated. Use Daru::DataFrame.from_sql and Daru::DataFrame#write_sql
+  module Database
+    class << self
+      # Read a database query and returns a Dataset
+      #
+      # == NOTE
+      # 
+      # Deprecated. Use Daru::DataFrame.from_sql instead.
+      def read(dbh,query)
+        raise NoMethodError, "Deprecated. Use Daru::DataFrame.from_sql instead."
+      end
+
+      # Insert each case of the Dataset on the selected table
+      #
+      # == NOTE
+      # 
+      # Deprecated. Use Daru::DataFrame#write_sql instead
+      def insert(ds, dbh, table)
+        raise NoMethodError, "Deprecated. Use Daru::DataFrame#write_sql instead."
+      end
+      # Create a sql, basen on a given Dataset
+      #
+      # == NOTE
+      # 
+      # Deprecated. Use Daru::DataFrame#create_sql instead.
+      def create_sql(ds,table,charset="UTF8")
+        raise NoMethodError, "Deprecated. Use Daru::DataFrame#create_sql instead."
+      end
+    end
+  end
+  module Mondrian
+    class << self
+      def write(dataset,filename)
+        File.open(filename,"wb") do |fp|
+          fp.puts dataset.vectors.to_a.join("\t")
+          dataset.each_row do |row|
+            row2 = row.map { |v| v.nil? ? "NA" : v.to_s.gsub(/\s+/,"_") }
+            fp.puts row2.join("\t")
+          end
+        end
+      end
+    end
+  end
+
+  class PlainText
+    class << self
+      def read(filename, fields)
+        raise NoMethodError, "Deprecated. Use Daru::DataFrame.from_plaintext instead."
+      end
+    end
+  end
+    
+  # This class has been DEPRECATED. Use Daru::DataFrame::from_excel 
+  # Daru::DataFrame#write_excel for XLS file operations.
+  class Excel
+    class << self
+      # Write a Excel spreadsheet based on a dataset
+      # * TODO: Format nicely date values
+      # 
+      # == NOTE
+      # 
+      # Deprecated. Use Daru::DataFrame#write_csv.
+      def write(dataset,filename)
+        raise NoMethodError, "Deprecated. Use Daru::DataFrame#write_excel instead."
+      end
+
+      # Returns a dataset based on a xls file
+      # 
+      # == NOTE
+      # 
+      # Deprecated. Use Daru::DataFrame.from_excel instead.
+      def read(filename, opts=Hash.new)
+        raise NoMethodError, "Deprecated. Use Daru::DataFrame.from_excel instead."
+      end
+    end
+  end
+
+  module Mx
+    class << self
+      def write(dataset,filename,type=:covariance)
+        puts "Writing MX File"
+        File.open(filename,"w") do |fp|
+          fp.puts "! #{filename}"
+          fp.puts "! Output generated by Statsample"
+          fp.puts "Data Ninput=#{dataset.fields.size} Nobservations=#{dataset.cases}"
+          fp.puts "Labels " + dataset.vectors.to_a.join(" ")
+          case type
+            when :raw
+            fp.puts "Rectangular"
+            dataset.each do |row|
+              out=dataset.vectors.to_a.collect do |f|
+                if dataset[f].is_valid? row[f]
+                  row[f]
+                else
+                  "."
+                end
+              end
+              fp.puts out.join("\t")
+            end
+            fp.puts "End Rectangular"
+          when :covariance
+            fp.puts " CMatrix Full"
+            cm=Statsample::Bivariate.covariance_matrix(dataset)
+            d=(0...(cm.row_size)).collect {|row|
+              (0...(cm.column_size)).collect{|col|
+                cm[row,col].nil? ? "." : sprintf("%0.3f", cm[row,col])
+              }.join(" ")
+            }.join("\n")
+            fp.puts d
+          end
+        end
+      end
+    end
+  end
+	module GGobi
+		class << self
+      def write(dataset,filename,opt={})
+        File.open(filename,"w") {|fp|
+          fp.write(self.out(dataset,opt))
+        }
+      end
+			def out(dataset,opt={})
+				require 'ostruct'
+				default_opt = {:dataname => "Default", :description=>"", :missing=>"NA"}
+				default_opt.merge! opt
+				carrier=OpenStruct.new
+				carrier.categorials=[]
+				carrier.conversions={}
+				variables_def=dataset.vectors.to_a.collect{|k|
+					variable_definition(carrier,dataset[k],k)
+				}.join("\n")
+
+				indexes=carrier.categorials.inject({}) {|s,c|
+					s[dataset.vectors.to_a.index(c)]=c
+					s
+				}
+				records=""
+				dataset.each_row {|c|
+					indexes.each { |ik,iv|
+						c[ik] = carrier.conversions[iv][c[ik]]
+					}
+					records << "<record>#{values_definition(c, default_opt[:missing])}</record>\n"
+				}
+
+out=<<EOC
+<?xml version="1.0"?>
+<!DOCTYPE ggobidata SYSTEM "ggobi.dtd">
+<ggobidata count="1">
+<data name="#{default_opt[:dataname]}">
+<description>#{default_opt[:description]}</description>
+<variables count="#{dataset.fields.size}">
+#{variables_def}
+</variables>
+    <records count="#{dataset.cases}" missingValue="#{default_opt[:missing]}">
+#{records}
+</records>
+
+</data>
+</ggobidata>
+EOC
+
+out
+
+			end
+      def values_definition(c,missing)
+        c.collect{|v|
+          if v.nil?
+            "#{missing}"
+          elsif v.is_a? Numeric
+            "#{v}"
+          else
+            "#{v.gsub(/\s+/,"_")}"
+          end
+        }.join(" ")
+      end
+			# Outputs a string for a variable definition
+			# v = vector
+			# name = name of the variable
+			# nickname = nickname
+			def variable_definition(carrier,v,name,nickname=nil)
+				nickname = (nickname.nil? ? "" : "nickname=\"#{nickname}\"" )
+				if v.type==:object or v.to_a.find {|d|  d.is_a? String }
+					carrier.categorials.push(name)
+					carrier.conversions[name]={}
+					factors=v.factors
+					out ="<categoricalvariable name=\"#{name}\" #{nickname}>\n"
+					out << "<levels count=\"#{factors.size}\">\n"
+					out << (1..factors.size).to_a.collect{|i|
+						carrier.conversions[name][factors[i-1]]=i
+						"<level value=\"#{i}\">#{(v.labels[factors[i-1]] || factors[i-1])}</level>"
+					}.join("\n")
+					out << "</levels>\n</categoricalvariable>\n"
+					out
+				elsif v.to_a.find {|d| d.is_a? Float}
+					"<realvariable name=\"#{name}\" #{nickname} />"
+				else
+					"<integervariable name=\"#{name}\" #{nickname} />"
+				end
+			end
+		end
+	end
+end
+
+require 'statsample/converter/csv.rb'
+