daru 0.0.5 → 0.1.0

data/lib/daru/extensions/rserve.rb

@@ -0,0 +1,21 @@
+# Support for converting data to R data structures to support rserve-client
+
+module Daru
+  class DataFrame
+    def to_REXP
+      names = @vectors.to_a
+      data  = names.map do |f|
+        Rserve::REXP::Wrapper.wrap(self[f].to_a)
+      end
+      l = Rserve::Rlist.new(data, names.map(&:to_s))
+
+      Rserve::REXP.create_data_frame(l)
+    end
+  end
+
+  class Vector
+    def to_REXP
+      Rserve::REXP::Wrapper.wrap(self.to_a)
+    end
+  end
+end

data/lib/daru/index.rb

@@ -101,8 +101,22 @@ module Daru
       @relation_hash.has_key? index
     end
 
+    def empty?
+      @relation_hash.empty?
+    end
+
     def dup
       Daru::Index.new @relation_hash.keys
     end
+
+    def _dump depth
+      Marshal.dump({relation_hash: @relation_hash})
+    end
+
+    def self._load data
+      h = Marshal.load data
+
+      Daru::Index.new(h[:relation_hash].keys, h[:relation_hash].values)
+    end
   end
 end

data/lib/daru/io/io.rb

@@ -1,22 +1,184 @@
 module Daru
+  module IOHelpers
+    class << self
+      def process_row(row,empty)
+        row.to_a.map do |c|
+          if empty.include?(c)
+            nil
+          else
+            if c.is_a? String and c.is_number?
+              c =~ /^\d+$/ ? c.to_i : c.gsub(",",".").to_f
+            else
+              c
+            end
+          end
+        end
+      end
+    end
+  end
+
   module IO
     class << self
+      # Functions for loading/writing Excel files.
+
+      def from_excel path, opts={}
+        opts = {
+          :worksheet_id => 0,
+        }.merge opts
+
+        worksheet_id = opts[:worksheet_id]
+        book         = Spreadsheet.open path
+        worksheet    = book.worksheet worksheet_id
+        headers      = worksheet.row(0).recode_repeated.map(&:to_sym)
+
+        df = Daru::DataFrame.new({})
+        headers.each_with_index do |h,i|
+          col = worksheet.column(i).to_a
+          col.delete_at 0
+          df[h] = col
+        end
+
+        df
+      end
+
+      def dataframe_write_excel dataframe, path, opts={}
+        book   = Spreadsheet::Workbook.new
+        sheet  = book.create_worksheet
+        format = Spreadsheet::Format.new :color => :blue, :weight => :bold
+
+        sheet.row(0).concat(dataframe.vectors.to_a.map(&:to_s)) # Unfreeze strings
+        sheet.row(0).default_format = format
+        i = 1
+        dataframe.each_row do |row|
+          sheet.row(i).concat(row.to_a)
+          i += 1
+        end
+
+        book.write(path)
+      end
+
+      # Functions for loading/writing CSV files
+
       def from_csv path, opts={}
         opts[:col_sep]           ||= ','
-        opts[:headers]           ||= true
         opts[:converters]        ||= :numeric
-        opts[:header_converters] ||= :symbol
 
-        csv = CSV.read(path, 'r', opts)
+        daru_options = opts.keys.inject({}) do |hash, k|
+          if [:clone, :order, :index, :name].include?(k)
+            hash[k] = opts[k]
+            opts.delete k
+          end
+
+          hash
+        end
+
+        # Preprocess headers for detecting and correcting repetition in 
+        # case the :headers option is not specified.
+        unless opts[:headers]
+          csv = ::CSV.open(path, 'rb', opts)
+          yield csv if block_given?
+
+          csv_as_arrays = csv.to_a
+          headers       = csv_as_arrays[0].recode_repeated.map(&:to_sym)
+          csv_as_arrays.delete_at 0
+          csv_as_arrays = csv_as_arrays.transpose
 
-        yield csv if block_given?
+          hsh = {}
+          headers.each_with_index do |h, i|
+            hsh[h] = csv_as_arrays[i]
+          end
+        else
+          opts[:header_converters] ||= :symbol
+          
+          csv = ::CSV.read(path, 'rb',opts)
+          yield csv if block_given?
 
-        hsh = {}
-        csv.by_col!.each do |col_name, values|
-          hsh[col_name] = values
+          hsh = {}
+          csv.by_col.each do |col_name, values|
+            hsh[col_name] = values
+          end
         end
 
-        Daru::DataFrame.new(hsh)
+        Daru::DataFrame.new(hsh,daru_options)
+      end
+
+      def dataframe_write_csv dataframe, path, opts={}
+        options = {
+          converters: :numeric
+        }.merge(opts)
+
+        writer = ::CSV.open(path, 'w', options)
+        writer << dataframe.vectors.to_a
+
+        dataframe.each_row do |row|
+          if options[:convert_comma]
+            writer << row.map { |v| v.to_s.gsub('.', ',') }
+          else
+            writer << row.to_a
+          end
+        end
+
+        writer.close
+      end
+
+      # Loading/writing from SQL databases
+
+      def from_sql dbh, query
+        require 'dbi'
+        sth     = dbh.execute(query)
+        vectors = {}
+        fields  = []
+        sth.column_info.each do |c|
+          vectors[c[:name]] = Daru::Vector.new([])
+          vectors[c[:name]].rename c[:name]
+          fields.push(c[:name].to_sym)
+        end
+        ds=Daru::DataFrame.new(vectors,order: fields)
+        sth.fetch do |row|
+          ds.add_row(row.to_a)
+        end
+        ds.update
+        ds
+      end
+
+      def dataframe_write_sql ds, dbh, table
+        require 'dbi'
+        query = "INSERT INTO #{table} ("+ds.vectors.to_a.join(",")+") VALUES ("+((["?"]*ds.vectors.size).join(","))+")"
+        sth   =  dbh.prepare(query)
+        ds.each_row { |c| sth.execute(*c.to_a) }
+        return true
+      end
+
+      # Loading data from plain text files
+
+      def from_plaintext filename, fields
+        ds = Daru::DataFrame.new({}, order: fields)
+        fp = File.open(filename,"r")
+        fp.each_line do |line|
+          row = Daru::IOHelpers.process_row(line.strip.split(/\s+/),[""])
+          next if row == ["\x1A"]
+          ds.add_row(row)
+        end
+        ds.update
+        fields.each { |f| ds[f].rename f }
+        ds
+      end
+
+      # Loading and writing Marshalled DataFrame/Vector
+      def save klass, filename
+        fp = File.open(filename, 'w')
+        Marshal.dump(klass, fp)
+        fp.close
+      end
+
+      def load filename
+        if File.exist? filename
+          o = false
+          File.open(filename, 'r') { |fp| o = Marshal.load(fp) }
+          o
+        else
+          false
+        end
       end
     end
   end

data/lib/daru/maths/arithmetic/dataframe.rb

@@ -36,15 +36,16 @@ module Daru
 
         # Calculate exponenential of all vectors with numeric values.
         def exp
-          self.dup.map_vectors! { |v| v.exp if v.type == :numeric }
+          only_numerics(clone: false).recode { |v| v.exp }
         end
 
+        # Calcuate square root of numeric vectors.
         def sqrt
-          self.dup.map_vectors! { |v| v.sqrt if v.type == :numeric }
+          only_numerics(clone: false).recode { |v| v.sqrt }
         end
 
         def round precision=0
-          self.dup.map_vectors! { |v| v.round(precision) if v.type == :numeric }
+          only_numerics(clone: false).recode { |v| v.round(precision) }
         end
        private
 

data/lib/daru/maths/arithmetic/vector.rb

@@ -35,17 +35,17 @@ module Daru
         end
 
         def abs
-          self.dup.map! { |e| e.abs unless e.nil? }
+          self.recode { |e| e.abs unless e.nil? }
         end
 
         def round precision=0
-          self.dup.map! { |e| e.round(precision) unless e.nil? }
+          self.recode { |e| e.round(precision) unless e.nil? }
         end
 
        private
 
         def math_unary_op operation
-          self.dup.map! { |e| Math.send(operation, e) unless e.nil? }
+          self.recode { |e| Math.send(operation, e) unless e.nil? }
         end
 
         def binary_op operation, other
@@ -65,7 +65,7 @@ module Daru
         def v2v_binary operation, other
           common_idxs = []
           elements    = []
-          index = (@index.to_a + other.index.to_a).uniq.sort
+          index = (@index.to_a | other.index.to_a).sort
 
           index.each do |idx|
             this = self[idx]

data/lib/daru/maths/statistics/dataframe.rb

@@ -37,6 +37,15 @@ module Daru
           compute_stats :product
         end
 
+        def standardize
+          df = self.only_numerics clone: true
+          df.map! do |v|
+            v.standardize
+          end
+
+          df
+        end
+
         # Create a summary of mean, standard deviation, count, max and min of 
         # each numeric vector in the dataframe in one shot.
         # 
@@ -55,40 +64,39 @@ module Daru
           Daru::DataFrame.new(description_hash, index: methods)
         end
 
-        # Calculate variance-covariance between the numeric vectors.
-        # 
-        # == Arguments
-        # 
-        # +for_sample_data+ - If set to false, will calculate the population 
-        # covariance (denominator N), otherwise calculates the sample covariance
-        # matrix. Default to true.
-        def covariance for_sample_data=true
-          cov_arry = 
-          if defined? NMatrix and NMatrix.respond_to?(:cov)
-            to_nmatrix.cov(for_sample_data).to_a
-          else
-            df_as_matrix = to_matrix
-            denominator  = for_sample_data ? rows - 1 : rows
-            ones         = Matrix.column_vector [1]*rows
-            deviation_scores = df_as_matrix - (ones * ones.transpose * df_as_matrix) / rows
-            ((deviation_scores.transpose * deviation_scores) / denominator).to_a
+        # Calculate sample variance-covariance between the numeric vectors.
+        def covariance
+          cache={}
+          vectors = self.numeric_vectors
+
+          mat_rows = vectors.collect do |row|
+            vectors.collect do |col|
+              if row == col
+                self[row].variance
+              else
+                if cache[[col,row]].nil?
+                  cov = vector_cov(self[row],self[col])
+                  cache[[row,col]] = cov
+                  cov
+                else
+                  cache[[col,row]]
+                end
+              end
+            end
           end
 
-          Daru::DataFrame.rows(cov_arry, index: numeric_vectors, order: numeric_vectors)
+          Daru::DataFrame.rows(mat_rows, index: numeric_vectors, order: numeric_vectors)
         end
 
         alias :cov :covariance
           
         # Calculate the correlation between the numeric vectors.
         def correlation
-          corr_arry = 
-          if defined? NMatrix and NMatrix.respond_to?(:corr)
-            to_nmatrix.corr.to_a
-          else
-            standard_deviation = std.to_matrix
-            (cov.to_matrix.elementwise_division(standard_deviation.transpose * 
-              standard_deviation)).to_a
-          end
+          standard_deviation = std.to_matrix
+          corr_arry = (cov
+            .to_matrix
+            .elementwise_division(standard_deviation.transpose * 
+            standard_deviation)).to_a
 
           Daru::DataFrame.rows(corr_arry, index: numeric_vectors, order: numeric_vectors)
         end
@@ -97,12 +105,25 @@ module Daru
 
        private
 
+        def vector_cov v1a, v2a
+          sum_of_squares(v1a,v2a) / (v1a.size - 1)
+        end
+
+        def sum_of_squares v1, v2
+          v1a,v2a = v1.only_valid ,v2.only_valid
+          v1a.reset_index!
+          v2a.reset_index!        
+          m1 = v1a.mean
+          m2 = v2a.mean
+          (v1a.size).times.inject(0) {|ac,i| ac+(v1a[i]-m1)*(v2a[i]-m2)}
+        end
+
         def compute_stats method
           Daru::Vector.new(
             numeric_vectors.inject({}) do |hash, vec|
               hash[vec] = self[vec].send(method)
               hash
-            end
+            end, name: method
           )
         end
       end

data/lib/daru/maths/statistics/vector.rb

@@ -1,8 +1,8 @@
 module Daru
   module Maths
     # Encapsulates statistics methods for vectors. Most basic stuff like mean, etc.
-    #   is done inside the wrapper, so that native methods can be used for most of 
-    #   the computationally intensive tasks.
+    # is done inside the wrapper, so that native methods can be used for most of 
+    # the computationally intensive tasks.
     module Statistics
       module Vector
         def mean
@@ -26,7 +26,7 @@ module Daru
         end
 
         def median
-          percentile 50
+          @data.respond_to?(:median) ? @data.median : percentile(50)
         end
 
         def mode
@@ -36,15 +36,21 @@ module Daru
 
         def median_absolute_deviation
           m = median
-          map {|val| (val - m).abs }.median
+          recode {|val| (val - m).abs }.median
         end
+        alias :mad :median_absolute_deviation
 
         def standard_error
-          standard_deviation_sample/(Math::sqrt((@size - @nil_positions.size)))
+          standard_deviation_sample/(Math::sqrt((n_valid)))
         end
 
         def sum_of_squared_deviation
-          (@data.to_a.inject(0) { |a,x| x.square + a } - (sum.square.quo((@size - @nil_positions.size)))).to_f
+          (@data.inject(0) { |a,x| x.square + a } - (sum.square.quo(n_valid)).to_f).to_f
+        end
+
+        # Retrieve unique values of non-nil data
+        def factors
+          only_valid.uniq.reset_index!
         end
 
         # Maximum element of the vector.
@@ -69,12 +75,18 @@ module Daru
 
         def frequencies
           @data.inject({}) do |hash, element|
-            hash[element] ||= 0
-            hash[element] += 1
+            unless element.nil?
+              hash[element] ||= 0
+              hash[element] += 1
+            end
             hash
           end
         end
 
+        def freqs
+          Daru::Vector.new(frequencies)
+        end
+
         def proportions
           len = n_valid
           frequencies.inject({}) { |hash, arr| hash[arr[0]] = arr[1] / len; hash }
@@ -83,13 +95,12 @@ module Daru
         def ranked
           sum = 0
           r = frequencies.sort.inject( {} ) do |memo, val|
-            memo[val[0]] = ((sum + 1) + (sum + val[1])) / 2
+            memo[val[0]] = ((sum + 1) + (sum + val[1])).quo(2)
             sum += val[1]
             memo
           end
 
-          Daru::Vector.new @data.map { |e| r[e] }, index: self.index,
-            name: self.name, dtype: self.dtype, nm_dtype: self.nm_dtype
+          recode { |e| r[e] }
         end
 
         def coefficient_of_variation
@@ -107,69 +118,239 @@ module Daru
             val = frequencies[value]
             val.nil? ? 0 : val
           else
-            size - @nil_positions.size
+            size - @missing_positions.size
           end
         end
 
         def proportion value=1
-          frequencies[value] / n_valid
+          frequencies[value].quo(n_valid).to_f
         end
 
         # Sample variance with denominator (N-1)
         def variance_sample m=nil
           m ||= self.mean
-          sum_of_squares(m).quo((@size - @nil_positions.size) - 1)
+          if @data.respond_to? :variance_sample
+            @data.variance_sample m
+          else
+            sum_of_squares(m).quo((n_valid) - 1)
+          end
         end
 
         # Population variance with denominator (N)
         def variance_population m=nil
           m ||= mean
-          sum_of_squares(m).quo((@size - @nil_positions.size)).to_f
+          if @data.respond_to? :variance_population
+            @data.variance_population m
+          else
+            sum_of_squares(m).quo((n_valid)).to_f            
+          end
         end
 
         def sum_of_squares(m=nil)
           m ||= mean
-          @data.inject(0) { |memo, val| memo + (val - m)**2 }
+          @data.inject(0) { |memo, val| 
+            @missing_values.has_key?(val) ? memo : (memo + (val - m)**2) 
+          }
         end
 
         def standard_deviation_population m=nil
           m ||= mean
-          Math::sqrt(variance_population(m))
+          if @data.respond_to? :standard_deviation_population
+            @data.standard_deviation_population(m)
+          else
+            Math::sqrt(variance_population(m))
+          end
         end
 
         def standard_deviation_sample m=nil
-          Math::sqrt(variance_sample(m))
+          m ||= mean
+          if @data.respond_to? :standard_deviation_sample
+            @data.standard_deviation_sample m
+          else
+            Math::sqrt(variance_sample(m))
+          end
         end
 
         # Calculate skewness using (sigma(xi - mean)^3)/((N)*std_dev_sample^3)
         def skew m=nil
-          m ||= mean
-          th  = @data.inject(0) { |memo, val| memo + ((val - m)**3) }
-          th.quo ((@size - @nil_positions.size) * (standard_deviation_sample(m)**3))
+          if @data.respond_to? :skew
+            @data.skew
+          else
+            m ||= mean
+            th  = @data.inject(0) { |memo, val| memo + ((val - m)**3) }
+            th.quo ((@size - @missing_positions.size) * (standard_deviation_sample(m)**3))
+          end
         end
 
         def kurtosis m=nil
-          m ||= mean
-          fo  = @data.inject(0){ |a, x| a + ((x - m) ** 4) }
-          fo.quo((@size - @nil_positions.size) * standard_deviation_sample(m) ** 4) - 3
+          if @data.respond_to? :kurtosis
+            @data.kurtosis
+          else
+            m ||= mean
+            fo  = @data.inject(0){ |a, x| a + ((x - m) ** 4) }
+            fo.quo((@size - @missing_positions.size) * standard_deviation_sample(m) ** 4) - 3
+          end
         end
 
         def average_deviation_population m=nil
+          type == :numeric or raise TypeError, "Vector must be numeric"
+          m ||= mean
+          (@data.inject( 0 ) { |memo, val| 
+            @missing_values.has_key?(val) ? memo : ( val - m ).abs + memo
+          }).quo( n_valid )
+        end
+
+        # Returns the value of the percentile q
+        #
+        # Accepts an optional second argument specifying the strategy to interpolate
+        # when the requested percentile lies between two data points a and b
+        # Valid strategies are:
+        # * :midpoint (Default): (a + b) / 2
+        # * :linear : a + (b - a) * d where d is the decimal part of the index between a and b.
+        # == References
+        # 
+        # This is the NIST recommended method (http://en.wikipedia.org/wiki/Percentile#NIST_method)
+        def percentile(q, strategy = :midpoint)
+          sorted = only_valid(:array).sort
+
+          case strategy
+          when :midpoint
+            v = (n_valid * q).quo(100)
+            if(v.to_i!=v)
+              sorted[v.to_i]
+            else
+              (sorted[(v-0.5).to_i].to_f + sorted[(v+0.5).to_i]).quo(2)
+            end
+          when :linear
+            index = (q / 100.0) * (n_valid + 1)
+
+            k = index.truncate
+            d = index % 1
+
+            if k == 0
+              sorted[0]
+            elsif k >= sorted.size
+              sorted[-1]
+            else
+              sorted[k - 1] + d * (sorted[k] - sorted[k - 1])
+            end
+          else
+            raise NotImplementedError.new "Unknown strategy #{strategy.to_s}"
+          end
+        end
+
+        # Dichotomize the vector with 0 and 1, based on lowest value.
+        # If parameter is defined, this value and lower will be 0 
+        # and higher, 1.
+        def dichotomize(low = nil)
+          low ||= factors.min
+
+          self.recode do |x|
+            if x.nil? 
+              nil
+            elsif x > low
+              1
+            else
+              0
+            end
+          end
+        end
+
+        # Center data by subtracting the mean from each non-nil value.
+        def center
+          self - mean
+        end
+
+        # Standardize data.
+        # 
+        # == Arguments
+        # 
+        # * use_population - Pass as *true* if you want to use population
+        # standard deviation instead of sample standard deviation.
+        def standardize use_population=false
           m ||= mean
-          (@data.inject(0) {|memo, val| val + (val - m).abs }) / n_valid
+          sd = use_population ? sdp : sds
+          return Daru::Vector.new([nil]*@size) if m.nil? or sd == 0.0
+
+          vector_standardized_compute m, sd
         end
 
-        def recode!(&block)
-          @data.recode!(&block)
+        def box_cox_transformation lambda # :nodoc:
+          raise "Should be a numeric" unless @type == :numeric
+
+          self.recode do |x|
+            if !x.nil?
+              if(lambda == 0)
+                Math.log(x)
+              else
+                (x ** lambda - 1).quo(lambda)
+              end
+            else
+              nil
+            end
+          end
         end
 
-        def percentile percent
-          sorted = @data.sort
-          v      = (n_valid * percent).quo(100)
-          if v.to_i != v
-            sorted[v.round]
+        # Replace each non-nil value in the vector with its percentile.
+        def vector_percentile
+          c = size - missing_positions.size
+          ranked.recode! { |i| i.nil? ? nil : (i.quo(c)*100).to_f }
+        end
+
+        def vector_standardized_compute(m,sd)
+          if @data.respond_to? :vector_standardized_compute
+            @data.vector_standardized_compute(m,sd)
           else
-            (sorted[(v - 0.5).round].to_f + sorted[(v + 0.5).round]).quo(2)
+            Daru::Vector.new @data.collect { |x| x.nil? ? nil : (x.to_f - m).quo(sd) },
+              index: index, name: name, dtype: dtype
+          end
+        end
+        
+        def vector_centered_compute(m)
+          if @data.respond_to? :vector_centered_compute
+            @data.vector_centered_compute(m)
+          else
+            Daru::Vector.new @data.collect { |x| x.nil? ? nil : x.to_f-m },
+              index: index, name: name, dtype: dtype
+          end
+        end
+
+        # Returns an random sample of size n, with replacement,
+        # only with non-nil data.
+        #
+        # In all the trails, every item have the same probability
+        # of been selected.
+        def sample_with_replacement(sample=1)
+          if @data.respond_to? :sample_with_replacement
+            @data.sample_with_replacement sample
+          else
+            valid = missing_positions.empty? ? self : self.only_valid
+            vds = valid.size
+            (0...sample).collect{ valid[rand(vds)] }
+          end
+        end
+        
+        # Returns an random sample of size n, without replacement,
+        # only with valid data.
+        #
+        # Every element could only be selected once.
+        #
+        # A sample of the same size of the vector is the vector itself.
+        def sample_without_replacement(sample=1)
+          if @data.respond_to? :sample_without_replacement
+            @data.sample_without_replacement sample
+          else
+            valid = missing_positions.empty? ? self : self.only_valid 
+            raise ArgumentError, "Sample size couldn't be greater than n" if 
+              sample > valid.size
+            out  = []
+            size = valid.size
+            while out.size < sample
+              value = rand(size)
+              out.push(value) if !out.include?(value)
+            end
+
+            out.collect{|i| valid[i]}
           end
         end
 
@@ -182,6 +363,7 @@ module Daru
         alias :sd :standard_deviation_sample
         alias :ss :sum_of_squares
         alias :percentil :percentile
+        alias :se :standard_error
       end
     end
   end