daru 0.0.4 → 0.0.5

data/lib/daru/index.rb

@@ -2,10 +2,13 @@ module Daru
   class Index
     include Enumerable
 
-    # needs to iterate over keys sorted by their values. Happens right now by 
-    # virtue of ordered Hashes (ruby).
     def each(&block)
       @relation_hash.each_key(&block)
+      self
+    end
+
+    def map(&block)
+      to_a.map(&block)
     end
 
     attr_reader :relation_hash
@@ -19,6 +22,7 @@ module Daru
 
       index = 0                         if index.nil?
       index = Array.new(index) { |i| i} if index.is_a? Integer
+      index = index.to_a                if index.is_a? Daru::Index
 
       if values.nil?
         index.each_with_index do |n, idx|
@@ -36,7 +40,6 @@ module Daru
       end
 
       @relation_hash.freeze
-
       @size = @relation_hash.size
 
       if index[0].is_a?(Integer)
@@ -49,22 +52,28 @@ module Daru
     def ==(other)
       return false if other.size != @size
 
-      @relation_hash.keys == other.to_a
+      @relation_hash.keys == other.to_a and @relation_hash.values == other.relation_hash.values
     end
 
     def [](key)
       case key
       when Range
-        first = @relation_hash[key.first]
-        last  = @relation_hash[key.last]
+        if key.first.is_a?(Integer) and key.last.is_a?(Integer)
+          first = key.first
+          last = key.last
+        else
+          first = @relation_hash[key.first]
+          last  = @relation_hash[key.last]
+        end
 
         indexes = []
-
         (first..last).each do |idx|
           indexes << @relation_hash.key(idx)
         end
 
         Daru::Index.new indexes, (first..last).to_a
+      when Array # works only with numeric indices
+        Daru::Index.new key.map { |k| @relation_hash.key(k) }, key
       else
         @relation_hash[key]
       end
@@ -85,11 +94,7 @@ module Daru
     end
 
     def key(value)
-      @relation_hash.key value
-    end
-
-    def re_index new_index
-      new_index.to_index
+      @relation_hash.keys[value]
     end
 
     def include? index
@@ -99,9 +104,5 @@ module Daru
     def dup
       Daru::Index.new @relation_hash.keys
     end
-
-    def to_index
-      self
-    end
   end
 end

data/lib/daru/io/io.rb

@@ -7,23 +7,16 @@ module Daru
         opts[:converters]        ||= :numeric
         opts[:header_converters] ||= :symbol
 
-        csv = CSV.open(path, 'r', opts)
+        csv = CSV.read(path, 'r', opts)
 
         yield csv if block_given?
 
-        first = true
-        df    = nil
-
-        csv.each_with_index do |row, index|
-          if first
-            df    = Daru::DataFrame.new({}, order: csv.headers, name: opts[:name])
-            first = false
-          end
-
-          df.row[index] = row.fields
+        hsh = {}
+        csv.by_col!.each do |col_name, values|
+          hsh[col_name] = values
         end
 
-        df
+        Daru::DataFrame.new(hsh)
       end
     end
   end

data/lib/daru/maths/arithmetic/dataframe.rb

@@ -1,26 +1,90 @@
 module Daru
   module Maths
+    # Module encapsulating all aritmetic methods on DataFrame.
     module Arithmetic
-      module DataFrame
-        
+      module DataFrame 
+
+        # Add a scalar or another DataFrame
         def + other
-          
+          binary_operation :+, other
         end
 
-        def - other
-          
+        # Subtract a scalar or another DataFrame.
+        def - other 
+          binary_operation :-, other
         end
 
+        # Multiply a scalar or another DataFrame.
         def * other
-          
+          binary_operation :*, other
         end
 
+        # Divide a scalar or another DataFrame. 
         def / other
-          
+          binary_operation :/, other
         end
 
+        # Modulus with a scalar or another DataFrame.
         def % other
-          
+          binary_operation :%, other
+        end
+
+        # Exponent with a scalar or another DataFrame.
+        def ** other
+          binary_operation :**, other
+        end
+
+        # Calculate exponenential of all vectors with numeric values.
+        def exp
+          self.dup.map_vectors! { |v| v.exp if v.type == :numeric }
+        end
+
+        def sqrt
+          self.dup.map_vectors! { |v| v.sqrt if v.type == :numeric }
+        end
+
+        def round precision=0
+          self.dup.map_vectors! { |v| v.round(precision) if v.type == :numeric }
+        end
+       private
+
+        def binary_operation operation, other
+          case other
+          when Daru::DataFrame
+            dataframe_binary_operation operation, other
+          else
+            scalar_binary_operation operation, other
+          end
+        end
+
+        def dataframe_binary_operation operation, other
+          all_vectors = (self.vectors.to_a | other.vectors.to_a).sort
+          all_indexes = (self.index.to_a   | other.index.to_a).sort
+
+          hsh = {}
+          all_vectors.each do |vector_name|
+            this = self .has_vector?(vector_name) ? self .vector[vector_name] : nil
+            that = other.has_vector?(vector_name) ? other.vector[vector_name] : nil
+
+            if this and that
+              hsh[vector_name] = this.send(operation, that)
+            else
+              hsh[vector_name] = Daru::Vector.new([], index: all_indexes, 
+                name: vector_name)
+            end
+          end
+
+          Daru::DataFrame.new(hsh, index: all_indexes, name: @name, dtype: @dtype)          
+        end
+
+        def scalar_binary_operation operation, other
+          clone = self.dup
+          clone.map_vectors! do |vector|
+            vector = vector.send(operation, other) if vector.type == :numeric
+            vector
+          end
+
+          clone
         end
       end
     end

data/lib/daru/maths/arithmetic/vector.rb

@@ -27,19 +27,27 @@ module Daru
         end
 
         def exp
-          
+          math_unary_op :exp
         end
 
         def sqrt
-          
+          math_unary_op :sqrt
         end
 
-        def round
-          
+        def abs
+          self.dup.map! { |e| e.abs unless e.nil? }
+        end
+
+        def round precision=0
+          self.dup.map! { |e| e.round(precision) unless e.nil? }
         end
 
        private
 
+        def math_unary_op operation
+          self.dup.map! { |e| Math.send(operation, e) unless e.nil? }
+        end
+
         def binary_op operation, other
           case other
           when Daru::Vector
@@ -50,20 +58,25 @@ module Daru
         end
 
         def v2o_binary operation, other
-          Daru::Vector.new self.map { |e| e.send(operation, other) }, name: @name, index: @index
+          Daru::Vector.new self.map { |e| e.nil? ? nil : e.send(operation, other) },
+           name: @name, index: @index
         end
 
         def v2v_binary operation, other
           common_idxs = []
           elements    = []
+          index = (@index.to_a + other.index.to_a).uniq.sort
 
-          @index.each do |idx|
+          index.each do |idx|
             this = self[idx]
             that = other[idx]
 
             if this and that
               elements << this.send(operation ,that)
               common_idxs << idx
+            else
+              elements << nil
+              common_idxs << idx
             end
           end
 

data/lib/daru/maths/statistics/dataframe.rb

@@ -2,8 +2,109 @@ module Daru
   module Maths
     module Statistics
       module DataFrame
-        
-        
+        # Calculate mean of numeric vectors.
+        def mean
+          compute_stats :mean
+        end
+
+        # Calculate sample standard deviation of numeric vectors.
+        def std
+          compute_stats :std
+        end
+
+        # Calculate sum of numeric vectors
+        def sum
+          compute_stats :sum
+        end
+
+        # Count the number of non-nil values in each vector.
+        def count
+          compute_stats :count
+        end
+
+        # Calculate the maximum value of each numeric vector.
+        def max
+          compute_stats :max
+        end
+
+        # Calculate the minimmum value of each numeric vector.
+        def min
+          compute_stats :min
+        end
+
+        # Compute the product of each numeric vector.
+        def product
+          compute_stats :product
+        end
+
+        # Create a summary of mean, standard deviation, count, max and min of 
+        # each numeric vector in the dataframe in one shot.
+        # 
+        # == Arguments
+        # 
+        # +methods+ - An array with aggregation methods specified as symbols to 
+        # be applied to numeric vectors. Default is [:count, :mean, :std, :max, 
+        # :min]. Methods will be applied in the specified order.
+        def describe methods=nil
+          methods ||= [:count, :mean, :std, :min, :max]
+
+          description_hash = {}
+          numeric_vectors.each do |vec|
+            description_hash[vec] = methods.map { |m| self[vec].send(m) }
+          end
+          Daru::DataFrame.new(description_hash, index: methods)
+        end
+
+        # Calculate variance-covariance between the numeric vectors.
+        # 
+        # == Arguments
+        # 
+        # +for_sample_data+ - If set to false, will calculate the population 
+        # covariance (denominator N), otherwise calculates the sample covariance
+        # matrix. Default to true.
+        def covariance for_sample_data=true
+          cov_arry = 
+          if defined? NMatrix and NMatrix.respond_to?(:cov)
+            to_nmatrix.cov(for_sample_data).to_a
+          else
+            df_as_matrix = to_matrix
+            denominator  = for_sample_data ? rows - 1 : rows
+            ones         = Matrix.column_vector [1]*rows
+            deviation_scores = df_as_matrix - (ones * ones.transpose * df_as_matrix) / rows
+            ((deviation_scores.transpose * deviation_scores) / denominator).to_a
+          end
+
+          Daru::DataFrame.rows(cov_arry, index: numeric_vectors, order: numeric_vectors)
+        end
+
+        alias :cov :covariance
+          
+        # Calculate the correlation between the numeric vectors.
+        def correlation
+          corr_arry = 
+          if defined? NMatrix and NMatrix.respond_to?(:corr)
+            to_nmatrix.corr.to_a
+          else
+            standard_deviation = std.to_matrix
+            (cov.to_matrix.elementwise_division(standard_deviation.transpose * 
+              standard_deviation)).to_a
+          end
+
+          Daru::DataFrame.rows(corr_arry, index: numeric_vectors, order: numeric_vectors)
+        end
+
+        alias :corr :correlation
+
+       private
+
+        def compute_stats method
+          Daru::Vector.new(
+            numeric_vectors.inject({}) do |hash, vec|
+              hash[vec] = self[vec].send(method)
+              hash
+            end
+          )
+        end
       end
     end
   end

data/lib/daru/maths/statistics/vector.rb

@@ -1,47 +1,59 @@
 module Daru
   module Maths
+    # Encapsulates statistics methods for vectors. Most basic stuff like mean, etc.
+    #   is done inside the wrapper, so that native methods can be used for most of 
+    #   the computationally intensive tasks.
     module Statistics
       module Vector
-
         def mean
-          @vector.mean
+          @data.mean
         end
 
-        def median
-          @vector.median
+        def sum
+          @data.sum
         end
 
-        def mode
-          @vector.mode
+        def product
+          @data.product
         end
 
-        def sum
-          @vector.sum
+        def min
+          @data.min
         end
 
-        def product
-          @vector.product
+        def range
+          max - min
+        end
+
+        def median
+          percentile 50
+        end
+
+        def mode
+          freqs = frequencies.values
+          @data[freqs.index(freqs.max)]
         end
 
         def median_absolute_deviation
-          @vector.median_absolute_deviation  
+          m = median
+          map {|val| (val - m).abs }.median
         end
 
         def standard_error
-          @vector.standard_error
+          standard_deviation_sample/(Math::sqrt((@size - @nil_positions.size)))
         end
 
         def sum_of_squared_deviation
-          @vector.sum_of_squared_deviation
+          (@data.to_a.inject(0) { |a,x| x.square + a } - (sum.square.quo((@size - @nil_positions.size)))).to_f
         end
 
         # Maximum element of the vector.
         # 
         # @param return_type [Symbol] Data type of the returned value. Defaults
-        # to returning only the maximum number but passing *:vector* will return
-        # a Daru::Vector with the index of the corresponding maximum value.
+        #   to returning only the maximum number but passing *:vector* will return
+        #   a Daru::Vector with the index of the corresponding maximum value.
         def max return_type=:stored_type
-          max_value = @vector.max
+          max_value = @data.max
           if return_type == :vector
             Daru::Vector.new({index_of(max_value) => max_value}, name: @name, dtype: @dtype)
           else
@@ -49,98 +61,127 @@ module Daru
           end
         end
 
-        def min
-          @vector.min
-        end
-
-        def has_missing_data?
-          @vector.has_missing_data?
-        end
-
-        def range
-          @vector.range
+        # Return a Vector with the max element and its index.
+        # @return [Daru::Vector]
+        def max_index
+          max :vector
         end
 
         def frequencies
-          @vector.frequencies
+          @data.inject({}) do |hash, element|
+            hash[element] ||= 0
+            hash[element] += 1
+            hash
+          end
         end
 
         def proportions
-          @vector.proportions
+          len = n_valid
+          frequencies.inject({}) { |hash, arr| hash[arr[0]] = arr[1] / len; hash }
         end
 
         def ranked
-          @vector.ranked
+          sum = 0
+          r = frequencies.sort.inject( {} ) do |memo, val|
+            memo[val[0]] = ((sum + 1) + (sum + val[1])) / 2
+            sum += val[1]
+            memo
+          end
+
+          Daru::Vector.new @data.map { |e| r[e] }, index: self.index,
+            name: self.name, dtype: self.dtype, nm_dtype: self.nm_dtype
         end
 
         def coefficient_of_variation
-          @vector.coefficient_of_variation
+          standard_deviation_sample / mean
         end
 
-        # Retrieves number of cases which comply condition.
-        # If block given, retrieves number of instances where
-        # block returns true.
-        # If other values given, retrieves the frequency for
-        # this value.
+        # Retrieves number of cases which comply condition. If block given, 
+        # retrieves number of instances where block returns true. If other 
+        # values given, retrieves the frequency for this value. If no value
+        # given, counts the number of non-nil elements in the Vector.
         def count value=false
-          @vector.count value
+          if block_given?
+            @data.inject(0){ |memo, val| memo += 1 if yield val; memo}
+          elsif value
+            val = frequencies[value]
+            val.nil? ? 0 : val
+          else
+            size - @nil_positions.size
+          end
         end
 
         def proportion value=1
-          @vector.proportion value
-        end
-
-        # Population variance with denominator (N)
-        def variance_population m=nil
-          @vector.variance_population m
+          frequencies[value] / n_valid
         end
 
         # Sample variance with denominator (N-1)
         def variance_sample m=nil
-          @vector.variance_sample m
+          m ||= self.mean
+          sum_of_squares(m).quo((@size - @nil_positions.size) - 1)
         end
 
-        def sum_of_squares m=nil
-          @vector.sum_of_squares m
+        # Population variance with denominator (N)
+        def variance_population m=nil
+          m ||= mean
+          sum_of_squares(m).quo((@size - @nil_positions.size)).to_f
         end
 
-        def standard_deviation_sample m=nil
-          @vector.standard_deviation_sample m
+        def sum_of_squares(m=nil)
+          m ||= mean
+          @data.inject(0) { |memo, val| memo + (val - m)**2 }
         end
 
         def standard_deviation_population m=nil
-          @vector.standard_deviation_population m
+          m ||= mean
+          Math::sqrt(variance_population(m))
+        end
+
+        def standard_deviation_sample m=nil
+          Math::sqrt(variance_sample(m))
         end
 
         # Calculate skewness using (sigma(xi - mean)^3)/((N)*std_dev_sample^3)
         def skew m=nil
-          @vector.skew m
+          m ||= mean
+          th  = @data.inject(0) { |memo, val| memo + ((val - m)**3) }
+          th.quo ((@size - @nil_positions.size) * (standard_deviation_sample(m)**3))
         end
 
         def kurtosis m=nil
-          @vector.kurtosis m
+          m ||= mean
+          fo  = @data.inject(0){ |a, x| a + ((x - m) ** 4) }
+          fo.quo((@size - @nil_positions.size) * standard_deviation_sample(m) ** 4) - 3
         end
 
         def average_deviation_population m=nil
-          @vector.average_deviation_population m
+          m ||= mean
+          (@data.inject(0) {|memo, val| val + (val - m).abs }) / n_valid
         end
 
         def recode!(&block)
-          @vector.recode!(&block)
+          @data.recode!(&block)
         end
 
         def percentile percent
-          @vector.percentile percent
+          sorted = @data.sort
+          v      = (n_valid * percent).quo(100)
+          if v.to_i != v
+            sorted[v.round]
+          else
+            (sorted[(v - 0.5).round].to_f + sorted[(v + 0.5).round]).quo(2)
+          end
         end
 
-        alias_method :sdp, :standard_deviation_population
-        alias_method :sds, :standard_deviation_sample
-        alias_method :adp, :average_deviation_population
-        # alias_method :cov, :coefficient_of_variation
-        # alias_method :variance, :variance_sample    
-        alias_method :sd, :standard_deviation_sample
-        alias_method :ss, :sum_of_squares
-        alias_method :percentil, :percentile
+        alias :sdp :standard_deviation_population
+        alias :sds :standard_deviation_sample
+        alias :std :sds
+        alias :adp :average_deviation_population
+        alias :cov :coefficient_of_variation
+        alias :variance :variance_sample    
+        alias :sd :standard_deviation_sample
+        alias :ss :sum_of_squares
+        alias :percentil :percentile
       end
     end
   end