daru 0.1.3.1 → 0.1.4

data/lib/daru/iruby/templates/vector.html.erb

@@ -0,0 +1,27 @@
+<table>
+  <tr>
+    <th colspan="2">Daru::Vector(<%= size %>)<%= ':category' if category? %></th>
+  </tr>
+  <% if name %>
+    <tr>
+        <th> </th>
+        <th><%= name %></th>
+    </tr>
+  <% end %>
+
+  <% @index.each_with_index.first(threshold).each do |index, pos| %>
+    <tr>
+      <td><%= index %></td>
+      <td><%= self.at(pos) %></td>
+    </tr>
+  <% end %>
+
+  <% if size > threshold %>
+    <% last_index = @index.size-1 %>
+    <tr><td>...</td><td>...</td></tr>
+    <tr>
+      <td><%= last_index %></td>
+      <td><%= self.at last_index %></td>
+    </tr>
+  <% end %>
+</table>

data/lib/daru/iruby/templates/vector_mi.html.erb

@@ -0,0 +1,36 @@
+<table>
+  <tr>
+    <th colspan="<%= index.width+1 %>">Daru::Vector(<%= size %>)<%= ':category' if category? %></th>
+  </tr>
+  <% if name %>
+    <tr>
+        <th colspan="<%= index.width %>"> </th>
+        <th><%= name %></th>
+    </tr>
+  <% end %>
+
+  <% Daru::IRuby::Helpers.tuples_with_rowspans(@index).first(threshold).zip(to_a).each do |tuple, value| %>
+    <tr>
+      <% tuple.each do |idx, span| %>
+        <th rowspan="<%= span %>"><%= idx %></th>
+      <% end %>
+      <td><%= value %></td>
+    </tr>
+  <% end %>
+
+  <% if size > threshold %>
+    <% last_index = @index.to_a.last %>
+    <tr>
+      <% last_index.size.times do %>
+        <th>...</th>
+      <% end %>
+      <td>...</td>
+    </tr>
+    <tr>
+      <% last_index.each do |idx| %>
+        <th><%= idx %></td>
+      <% end %>
+      <td><%= self[last_index] %></td>
+    </tr>
+  <% end %>
+</table>

data/lib/daru/maths/arithmetic/dataframe.rb

@@ -62,30 +62,28 @@ module Daru
           all_vectors = (vectors.to_a | other.vectors.to_a).sort
           all_indexes = (index.to_a   | other.index.to_a).sort
 
-          hsh = {}
-          all_vectors.each do |vector_name|
-            this = has_vector?(vector_name) ? self[vector_name] : nil
-            that = other.has_vector?(vector_name) ? other[vector_name] : nil
-
-            hsh[vector_name] =
-              if this && that
-                this.send(operation, that)
-              else
-                Daru::Vector.new([], index: all_indexes, name: vector_name)
-              end
-          end
+          hsh =
+            all_vectors.map do |vector_name|
+              vector = dataframe_binary_operation_on_vectors other, vector_name, operation, all_indexes
+
+              [vector_name, vector]
+            end.to_h
 
           Daru::DataFrame.new(hsh, index: all_indexes, name: @name, dtype: @dtype)
         end
 
-        def scalar_binary_operation operation, other
-          clone = dup
-          clone.map_vectors! do |vector|
-            vector = vector.send(operation, other) if vector.type == :numeric
-            vector
+        def dataframe_binary_operation_on_vectors other, name, operation, indexes
+          if has_vector?(name) && other.has_vector?(name)
+            self[name].send(operation, other[name])
+          else
+            Daru::Vector.new([], index: indexes, name: name)
           end
+        end
 
-          clone
+        def scalar_binary_operation operation, other
+          dup.map_vectors! do |vector|
+            vector.numeric? ? vector.send(operation, other) : vector
+          end
         end
       end
     end

data/lib/daru/maths/arithmetic/vector.rb

@@ -63,19 +63,17 @@ module Daru
         end
 
         def v2v_binary operation, other
-          common_idxs = []
-          elements    = []
+          # FIXME: why the sorting?.. - zverok, 2016-05-18
           index = (@index.to_a | other.index.to_a).sort
 
-          index.each do |idx|
+          elements = index.map do |idx|
             this = self.index.include?(idx) ? self[idx] : nil
             that = other.index.include?(idx) ? other[idx] : nil
 
-            elements << (this && that ? this.send(operation, that) : nil)
-            common_idxs << idx
+            this && that ? this.send(operation, that) : nil
           end
 
-          Daru::Vector.new(elements, name: @name, index: common_idxs)
+          Daru::Vector.new(elements, name: @name, index: index)
         end
       end
     end

data/lib/daru/maths/statistics/dataframe.rb

@@ -132,17 +132,15 @@ module Daru
 
         # Calculate sample variance-covariance between the numeric vectors.
         def covariance
-          cache={}
+          cache = Hash.new do |h, (col, row)|
+            h[[col, row]] = vector_cov(self[row],self[col])
+          end
           vectors = numeric_vectors
 
           mat_rows = vectors.collect do |row|
             vectors.collect do |col|
               if row == col
                 self[row].variance
-              elsif cache[[col,row]].nil?
-                cov = vector_cov(self[row],self[col])
-                cache[[row,col]] = cov
-                cov
               else
                 cache[[col,row]]
               end
@@ -170,16 +168,11 @@ module Daru
         private
 
         def apply_method_to_numerics method, *args
-          order = []
-          computed = @vectors.to_a.each_with_object([]) do |n, memo|
-            v = @data[@vectors[n]]
-            if v.type == :numeric
-              memo << v.send(method, *args)
-              order << n
-            end
-          end
+          numerics = @vectors.to_a.map { |n| [n, @data[@vectors[n]]] }
+                             .select { |_n, v| v.numeric? }
+          computed = numerics.map { |_n, v| v.send(method, *args) }
 
-          Daru::DataFrame.new(computed, index: @index, order: order,clone: false)
+          Daru::DataFrame.new(computed, index: @index, order: numerics.map(&:first), clone: false)
         end
 
         def vector_cov v1a, v2a
@@ -187,7 +180,7 @@ module Daru
         end
 
         def sum_of_squares v1, v2
-          v1a,v2a = v1.only_valid,v2.only_valid
+          v1a,v2a = v1.reject_values(*Daru::MISSING_VALUES),v2.reject_values(*Daru::MISSING_VALUES)
           v1a.reset_index!
           v2a.reset_index!
           m1 = v1a.mean

data/lib/daru/maths/statistics/vector.rb

@@ -4,7 +4,7 @@ module Daru
     # is done inside the wrapper, so that native methods can be used for most of
     # the computationally intensive tasks.
     module Statistics
-      module Vector
+      module Vector # rubocop:disable Metrics/ModuleLength
         def mean
           @data.mean
         end
@@ -55,16 +55,16 @@ module Daru
         alias :mad :median_absolute_deviation
 
         def standard_error
-          standard_deviation_sample/Math.sqrt(n_valid)
+          standard_deviation_sample/Math.sqrt(size - count_values(*Daru::MISSING_VALUES))
         end
 
         def sum_of_squared_deviation
-          (@data.inject(0) { |a,x| x.square + a } - sum.square.quo(n_valid).to_f).to_f
+          (@data.inject(0) { |a,x| x**2 + a } - (sum**2).quo(size - count_values(*Daru::MISSING_VALUES)).to_f).to_f
         end
 
         # Retrieve unique values of non-nil data
         def factors
-          only_valid.uniq.reset_index!
+          reject_values(*Daru::MISSING_VALUES).uniq.reset_index!
         end
 
         # Maximum element of the vector.
@@ -98,15 +98,17 @@ module Daru
         end
 
         def proportions
-          len = n_valid
-          frequencies.each_with_object({}) { |arr, hash| hash[arr[0]] = arr[1] / len }
+          len = size - count_values(*Daru::MISSING_VALUES)
+          frequencies.each_with_object({}) do |(el, count), hash|
+            hash[el] = count / len
+          end
         end
 
         def ranked
           sum = 0
-          r = frequencies.sort.each_with_object({}) do |val, memo|
-            memo[val[0]] = ((sum + 1) + (sum + val[1])).quo(2)
-            sum += val[1]
+          r = frequencies.sort.each_with_object({}) do |(el, count), memo|
+            memo[el] = ((sum + 1) + (sum + count)).quo(2)
+            sum += count
           end
 
           recode { |e| r[e] }
@@ -120,29 +122,27 @@ module Daru
         # retrieves number of instances where block returns true. If other
         # values given, retrieves the frequency for this value. If no value
         # given, counts the number of non-nil elements in the Vector.
-        def count value=false
+        def count value=false, &block
           if block_given?
-            @data.select { |val| yield(val) }.count
+            @data.select(&block).count
           elsif value
-            val = frequencies[value]
-            val.nil? ? 0 : val
+            count { |val| val == value }
           else
-            size - @missing_positions.size
+            size - indexes(*Daru::MISSING_VALUES).size
           end
         end
 
         # Count number of occurrences of each value in the Vector
         def value_counts
-          values = {}
-          @data.each do |d|
-            values[d] ? values[d] += 1 : values[d] = 1
+          values = @data.each_with_object(Hash.new(0)) do |d, memo|
+            memo[d] += 1
           end
 
           Daru::Vector.new(values)
         end
 
         def proportion value=1
-          frequencies[value].quo(n_valid).to_f
+          frequencies[value].quo(size - count_values(*Daru::MISSING_VALUES)).to_f
         end
 
         # Sample variance with denominator (N-1)
@@ -151,7 +151,7 @@ module Daru
           if @data.respond_to? :variance_sample
             @data.variance_sample m
           else
-            sum_of_squares(m).quo(n_valid - 1)
+            sum_of_squares(m).quo(size - count_values(*Daru::MISSING_VALUES) - 1)
           end
         end
 
@@ -161,38 +161,26 @@ module Daru
           if @data.respond_to? :variance_population
             @data.variance_population m
           else
-            sum_of_squares(m).quo(n_valid).to_f
+            sum_of_squares(m).quo(size - count_values(*Daru::MISSING_VALUES)).to_f
           end
         end
 
         # Sample covariance with denominator (N-1)
         def covariance_sample other
-          @size == other.size or raise ArgumentError, 'size of both the vectors must be equal'
-          mean_x = mean
-          mean_y = other.mean
-          sum = 0
-          (0...size).each do |i|
-            sum += ((@missing_values.key?(@data[i]) || other.missing_values.include?(other[i])) ? 0 : (@data[i] - mean_x) * (other.data[i] - mean_y))
-          end
-          sum / (n_valid - 1)
+          size == other.size or raise ArgumentError, 'size of both the vectors must be equal'
+          covariance_sum(other) / (size - count_values(*Daru::MISSING_VALUES) - 1)
         end
 
         # Population covariance with denominator (N)
         def covariance_population other
-          @size == other.size or raise ArgumentError, 'size of both the vectors must be equal'
-          mean_x = mean
-          mean_y = other.mean
-          sum = 0
-          (0...size).each do |i|
-            sum += ((@missing_values.key?(@data[i]) || other.missing_values.include?(other[i])) ? 0 : (@data[i] - mean_x) * (other.data[i] - mean_y))
-          end
-          sum / n_valid
+          size == other.size or raise ArgumentError, 'size of both the vectors must be equal'
+          covariance_sum(other) / (size - count_values(*Daru::MISSING_VALUES))
         end
 
         def sum_of_squares(m=nil)
           m ||= mean
-          @data.inject(0) { |memo, val|
-            @missing_values.key?(val) ? memo : (memo + (val - m)**2)
+          reject_values(*Daru::MISSING_VALUES).data.inject(0) { |memo, val|
+            memo + (val - m)**2
           }
         end
 
@@ -221,7 +209,7 @@ module Daru
           else
             m ||= mean
             th  = @data.inject(0) { |memo, val| memo + ((val - m)**3) }
-            th.quo((@size - @missing_positions.size) * (standard_deviation_sample(m)**3))
+            th.quo((size - indexes(*Daru::MISSING_VALUES).size) * (standard_deviation_sample(m)**3))
           end
         end
 
@@ -231,16 +219,16 @@ module Daru
           else
             m ||= mean
             fo  = @data.inject(0) { |a, x| a + ((x - m) ** 4) }
-            fo.quo((@size - @missing_positions.size) * standard_deviation_sample(m) ** 4) - 3
+            fo.quo((size - indexes(*Daru::MISSING_VALUES).size) * standard_deviation_sample(m) ** 4) - 3
           end
         end
 
         def average_deviation_population m=nil
-          type == :numeric or raise TypeError, 'Vector must be numeric'
+          must_be_numeric!
           m ||= mean
-          (@data.inject(0) { |memo, val|
-            @missing_values.key?(val) ? memo : (val - m).abs + memo
-          }).quo(n_valid)
+          reject_values(*Daru::MISSING_VALUES).data.inject(0) { |memo, val|
+            (val - m).abs + memo
+          }.quo(size - count_values(*Daru::MISSING_VALUES))
         end
 
         # Returns the value of the percentile q
@@ -254,31 +242,13 @@ module Daru
         #
         # This is the NIST recommended method (http://en.wikipedia.org/wiki/Percentile#NIST_method)
         def percentile(q, strategy=:midpoint)
-          sorted = only_valid(:array).sort
-
           case strategy
           when :midpoint
-            v = (n_valid * q).quo(100)
-            if v.to_i!=v
-              sorted[v.to_i]
-            else
-              (sorted[(v-0.5).to_i].to_f + sorted[(v+0.5).to_i]).quo(2)
-            end
+            midpoint_percentile(q)
           when :linear
-            index = (q / 100.0) * (n_valid + 1)
-
-            k = index.truncate
-            d = index % 1
-
-            if k == 0
-              sorted[0]
-            elsif k >= sorted.size
-              sorted[-1]
-            else
-              sorted[k - 1] + d * (sorted[k] - sorted[k - 1])
-            end
+            linear_percentile(q)
           else
-            raise NotImplementedError, "Unknown strategy #{strategy}"
+            raise ArgumentError, "Unknown strategy #{strategy}"
           end
         end
 
@@ -313,17 +283,18 @@ module Daru
         def standardize use_population=false
           m ||= mean
           sd = use_population ? sdp : sds
-          return Daru::Vector.new([nil]*@size) if m.nil? || sd == 0.0
+          return Daru::Vector.new([nil]*size) if m.nil? || sd == 0.0
 
           vector_standardized_compute m, sd
         end
 
+        # :nocov:
         def box_cox_transformation lambda # :nodoc:
-          raise 'Should be a numeric' unless @type == :numeric
+          must_be_numeric!
 
           recode do |x|
             if !x.nil?
-              if lambda == 0
+              if lambda.zero?
                 Math.log(x)
               else
                 (x ** lambda - 1).quo(lambda)
@@ -333,10 +304,11 @@ module Daru
             end
           end
         end
+        # :nocov:
 
         # Replace each non-nil value in the vector with its percentile.
         def vector_percentile
-          c = size - missing_positions.size
+          c = size - indexes(*Daru::MISSING_VALUES).size
           ranked.recode! { |i| i.nil? ? nil : (i.quo(c)*100).to_f }
         end
 
@@ -367,7 +339,7 @@ module Daru
           if @data.respond_to? :sample_with_replacement
             @data.sample_with_replacement sample
           else
-            valid = missing_positions.empty? ? self : only_valid
+            valid = indexes(*Daru::MISSING_VALUES).empty? ? self : reject_values(*Daru::MISSING_VALUES)
             vds = valid.size
             (0...sample).collect { valid[rand(vds)] }
           end
@@ -383,17 +355,7 @@ module Daru
           if @data.respond_to? :sample_without_replacement
             @data.sample_without_replacement sample
           else
-            valid = missing_positions.empty? ? self : only_valid
-            raise ArgumentError, "Sample size couldn't be greater than n" if
-              sample > valid.size
-            out  = []
-            size = valid.size
-            while out.size < sample
-              value = rand(size)
-              out.push(value) unless out.include?(value)
-            end
-
-            out.collect { |i| valid[i] }
+            raw_sample_without_replacement(sample)
           end
         end
 
@@ -407,7 +369,7 @@ module Daru
         #   vector = Daru::Vector.new([4,6,6,8,10],index: ['a','f','t','i','k'])
         #   vector.percent_change
         #   #=>
-        #   #   <Daru::Vector:28713060 @name = nil @size: 5 >
+        #   #   <Daru::Vector:28713060 @name = nil size: 5 >
         #   #              nil
         #   #   a
         #   #   f	   0.5
@@ -415,19 +377,19 @@ module Daru
         #   #   i	   0.3333333333333333
         #   #   k          0.25
         def percent_change periods=1
-          type == :numeric or raise TypeError, 'Vector must be numeric'
-          value = only_valid
-          arr = []
-          i = 1
-          ind = @data.find_index { |x| !x.nil? }
-          (periods...size).each do |j|
-            if j==ind || @missing_values.key?(@data[j])
-              arr[j] = nil
+          must_be_numeric!
+
+          prev = nil
+          arr = @data.each_with_index.map do |cur, i|
+            if i < periods ||
+               include_with_nan?(Daru::MISSING_VALUES, cur) ||
+               include_with_nan?(Daru::MISSING_VALUES, prev)
+              nil
             else
-              arr[j] = (value.data[i] - value.data[i - 1]) / value.data[i - 1].to_f
-              i+=1
-            end
+              (cur - prev) / prev.to_f
+            end.tap { prev = cur if cur }
           end
+
           Daru::Vector.new(arr, index: @index, name: @name)
         end
 
@@ -533,7 +495,7 @@ module Daru
         #   ts.ema   # => [ ... nil, 0.455... , 0.395..., 0.323..., ... ]
         #
         # @return [Daru::Vector] Contains EMA
-        def ema(n=10, wilder=false)
+        def ema(n=10, wilder=false) # rubocop:disable Metrics/AbcSize
           smoother = wilder ? 1.0 / n : 2.0 / (n + 1)
           # need to start everything from the first non-nil observation
           start = @data.index { |i| !i.nil? }
@@ -567,7 +529,7 @@ module Daru
         #   ts.emv   # => [ ... nil, 0.073... , 0.082..., 0.080..., ...]
         #
         # @return [Daru::Vector] contains EMV
-        def emv(n=10, wilder=false)
+        def emv(n=10, wilder=false) # rubocop:disable Metrics/AbcSize
           smoother = wilder ? 1.0 / n : 2.0 / (n + 1)
           # need to start everything from the first non-nil observation
           start = @data.index { |i| !i.nil? }
@@ -651,7 +613,7 @@ module Daru
           max_lags ||= (10 * Math.log10(size)).to_i
 
           (0..max_lags).map do |i|
-            if i == 0
+            if i.zero?
               1.0
             else
               m = mean
@@ -672,7 +634,7 @@ module Daru
         # == Returns
         #
         # Autocovariance value
-        def acvf(demean=true, unbiased=true)
+        def acvf(demean=true, unbiased=true) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
           opts = {
             demean: true,
             unbaised: true
@@ -700,7 +662,7 @@ module Daru
           result = []
           acc = 0
           @data.each do |d|
-            if @missing_values.key?(d)
+            if include_with_nan? Daru::MISSING_VALUES, d
               result << nil
             else
               acc += d
@@ -722,6 +684,66 @@ module Daru
         alias :ss :sum_of_squares
         alias :percentil :percentile
         alias :se :standard_error
+
+        private
+
+        def must_be_numeric!
+          numeric? or raise TypeError, 'Vector must be numeric'
+        end
+
+        def covariance_sum other
+          self_mean = mean
+          other_mean = other.mean
+          @data
+            .zip(other.data).inject(0) do |res, (d, o)|
+              res + if !d || !o
+                      0
+                    else
+                      (d - self_mean) * (o - other_mean)
+                    end
+            end
+        end
+
+        def midpoint_percentile(q) # rubocop:disable Metrics/AbcSize
+          sorted = reject_values(*Daru::MISSING_VALUES).to_a.sort
+
+          v = ((size - count_values(*Daru::MISSING_VALUES)) * q).quo(100)
+          if v.to_i!=v
+            sorted[v.to_i]
+          else
+            (sorted[(v-0.5).to_i].to_f + sorted[(v+0.5).to_i]).quo(2)
+          end
+        end
+
+        def linear_percentile(q) # rubocop:disable Metrics/AbcSize
+          sorted = reject_values(*Daru::MISSING_VALUES).to_a.sort
+          index = (q / 100.0) * ((size - count_values(*Daru::MISSING_VALUES)) + 1)
+
+          k = index.truncate
+          d = index % 1
+
+          if k.zero?
+            sorted[0]
+          elsif k >= sorted.size
+            sorted[-1]
+          else
+            sorted[k - 1] + d * (sorted[k] - sorted[k - 1])
+          end
+        end
+
+        def raw_sample_without_replacement sample
+          valid = indexes(*Daru::MISSING_VALUES).empty? ? self : reject_values(*Daru::MISSING_VALUES)
+          raise ArgumentError, "Sample size couldn't be greater than n" if
+            sample > valid.size
+          out  = []
+          size = valid.size
+          while out.size < sample
+            value = rand(size)
+            out.push(value) unless out.include?(value)
+          end
+
+          out.collect { |i| valid[i] }
+        end
       end
     end
   end