daru 0.0.4 → 0.0.5

data/daru.gemspec

@@ -4,14 +4,12 @@ $:.unshift File.expand_path("../lib", __FILE__)
 require 'version.rb'
 
 DESCRIPTION = <<MSG
-Daru (Data Analysis in RUby) is a library for storage, analysis and manipulation
+Daru (Data Analysis in RUby) is a library for analysis, manipulation and visualization
 of data.
 
-Daru works with Ruby arrays, NMatrix and MDArray, thus working seamlessly accross
-ruby interpreters, at the same time providing speed for those who need it.
-
-This library is under active development so NMatrix and MDArray support is
-somewhat limited, but should be available soon!
+Daru works with Ruby arrays and NMatrix, thus working seamlessly accross
+ruby interpreters, at the same time providing speed for those who need it, while 
+making working with data super simple and intuitive.
 MSG
 
 Gem::Specification.new do |spec|
@@ -35,6 +33,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'awesome_print'
   spec.add_development_dependency 'nyaplot'
   if RUBY_ENGINE != 'jruby'
-    spec.add_development_dependency 'nmatrix', '~> 0.1.0.rc5'
+    spec.add_development_dependency 'nmatrix', '~> 0.1.0'
   end
 end

data/lib/daru.rb

@@ -1,7 +1,15 @@
-require 'securerandom'
+def jruby?
+  RUBY_ENGINE == 'jruby'
+end
+
 require 'csv'
+require 'matrix'
+require 'securerandom'
 
 require 'daru/index.rb'
+require 'daru/multi_index.rb'
 require 'daru/vector.rb'
 require 'daru/dataframe.rb'
-require 'daru/monkeys.rb'
+require 'daru/monkeys.rb'
+
+require 'daru/core/group_by.rb'

data/lib/daru/accessors/array_wrapper.rb

@@ -2,254 +2,92 @@ module Daru
   module Accessors
     # Internal class for wrapping ruby array
     class ArrayWrapper
-      module Statistics
-
-        def average_deviation_population m=nil
-          m ||= mean
-          (@vector.inject(0) {|memo, val| val + (val - m).abs }) / n_valid
-        end
-
-        def coefficient_of_variation
-          standard_deviation_sample / mean
-        end
-
-        def count value=false
-          if block_given?
-            @vector.inject(0){ |memo, val| memo += 1 if yield val; memo}
-          else
-            val = frequencies[value]
-            val.nil? ? 0 : val
-          end
-        end
-
-        def factors
-          index = @data.sorted_indices
-          index.reduce([]){|memo, val| memo.push(@data[val]) if memo.last != @data[val]; memo}
-        end # TODO
-
-        def frequencies
-          @vector.inject({}) do |hash, element|
-            hash[element] ||= 0
-            hash[element] += 1
-            hash
-          end
-        end
-
-        def has_missing_data?
-          has_missing_data
-        end
-
-        def kurtosis m=nil
-          m ||= mean
-          fo  = @vector.inject(0){ |a, x| a + ((x - m) ** 4) }
-          fo.quo(@size * standard_deviation_sample(m) ** 4) - 3
-        end
-
-        def mean
-          sum.quo(@size).to_f
-        end
-
-        def median
-          percentile 50
-        end
-
-        def median_absolute_deviation
-          m = median
-          recode {|val| (val - m).abs }.median
-        end
-
-        def mode
-          freqs = frequencies.values
-
-          @vector[freqs.index(freqs.max)]
-        end
-
-        def n_valid
-          @size
-        end
-
-        def percentile percent
-          sorted = @vector.sort
-          v      = (n_valid * percent).quo(100)
-          if v.to_i != v
-            sorted[v.round]
-          else
-            (sorted[(v - 0.5).round].to_f + sorted[(v + 0.5).round]).quo(2)
-          end
-        end
-
-        def product
-          @vector.inject(:*)
-        end
-
-        def max
-          @vector.max
-        end
-
-        def min
-          @vector.min
-        end
-
-        def proportion value=1
-          frequencies[value] / n_valid
-        end
-
-        def proportions
-          len = n_valid
-          frequencies.inject({}) { |hash, arr| hash[arr[0]] = arr[1] / len; hash }
-        end
-
-        def range
-          max - min
-        end
-
-        def ranked
-          sum = 0
-          r = frequencies.sort.inject( {} ) do |memo, val|
-            memo[val[0]] = ((sum + 1) + (sum + val[1])) / 2
-            sum += val[1]
-            memo
-          end
-
-          Daru::Vector.new @vector.map { |e| r[e] }, index: @caller.index,
-            name: @caller.name, dtype: @caller.dtype
-        end
-
-        def recode(&block)
-          @vector.map(&block)
-        end
-
-        def recode!(&block)
-          @vector.map!(&block)
-        end
-
-        # Calculate skewness using (sigma(xi - mean)^3)/((N)*std_dev_sample^3)
-        def skew m=nil
-          m ||= mean
-          th  = @vector.inject(0) { |memo, val| memo + ((val - m)**3) }
-          th.quo (@size * (standard_deviation_sample(m)**3))
-        end
-
-        def standard_deviation_population m=nil
-          m ||= mean
-          Math::sqrt(variance_population(m))
-        end
-
-        def standard_deviation_sample m=nil
-          Math::sqrt(variance_sample(m))
-        end
-
-        def standard_error
-          standard_deviation_sample/(Math::sqrt(@size))
-        end
-
-        def sum_of_squared_deviation
-          (@vector.inject(0) { |a,x| x.square + a } - (sum.square.quo(@size))).to_f
-        end
-
-        def sum_of_squares(m=nil)
-          m ||= mean
-          @vector.inject(0) { |memo, val| memo + (val - m)**2 }
-        end
-
-        def sum
-          @vector.inject(:+)
-        end
-
-        # Sample variance with denominator (N-1)
-        def variance_sample m=nil
-          m ||= self.mean
-
-          sum_of_squares(m).quo(@size - 1)
-        end
-
-        # Population variance with denominator (N)
-        def variance_population m=nil
-          m ||= mean
-
-          sum_of_squares(m).quo(@size).to_f
-        end
-      end # module Statistics
-
-      include Statistics
       include Enumerable
 
       def each(&block)
-        @vector.each(&block)
+        @data.each(&block)
       end
 
       def map!(&block)
-        @vector.map!(&block)
+        @data.map!(&block)
       end
 
       attr_accessor :size
-      attr_reader   :vector
-      attr_reader   :has_missing_data
+      attr_reader   :data
 
-      def initialize vector, caller
-        @vector = vector
-        @caller = caller
+      def initialize vector, context
+        @data = vector.to_a
+        @context = context
 
         set_size
       end
 
       def [] index
-        @vector[index]
+        @data[index]
       end
 
       def []= index, value
-        has_missing_data = true if value.nil?
-        @vector[index] = value
+        @data[index] = value
         set_size
       end
 
       def == other
-        @vector == other
+        @data == other
       end
 
       def delete_at index
-        @vector.delete_at index
+        @data.delete_at index
         set_size
       end
 
       def index key
-        @vector.index key
+        @data.index key
       end
 
       def << element
-        @vector << element
+        @data << element
         set_size
       end
 
       def uniq
-        @vector.uniq
+        @data.uniq
       end
 
       def to_a
-        @vector
+        @data
       end
 
       def dup
-        ArrayWrapper.new @vector.dup, @caller
+        ArrayWrapper.new @data.dup, @context
+      end
+
+      def mean
+        sum.quo(@size - @context.nil_positions.size).to_f
+      end
+
+      def product
+        @data.inject(1) { |m,e| m*e unless e.nil? }
+      end
+
+      def max
+        @data.max
+      end
+
+      def min
+        @data.min
       end
 
-      def coerce dtype
-        case
-        when dtype == Array
-          self
-        when dtype == NMatrix
-          Daru::Accessors::NMatrixWrapper.new @vector, @caller
-        when dtype == MDArray
-          raise NotImplementedError
-        else
-          raise ArgumentError, "Cant coerce to dtype #{dtype}"
+      def sum
+        @data.inject(0) do |memo ,e|
+          memo += e unless e.nil? #TODO: Remove this conditional somehow!
+          memo
         end
       end
 
      private
 
       def set_size
-        @size = @vector.size
+        @size = @data.size
       end
     end
   end

data/lib/daru/accessors/nmatrix_wrapper.rb

@@ -1,260 +1,111 @@
-require 'nmatrix'
+begin
+  require 'nmatrix' unless jruby?
+rescue LoadError => e
+  puts "Please install the nmatrix gem for fast and efficient data storage."
+end
 
 module Daru
   module Accessors
-
     # Internal class for wrapping NMatrix
     class NMatrixWrapper
-      module Statistics
-        # def average_deviation_population m=nil
-        #   m ||= self.mean
-        #   (self.reduce(0){|memo, val| val + (val - m).abs})/self.length
-        # end
-
-        # def coefficient_of_variation
-        #   self.standard_deviation_sample/self.mean
-        # end
-
-        # def count x=false
-        #   if block_given?
-        #     self.reduce(0){|memo, val| memo += 1 if yield val; memo}
-        #   else
-        #     val = self.frequencies[x]
-        #     val.nil? ? 0 : val
-        #   end
-        # end
-
-        # def factors
-        #   index = @data.sorted_indices
-        #   index.reduce([]){|memo, val| memo.push(@data[val]) if memo.last != @data[val]; memo}
-        # end
-
-        # def frequencies
-        #   index = @data.sorted_indices
-        #   index.reduce({}){|memo, val| memo[@data[val]] ||= 0; memo[@data[val]] += 1; memo}
-        # end
-
-        # def has_missing_data?
-        #   @missing_data
-        # end
-
-        # def is_valid?
-        #   true
-        # end
-
-        # def kurtosis(m=nil)
-        #   m ||= self.mean
-        #   fo=self.reduce(0){|a, x| a+((x-m)**4)}
-        #   fo.quo(self.length*sd(m)**4)-3
-        # end
-
-        # def mean
-        #   @vector[0...@size].mean.first
-        # end
-
-        # def median
-        #   self.percentil(50)
-        # end
-
-        # def median_absolute_deviation
-        #   m = self.median
-        #   self.recode{|val| (val-m).abls}.median
-        # end
-
-        # def mode
-        #   self.frequencies.max
-        # end
-
-        # def ==(other)
-        #   @data==other
-        # end
-
-        # def n_valid
-        #   self.length
-        # end
-
-        # def percentil(percent)
-        #   index = @data.sorted_indices
-        #   pos = (self.length * percent)/100
-        #   if pos.to_i == pos
-        #     @data[index[pos.to_i]]
-        #   else
-        #     pos = (pos-0.5).to_i
-        #     (@data[index[pos]] + @data[index[pos+1]])/2
-        #   end
-        # end
-
-        # def product
-        #   @data.inject(1){|memo, val| memo*val}
-        # end
-
-        # def proportion(val=1)
-        #   self.frequencies[val]/self.n_valid
-        # end
-
-        # def proportion_confidence_interval_t
-        #   raise "NotImplementedError"
-        # end
-
-        # def proportion_confidence_interval_z
-        #   raise "NotImplementedError"
-        # end
-
-        # def proportions
-        #   len = self.n_valid
-        #   self.frequencies.reduce({}){|memo, arr| memo[arr[0]] = arr[1]/len}
-        # end
-
-        # def push(val)
-        #   self.expand(self.length+1)
-        #   self[self.length-1] = recode
-        # end
-
-        # def range
-        #   max - min
-        # end
-
-        # def ranked
-        #   sum = 0
-        #   r = self.frequencies.sort.reduce({}) do |memo, val|
-        #     memo[val[0]] = ((sum+1) + (sum+val[1]))/2
-        #     sum += val[1]
-        #     memo
-        #   end
-        #   Mikon::DArray.new(self.reduce{|val| r[val]})
-        # end
-
-        # def recode(&block)
-        #   Mikon::DArray.new(@data.map(&block))
-        # end
-
-        # def recode!(&block)
-        #   @data.map!(&block)
-        # end
-
-        # def skew(m=nil)
-        #   m ||= self.mean
-        #   th = self.reduce(0){|memo, val| memo + ((val - m)**3)}
-        #   th/((self.length)*self.sd(m)**3)
-        # end
-
-        # def standard_deviation_population(m=nil)
-        #   m ||= self.mean
-        #   Maths.sqrt(self.variance_population(m))
-        # end
-
-        # def standard_deviation_sample(m=nil)
-        #   if !m.nil?
-        #     Maths.sqrt(variance_sample(m))
-        #   else
-        #     @data.std.first
-        #   end
-        # end
-
-        # def standard_error
-        #   self.standard_deviation_sample/(Maths.sqrt(self.length))
-        # end
-
-        # def sum_of_squared_deviation
-        #   self.reduce(0){|memo, val| val**2 + memo}
-        # end
-
-        # def sum_of_squares(m=nil)
-        #   m ||= self.mean
-        #   self.reduce(0){|memo, val| memo + (val-m)**2}
-        # end
+      include Enumerable
 
-        # def sum
-        #   @data.sum.first
-        # end
+      def each(&block)
+        @data[0...@size].each(&block)
+      end
 
-        # def variance_sample(m=nil)
-        #   m ||= self.mean
-        #   self.sum_of_squares(m)/(self.length-1)
-        # end
-      end # module Statistics
+      def map(&block)
+        @data[0...@size].map(&block)
+      end
 
-      include Statistics
-      include Enumerable
+      def map!(&block)
+        @data = NMatrix.new [@size*2], map(&block).to_a, dtype: nm_dtype
+      end
 
-      def each(&block)
-        @vector.each(&block)
+      def inject(*args, &block)
+        @data[0...@size].inject(*args, &block)
       end
 
-      attr_reader :size, :vector, :missing_data
+      alias_method :recode, :map
+      alias_method :recode!, :map!
 
-      def initialize vector, caller
+      attr_reader :size, :data, :nm_dtype
+      
+      def initialize vector, context, nm_dtype=:int32
         @size = vector.size
-        @vector = NMatrix.new [@size*2], vector.to_a
-        @missing_data = false
-        @caller = caller
+        @data = NMatrix.new [@size*2], vector.to_a, dtype: nm_dtype
+        @context = context
+        @nm_dtype = @data.dtype
         # init with twice the storage for reducing the need to resize
       end
 
       def [] index
-        @vector[index]
+        return @data[index] if index < @size
+        nil
       end
  
       def []= index, value
-        resize if index >= @size
-
-        if value.nil?
-          @missing_data = true
-          @vector = @vector.cast(dtype: :object)
-        end
-        @vector[index] = value
+        resize     if index >= @data.size
+        @size += 1 if index == @size
+        
+        @data = @data.cast(dtype: :object) if value.nil?
+        @data[index] = value
       end 
  
       def == other
-        @vector == other and @size == other.size
+        @data == other and @size == other.size
       end
  
       def delete_at index
-        arry = @vector.to_a
+        arry = @data.to_a
         arry.delete_at index
-        @vector = NMatrix.new [@size-1], arry
+        @data = NMatrix.new [(2*@size-1)], arry, dtype: @nm_dtype
         @size -= 1
       end
  
       def index key
-        @vector.to_a.index key
+        @data.to_a.index key
       end
  
       def << element
-        if @size >= @vector.size
-          resize
-        end
-
+        resize if @size >= @data.size
         self[@size] = element
 
         @size += 1
       end
  
       def to_a
-        @vector.to_a
+        @data[0...@size].to_a
       end
  
       def dup
-        NMatrixWrapper.new @vector.to_a
+        NMatrixWrapper.new @data.to_a, @context, @nm_dtype
       end
 
-      def coerce dtype
-        case 
-        when dtype == Array
-          Daru::Accessors::ArrayWrapper.new @vector[0..(@size-1)].to_a, @caller
-        when dtype == NMatrix
-          self
-        when dtype == MDArray
-          raise NotImplementedError
-        else
-          raise ArgumentError, "Cant coerce to dtype #{dtype}"
-        end
+      def resize size = @size*2
+        raise ArgumentError, "Size must be greater than current size" if size < @size
+
+        @data = NMatrix.new [size], @data.to_a
       end
 
-      def resize size = @size*2
-        raise "Size must be greater than current size" if size < @size
+      def mean
+        @data[0...@size].mean.first
+      end
+
+      def product
+        @data[0...@size].inject(1) { |m,e| m*e }
+      end
+
+      def sum
+        @data[0...@size].inject(:+)
+      end
+
+      def max
+        @data.max
+      end
 
-        @vector = NMatrix.new [size], @vector.to_a
+      def min
+        @data.min
       end
     end
   end