daru 0.1.6 → 0.2.0

data/lib/daru/plotting/nyaplot/dataframe.rb

@@ -21,6 +21,8 @@ module Daru
         #   df = Daru::DataFrame.new({a:['A', 'B', 'C', 'D', 'E'], b:[10,20,30,40,50]})
         #   df.plot type: :bar, x: :a, y: :b
         def plot opts={}, &block
+          index_as_default_x_axis(opts) unless x_axis_defined?(opts)
+
           if opts[:categorized]
             plot_with_category(opts, &block)
           else
@@ -30,6 +32,15 @@ module Daru
 
         private
 
+        def x_axis_defined?(opts)
+          opts[:x] || opts.keys.any? { |k| k.to_s.match(/x\d+/) }
+        end
+
+        def index_as_default_x_axis(opts)
+          opts[:x]      = :_index
+          self[:_index] = @index.to_a
+        end
+
         def plot_without_category opts
           options = {type: :scatter}.merge(opts)
 

data/lib/daru/vector.rb

@@ -228,7 +228,7 @@ module Daru
     end
 
     # Returns vector of values given positional values
-    # @param [Array<object>] *positions positional values
+    # @param positions [Array<object>] positional values
     # @return [object] vector
     # @example
     #   dv = Daru::Vector.new 'a'..'e'
@@ -252,7 +252,7 @@ module Daru
     end
 
     # Change value at given positions
-    # @param [Array<object>] *positions positional values
+    # @param positions [Array<object>] positional values
     # @param [object] val value to assign
     # @example
     #   dv = Daru::Vector.new 'a'..'e'
@@ -385,11 +385,11 @@ module Daru
     # comparator methods to obtain meaningful results. See this notebook for
     # a good overview of using #where.
     #
-    # @param [Daru::Core::Query::BoolArray, Array<TrueClass, FalseClass>] bool_arry The
+    # @param bool_array [Daru::Core::Query::BoolArray, Array<TrueClass, FalseClass>] The
     #   collection containing the true of false values. Each element in the Vector
     #   corresponding to a `true` in the bool_arry will be returned alongwith it's
     #   index.
-    # @exmaple Usage of #where.
+    # @example Usage of #where.
     #   vector = Daru::Vector.new([2,4,5,51,5,16,2,5,3,2,1,5,2,5,2,1,56,234,6,21])
     #
     #   # Simple logic statement passed to #where.
@@ -451,7 +451,7 @@ module Daru
     deprecate :flawed?, :include_values?, 2016, 10
 
     # Check if any one of mentioned values occur in the vector
-    # @param [Array] *values values to check for
+    # @param values  [Array] values to check for
     # @return [true, false] returns true if any one of specified values
     #   occur in the vector
     # @example
@@ -467,7 +467,7 @@ module Daru
     # Return vector of booleans with value at ith position is either
     # true or false depending upon whether value at position i is equal to
     # any of the values passed in the argument or not
-    # @param [Array] *values values to equate with
+    # @param values [Array] values to equate with
     # @return [Daru::Vector] vector of boolean values
     # @example
     #   dv = Daru::Vector.new [1, 2, 3, 2, 1]
@@ -760,6 +760,43 @@ module Daru
       self
     end
 
+    # Rolling fillna
+    # replace all Float::NAN and NIL values with the preceeding or following value
+    #
+    # @param direction [Symbol] (:forward, :backward) whether replacement value is preceeding or following
+    #
+    # @example
+    #  dv = Daru::Vector.new([1, 2, 1, 4, nil, Float::NAN, 3, nil, Float::NAN])
+    #
+    #   2.3.3 :068 > dv.rolling_fillna(:forward)
+    #   => #<Daru::Vector(9)>
+    #   0   1
+    #   1   2
+    #   2   1
+    #   3   4
+    #   4   4
+    #   5   4
+    #   6   3
+    #   7   3
+    #   8   3
+    #
+    def rolling_fillna!(direction=:forward)
+      enum = direction == :forward ? index : index.reverse_each
+      last_valid_value = 0
+      enum.each do |idx|
+        if valid_value?(self[idx])
+          last_valid_value = self[idx]
+        else
+          self[idx] = last_valid_value
+        end
+      end
+    end
+
+    # Non-destructive version of rolling_fillna!
+    def rolling_fillna(direction=:forward)
+      dup.rolling_fillna!(direction)
+    end
+
     # Lags the series by `k` periods.
     #
     # Lags the series by `k` periods, "shifting" data and inserting `nil`s
@@ -818,7 +855,7 @@ module Daru
     deprecate :n_valid, :count_values, 2016, 10
 
     # Count the number of values specified
-    # @param [Array] *values values to count for
+    # @param values [Array] values to count for
     # @return [Integer] the number of times the values mentioned occurs
     # @example
     #   dv = Daru::Vector.new [1, 2, 1, 2, 3, 4, nil, nil]
@@ -942,7 +979,7 @@ module Daru
     end
 
     # Create a summary of the Vector
-    # @params [Fixnum] indent_level
+    # @param indent_level [Fixnum] indent level
     # @return [String] String containing the summary of the Vector
     # @example
     #   dv = Daru::Vector.new [1, 2, 3]
@@ -1182,10 +1219,10 @@ module Daru
     #
     # == Arguments
     #
-    # @as_a [Symbol] Passing :array will return only the elements
+    # @param as_a [Symbol] Passing :array will return only the elements
     # as an Array. Otherwise will return a Daru::Vector.
     #
-    # @duplicate [Symbol] In case no missing data is found in the
+    # @param _duplicate [Symbol] In case no missing data is found in the
     # vector, setting this to false will return the same vector.
     # Otherwise, a duplicate will be returned irrespective of
     # presence of missing data.
@@ -1207,7 +1244,7 @@ module Daru
     deprecate :only_valid, :reject_values, 2016, 10
 
     # Return a vector with specified values removed
-    # @param [Array] *values values to reject from resultant vector
+    # @param values [Array] values to reject from resultant vector
     # @return [Daru::Vector] vector with specified values removed
     # @example
     #   dv = Daru::Vector.new [1, 2, nil, Float::NAN]
@@ -1229,7 +1266,7 @@ module Daru
     end
 
     # Return indexes of values specified
-    # @param [Array] *values values to find indexes for
+    # @param values [Array] values to find indexes for
     # @return [Array] array of indexes of values specified
     # @example
     #   dv = Daru::Vector.new [1, 2, nil, Float::NAN], index: 11..14
@@ -1437,6 +1474,11 @@ module Daru
         end
     end
 
+    # Helper method returning validity of arbitrary value
+    def valid_value?(v)
+      v.respond_to?(:nan?) && v.nan? || v.nil? ? false : true
+    end
+
     def initialize_vector source, opts
       index, source = parse_source(source, opts)
       set_name opts[:name]
@@ -1502,7 +1544,7 @@ module Daru
     end
 
     # Note: To maintain sanity, this _MUST_ be the _ONLY_ place in daru where the
-    # @dtype variable is set and the underlying data type of vector changed.
+    # @param dtype [db_type] variable is set and the underlying data type of vector changed.
     def cast_vector_to dtype, source=nil, nm_dtype=nil
       source = @data.to_a if source.nil?
 

data/lib/daru/version.rb

@@ -1,3 +1,3 @@
 module Daru
-  VERSION = '0.1.6'.freeze
+  VERSION = '0.2.0'.freeze
 end

data/profile/vector_new.rb

@@ -0,0 +1,9 @@
+require_relative '_base'
+
+n = 40_0000
+idx = (1..n).to_a.map(&:to_s)
+
+
+__profile__ do
+  Daru::Vector.new(1..n, index: idx)
+end

data/spec/category_spec.rb

@@ -1596,7 +1596,7 @@ describe Daru::DataFrame, "categorical" do
       Daru::DataFrame.new({
         a: [1, 2, 3, 4, 5],
         b: ['first', 'second', 'first', 'second', 'third'],
-        c: ['a', 'b', 'a', 'b', 'c']
+        c: ['a', 'b', 'a', 'b', nil]
       })
     end
     before { df.to_category :b, :c }
@@ -1605,6 +1605,10 @@ describe Daru::DataFrame, "categorical" do
     it { is_expected.to be_a Daru::DataFrame }
     its(:'b.type') { is_expected.to eq :category }
     its(:'c.type') { is_expected.to eq :category }
+    its(:'a.count') { is_expected.to eq 5 }
+    its(:'c.count') { is_expected.to eq 5 }
+    it { expect(df.c.count('a')).to eq 2 }
+    it { expect(df.c.count(nil)).to eq 1 }
   end
 
   context "#interact_code" do

data/spec/core/group_by_spec.rb

@@ -465,4 +465,132 @@ describe Daru::Core::GroupBy do
 
     it { is_expected.to eq Daru::DataFrame.new({num: [6]}, index: ['a']) }
   end
+
+  context 'when dataframe tuples contain nils in mismatching positions' do
+
+    let(:df){
+      Daru::DataFrame.new(
+        {
+          'string1' => ["Color", "Color", "Color", "Color", nil, "Color", "Color", " Black and White"],
+          'string2' => ["Test", "test2", nil, "test3", nil, "test", "test3", "test5"],
+          'num' => [1, nil, 3, 4, 5, 6, 7, nil]
+        }
+      )
+    }
+
+    it 'groups by without errors' do
+      expect { df.group_by(df.vectors.map(&:to_s)) }.to_not raise_error(ArgumentError)
+    end
+  end
+  
+  context '#aggregate' do
+    let(:dataframe) { Daru::DataFrame.new({
+      employee: %w[John Jane Mark John Jane Mark],
+      month: %w[June June June July July July],
+      salary: [1000, 500, 700, 1200, 600, 600]})
+    }
+    context 'group and aggregate sum for particular single vector' do
+      subject { dataframe.group_by([:employee]).aggregate(salary: :sum) }
+
+      it { is_expected.to eq Daru::DataFrame.new({
+              salary: [1100, 2200, 1300]},
+              index: ['Jane', 'John', 'Mark'])
+      }
+    end
+
+    context 'group and aggregate sum for two vectors' do
+      subject {
+        dataframe.group_by([:employee, :month]).aggregate(salary: :sum) }
+
+      it { is_expected.to eq Daru::DataFrame.new({
+              salary: [600, 500, 1200, 1000, 600, 700]},
+              index: Daru::MultiIndex.from_tuples([
+                  ['Jane', 'July'],
+                  ['Jane', 'June'],
+                  ['John', 'July'],
+                  ['John', 'June'],
+                  ['Mark', 'July'],
+                  ['Mark', 'June']
+                ])
+      )}
+    end
+
+    context 'group and aggregate sum and lambda function for vectors' do
+      subject { dataframe.group_by([:employee]).aggregate(
+        salary: :sum,
+        month: ->(vec) { vec.to_a.join('/') }) }
+
+      it { is_expected.to eq Daru::DataFrame.new({
+        salary: [1100, 2200, 1300],
+        month: ['June/July', 'June/July', 'June/July']},
+        index: ['Jane', 'John', 'Mark'],
+        order: [:salary, :month])
+      }
+    end
+
+    context 'group and aggregate sum and lambda functions on dataframe' do
+      subject { dataframe.group_by([:employee]).aggregate(
+        salary: :sum,
+        month: ->(vec) { vec.to_a.join('/') },
+        mean_salary: ->(df) { df.salary.mean },
+        periods: ->(df) { df.size }
+      )}
+
+      it { is_expected.to eq Daru::DataFrame.new({
+        salary: [1100, 2200, 1300],
+        month: ['June/July', 'June/July', 'June/July'],
+        mean_salary: [550.0, 1100.0, 650.0],
+        periods: [2, 2, 2]},
+        index: ['Jane', 'John', 'Mark'], order: [:salary, :month,
+                                                :mean_salary, :periods]) }
+    end
+
+    context 'group_by and aggregate on mixed MultiIndex' do
+      let(:df) { Daru::DataFrame.new(
+                    name: ['Ram','Krishna','Ram','Krishna','Krishna'],
+                    visited: [
+                      'Hyderabad', 'Delhi', 'Mumbai', 'Raipur', 'Banglore']
+                 )
+                }
+      let(:df_mixed) { Daru::DataFrame.new(
+                    name: ['Krishna','Ram','Krishna','Krishna'],
+                    visited: [
+                      'Delhi', 'Mumbai', 'Raipur', 'Banglore']
+                 )
+                }
+      it 'group_by' do
+        expect(df.group_by(:name).df).to eq(
+          Daru::DataFrame.new({
+            visited: ['Delhi', 'Raipur', 'Banglore', 'Hyderabad', 'Mumbai']},
+            index: Daru::MultiIndex.from_tuples(
+                [['Krishna', 1], ['Krishna', 3], ['Krishna', 4],
+                ['Ram', 0], ['Ram', 2]]
+            )
+          )
+        )
+      end
+
+      it 'group_by and aggregate' do
+        expect(
+          df.group_by(:name).aggregate(
+            visited: -> (vec){vec.to_a.join(',')})).to eq(
+              Daru::DataFrame.new({
+                visited: ['Delhi,Raipur,Banglore', 'Hyderabad,Mumbai']},
+                index: ['Krishna', 'Ram']
+              )
+          )
+      end
+
+      it 'group_by and aggregate when anyone index is not multiple times' do
+        expect(
+          df_mixed.group_by(:name).aggregate(
+            visited: -> (vec){vec.to_a.join(',')})).to eq(
+              Daru::DataFrame.new({
+                visited: ['Delhi,Raipur,Banglore', 'Mumbai']},
+                index: ['Krishna', 'Ram']
+              )
+          )
+      end
+    end
+  end
 end

data/spec/dataframe_spec.rb

@@ -145,8 +145,15 @@ describe Daru::DataFrame do
   end
 
   context "#initialize" do
+
+    it "initializes an empty DataFrame with no arguments" do
+      df = Daru::DataFrame.new
+      expect(df.nrows).to eq(0)
+      expect(df.ncols).to eq(0)
+    end
+
     context Daru::Index do
-      it "initializes an empty DataFrame" do
+      it "initializes an empty DataFrame with empty source arg" do
         df = Daru::DataFrame.new({}, order: [:a, :b])
 
         expect(df.vectors).to eq(Daru::Index.new [:a, :b])
@@ -164,6 +171,13 @@ describe Daru::DataFrame do
         expect(df.a)      .to eq([1,2,3,4,5].dv(:a, df.index))
       end
 
+      it "initializes from a Hash and preserves default order" do
+        df = Daru::DataFrame.new({b: [11,12,13,14,15], a: [1,2,3,4,5]},
+          index: [:one, :two, :three, :four, :five])
+
+        expect(df.vectors).to eq(Daru::Index.new [:b, :a])
+      end
+
       it "initializes from a Hash of Vectors" do
         va = Daru::Vector.new([1,2,3,4,5], index: [:one, :two, :three, :four, :five])
         vb = Daru::Vector.new([11,12,13,14,15], index: [:one, :two, :three, :four, :five])
@@ -221,7 +235,7 @@ describe Daru::DataFrame do
         df = Daru::DataFrame.new({b: [11,12,13,14,15], a: [1,2,3,4,5]})
 
         expect(df.index)  .to eq(Daru::Index.new [0,1,2,3,4])
-        expect(df.vectors).to eq(Daru::Index.new [:a, :b])
+        expect(df.vectors).to eq(Daru::Index.new [:b, :a])
       end
 
       it "aligns indexes properly" do
@@ -1573,6 +1587,22 @@ describe Daru::DataFrame do
       }
     end
 
+    context 'with mulitiindex DF' do
+      subject(:data_frame) {
+        Daru::DataFrame.new({b: [11,12,13], a: [1,2,3],
+          c: [11,22,33]}, order: [:a, :b, :c],
+          index: Daru::MultiIndex.from_tuples([[:one, :two], [:one, :three], [:two, :four]]))
+      }
+
+      before { data_frame.add_row [100,200,300], [:two, :five] }
+
+      it { is_expected.to eq(Daru::DataFrame.new({
+          b: [11,12,13,200], a: [1,2,3,100],
+          c: [11,22,33,300]}, order: [:a, :b, :c],
+          index: Daru::MultiIndex.from_tuples([[:one, :two], [:one, :three], [:two, :four], [:two, :five]])))
+      }
+    end
+
     it "allows adding rows after making empty DF by specfying only order" do
       df = Daru::DataFrame.new({}, order: [:a, :b, :c])
       df.add_row [1,2,3]
@@ -1790,6 +1820,59 @@ describe Daru::DataFrame do
     end
   end
 
+  describe 'uniq' do
+    let(:df) do
+      Daru::DataFrame.from_csv 'spec/fixtures/duplicates.csv'
+    end
+
+    context 'with no args' do
+      it do
+        result = df.uniq
+        expect(result.shape.first).to eq 30
+      end
+    end
+
+    context 'given a vector' do
+      it do
+        result = df.uniq("color")
+        expect(result.shape.first).to eq 2
+      end
+    end
+
+    context 'given an array of vectors' do
+      it do
+        result = df.uniq("color", "director_name")
+        expect(result.shape.first).to eq 29
+      end
+    end
+  end
+
+  context '#rolling_fillna!' do
+    subject do
+      Daru::DataFrame.new({
+        a: [1,    2,          3,   nil,        Float::NAN, nil, 1,   7],
+        b: [:a,  :b,          nil, Float::NAN, nil,        3,   5,   nil],
+        c: ['a',  Float::NAN, 3,   4,          3,          5,   nil, 7]
+      })
+    end
+
+    context 'rolling_fillna! forwards' do
+      before { subject.rolling_fillna!(:forward) }
+      it { is_expected.to be_a Daru::DataFrame }
+      its(:'a.to_a') { is_expected.to eq [1, 2, 3, 3, 3, 3, 1, 7] }
+      its(:'b.to_a') { is_expected.to eq [:a,  :b, :b, :b, :b, 3, 5, 5] }
+      its(:'c.to_a') { is_expected.to eq ['a', 'a', 3, 4, 3, 5, 5, 7] }
+    end
+
+    context 'rolling_fillna! backwards' do
+      before { subject.rolling_fillna!(:backward) }
+      it { is_expected.to be_a Daru::DataFrame }
+      its(:'a.to_a') { is_expected.to eq [1, 2, 3, 1, 1, 1, 1, 7] }
+      its(:'b.to_a') { is_expected.to eq [:a, :b, 3, 3, 3, 3, 5, 0] }
+      its(:'c.to_a') { is_expected.to eq ['a', 3, 3, 4, 3, 5, 7, 7] }
+    end
+  end
+
   context "#clone" do
     it "returns a view of the whole dataframe" do
       cloned = @data_frame.clone
@@ -3160,6 +3243,15 @@ describe Daru::DataFrame do
       it { expect(subject['a'].to_a).to eq [1, 2, 3] }
       it { expect(subject[:b].to_a).to eq [4, 5, 6] }
     end
+
+    context "preserves indices for dataframes with same index" do
+      let(:index) { ['one','two','three'] }
+      let(:df1) { Daru::DataFrame.new({ 'a' => [1, 2, 3], 'b' => [3, 4, 5] }, index: index) }
+      let(:df2) { Daru::DataFrame.new({ 'c' => [4, 5, 6], 'd' => [7, 8, 9] }, index: index) }
+      subject { df1.merge df2 }
+
+      its(:index) { is_expected.to eq Daru::Index.new(index) }
+    end
   end
 
   context "#vector_by_calculation" do
@@ -3176,23 +3268,27 @@ describe Daru::DataFrame do
 
   context "#vector_sum" do
     before do
-      a1 = Daru::Vector.new [1, 2, 3, 4, 5, nil]
-      a2 = Daru::Vector.new [10, 10, 20, 20, 20, 30]
-      b1 = Daru::Vector.new [nil, 1, 1, 1, 1, 2]
-      b2 = Daru::Vector.new [2, 2, 2, nil, 2, 3]
+      a1 = Daru::Vector.new [1, 2, 3, 4, 5, nil, nil]
+      a2 = Daru::Vector.new [10, 10, 20, 20, 20, 30, nil]
+      b1 = Daru::Vector.new [nil, 1, 1, 1, 1, 2, nil]
+      b2 = Daru::Vector.new [2, 2, 2, nil, 2, 3, nil]
       @df = Daru::DataFrame.new({ :a1 => a1, :a2 => a2, :b1 => b1, :b2 => b2 })
     end
 
     it "calculates complete vector sum" do
-      expect(@df.vector_sum).to eq(Daru::Vector.new [nil, 15, 26, nil, 28, nil])
+      expect(@df.vector_sum).to eq(Daru::Vector.new [nil, 15, 26, nil, 28, nil, nil])
+    end
+
+    it "ignores nils if skipnil is true" do
+      expect(@df.vector_sum skipnil: true).to eq(Daru::Vector.new [13, 15, 26, 25, 28, 35, 0])
     end
 
     it "calculates partial vector sum" do
       a = @df.vector_sum([:a1, :a2])
       b = @df.vector_sum([:b1, :b2])
 
-      expect(a).to eq(Daru::Vector.new [11, 12, 23, 24, 25, nil])
-      expect(b).to eq(Daru::Vector.new [nil, 3, 3, nil, 3, 5])
+      expect(a).to eq(Daru::Vector.new [11, 12, 23, 24, 25, nil, nil])
+      expect(b).to eq(Daru::Vector.new [nil, 3, 3, nil, 3, 5, nil])
     end
   end
 
@@ -3380,7 +3476,8 @@ describe Daru::DataFrame do
       ev_b  = Daru::Vector.new [1, 1, 0]
       ev_c  = Daru::Vector.new [0, 1, 1]
       df2 = Daru::DataFrame.new({
-        :_id => ev_id, 'a' => ev_a, 'b' => ev_b, 'c' => ev_c })
+        :_id => ev_id, 'a' => ev_a, 'b' => ev_b, 'c' => ev_c },
+        order: ['a', 'b', 'c', :_id])
 
       expect(df2).to eq(df)
     end
@@ -3928,6 +4025,24 @@ describe Daru::DataFrame do
     end
   end
 
+  context '#aggregate' do
+    let(:cat_idx) { Daru::CategoricalIndex.new [:a, :b, :a, :a, :c] }
+    let(:df) { Daru::DataFrame.new(num: [52,12,07,17,01], cat_index: cat_idx) }
+    let(:df_cat_idx) {
+      Daru::DataFrame.new({num: [52,12,07,17,01]}, index: cat_idx) }
+
+    it 'lambda function on particular column' do
+      expect(df.aggregate(num_100_times: ->(df) { df.num*100 })).to eq(
+          Daru::DataFrame.new(num_100_times: [5200, 1200, 700, 1700, 100])
+        )
+    end
+    it 'aggregate sum on particular column' do
+      expect(df_cat_idx.aggregate(num: :sum)).to eq(
+          Daru::DataFrame.new({num: [76, 12, 1]}, index: [:a, :b, :c])
+        )
+    end
+  end
+
   context '#create_sql' do
     let(:df) { Daru::DataFrame.new({
         a: [1,2,3],