daru 0.1.2 → 0.1.3

data/lib/daru/version.rb

@@ -1,3 +1,3 @@
 module Daru
-  VERSION = "0.1.2"
-end
+  VERSION = '0.1.3'.freeze
+end

data/spec/core/group_by_spec.rb

@@ -41,6 +41,10 @@ describe Daru::Core::GroupBy do
     it 'groups by nil values' do
       expect(@df.group_by(:w_nils).groups[[nil]]).to eq([1,3,4])
     end
+
+    it "uses a multi-index when nils are part of the grouping keys" do
+      expect(@df.group_by(:a, :w_nils).send(:multi_indexed_grouping?)).to be true
+    end
   end
 
   context "#initialize" do
@@ -85,6 +89,11 @@ describe Daru::Core::GroupBy do
     it "returns a vector containing the size of each group" do
       expect(@dl_group.size).to eq(Daru::Vector.new([1,1,1,2,1,2], index: @dl_multi_index))
     end
+
+    it "returns an empty vector if given an empty dataframe" do
+      df = Daru::DataFrame.new({ a: [], b: [] })
+      expect(df.group_by(:a).size).to eq(Daru::Vector.new([]))
+    end
   end
 
   context "#get_group" do
@@ -336,4 +345,11 @@ describe Daru::Core::GroupBy do
   context "#[]" do
     pending
   end
+
+  context "#reduce" do
+    it "returns a vector that concatenates strings in a group" do
+      string_concat = lambda { |result, row| result += row[:b] }
+      expect(@sl_group.reduce('', &string_concat)).to eq(Daru::Vector.new(['onethreetwo', 'onetwotwoonethree'], index: @sl_index))
+    end
+  end
 end

data/spec/core/merge_spec.rb

@@ -15,51 +15,67 @@ describe Daru::DataFrame do
         :id => [1,1,1,1],
         :name => ['Rutabaga', 'Pirate', 'Darth Vader', 'Ninja']
       })
+      @empty = Daru::DataFrame.new({
+        :id => [],
+        :name => []
+      })
     end
 
     it "performs an inner join of two dataframes" do
       answer = Daru::DataFrame.new({
-        :id_1   => [1,3],
-        :name => ['Pirate', 'Ninja'],
-        :id_2   => [2,4]
+        :id_1   => [3,1],
+        :name => ['Ninja', 'Pirate'],
+        :id_2   => [4,2]
       }, order: [:id_1, :name, :id_2])
       expect(@left.join(@right, how: :inner, on: [:name])).to eq(answer)
     end
 
     it "performs an inner join of two dataframes that has one to many mapping" do
       answer = Daru::DataFrame.new({
-        :id => [1,1,1,1],
         :name_1 => ['Pirate', 'Pirate', 'Pirate', 'Pirate'],
+        :id => [1,1,1,1],
         :name_2 => ['Rutabaga', 'Pirate', 'Darth Vader', 'Ninja']
-      }, order: [:id, :name_1, :name_2])
+      }, order: [:name_1, :id, :name_2])
       expect(@left.join(@right_many, how: :inner, on: [:id])).to eq(answer)
     end
 
     it "performs a full outer join" do
       answer = Daru::DataFrame.new({
-        :id_1 => [1,2,3,4,nil,nil],
-        :name => ['Pirate', 'Monkey', 'Ninja', 'Spaghetti','Rutabaga', 'Darth Vader'],
-        :id_2 => [2,nil,4,nil,1,3]
+        :id_1 => [nil,2,3,1,nil,4],
+        :name => ["Darth Vader", "Monkey", "Ninja", "Pirate", "Rutabaga", "Spaghetti"],
+        :id_2 => [3,nil,4,2,1,nil]
       }, order: [:id_1, :name, :id_2])
       expect(@left.join(@right, how: :outer, on: [:name])).to eq(answer)
     end
 
     it "performs a left outer join", focus: true do
       answer = Daru::DataFrame.new({
-        :id_1 => [1,2,3,4],
-        :name => ['Pirate', 'Monkey', 'Ninja', 'Spaghetti'],
-        :id_2 => [2,nil,4,nil]
+        :id_1 => [2,3,1,4],
+        :name => ["Monkey", "Ninja", "Pirate", "Spaghetti"],
+        :id_2 => [nil,4,2,nil]
       }, order: [:id_1, :name, :id_2])
       expect(@left.join(@right, how: :left, on: [:name])).to eq(answer)
     end
 
+    it "performs a left join with an empty dataframe" do
+
+      answer = Daru::DataFrame.new({
+        :id_1 => [2,3,1,4],
+        :name => ["Monkey", "Ninja", "Pirate", "Spaghetti"],
+        :id_2 => [nil,nil,nil,nil]
+      }, order: [:id_1, :name, :id_2])
+
+      expect(@left.join(@empty, how: :left, on: [:name])).to eq(answer)
+    end
+
     it "performs a right outer join" do
       answer = Daru::DataFrame.new({
-        :id_1 => [nil,1,nil,3],
-        :name => ['Rutabaga','Pirate', 'Darth Vader', 'Ninja'],
-        :id_2 => [1,2,3,4]
+        :id_1 => [nil,3,1,nil],
+        :name => ["Darth Vader", "Ninja", "Pirate", "Rutabaga"],
+        :id_2 => [3,4,2,1]
       }, order: [:id_1, :name, :id_2])
       expect(@left.join(@right, how: :right, on: [:name])).to eq(answer)
     end
+
   end
 end

data/spec/dataframe_spec.rb

@@ -133,14 +133,17 @@ describe Daru::DataFrame do
       end
 
       it "initializes from a Hash of Vectors" do
-        df = Daru::DataFrame.new({b: [11,12,13,14,15].dv(:b, [:one, :two, :three, :four, :five]),
-          a: [1,2,3,4,5].dv(:a, [:one, :two, :three, :four, :five])}, order: [:a, :b],
-          index: [:one, :two, :three, :four, :five])
+        va = Daru::Vector.new([1,2,3,4,5], metadata: { cdc_type: 2 }, index: [:one, :two, :three, :four, :five])
+        vb = Daru::Vector.new([11,12,13,14,15], index: [:one, :two, :three, :four, :five])
+
+        df = Daru::DataFrame.new({ b: vb, a: va }, order: [:a, :b], index: [:one, :two, :three, :four, :five])
 
         expect(df.index)  .to eq(Daru::Index.new [:one, :two, :three, :four, :five])
         expect(df.vectors).to eq(Daru::Index.new [:a, :b])
         expect(df.a.class).to eq(Daru::Vector)
         expect(df.a)      .to eq([1,2,3,4,5].dv(:a, [:one, :two, :three, :four, :five]))
+        expect(df.a.metadata).to eq({ cdc_type: 2 })
+        expect(df.b.metadata).to eq({})
       end
 
       it "initializes from an Array of Hashes" do
@@ -419,6 +422,13 @@ describe Daru::DataFrame do
         expect(@df[:a, :b]).to eq(temp)
       end
 
+      it "returns a DataFrame with metadata" do
+        @df[:a].metadata = "alpha"
+        @df[:b].metadata = "beta"
+        subset_df = @df[:a, :b]
+        expect([:a, :b].map { |v| subset_df[v].metadata }).to eq(["alpha", "beta"])
+      end
+
       it "accesses vector with Integer index" do
         expect(@df[0]).to eq([1,2,3,4,5].dv(:a, [:one, :two, :three, :four, :five]))
       end
@@ -455,7 +465,7 @@ describe Daru::DataFrame do
       end
 
       it "returns a Vector if the last level of MultiIndex is tracked" do
-        expect(@df_mi[:a, :one]).to eq(
+        expect(@df_mi[:a, :one, :bar]).to eq(
           Daru::Vector.new(@vector_arry1, index: @multi_index))
       end
     end
@@ -521,6 +531,31 @@ describe Daru::DataFrame do
           }.to raise_error
       end
 
+      it "assigns correct name given empty dataframe" do
+        df_empty = Daru::DataFrame.new({})
+        df_empty[:a] = 1..5
+        df_empty[:b] = 1..5
+
+        expect(df_empty[:a].name).to equal(:a)
+        expect(df_empty[:b].name).to equal(:b)
+      end
+
+      it "copies metadata when the target is a vector" do
+        vec = Daru::Vector.new(1.upto(@df.size), index: @df.index, metadata: { cdc_type: 2 })
+        @df[:woo] = vec.dup
+        expect(@df[:woo].metadata).to eq vec.metadata
+      end
+
+      it "doesn't delete metadata when the source is a dataframe with empty vectors" do
+        empty_df = Daru::DataFrame.new({
+          a: Daru::Vector.new([], metadata: 'alpha'),
+          b: Daru::Vector.new([], metadata: 'beta'),
+          })
+
+        empty_df[:c] = Daru::Vector.new(1.upto(3))
+        expect(empty_df[:a].metadata).to eq 'alpha'
+      end
+
       it "appends multiple vectors at a time" do
         # TODO
       end
@@ -561,6 +596,13 @@ describe Daru::DataFrame do
 
         expect(@df_mi).to eq(answer)
       end
+
+      it "assigns correct name given empty dataframe" do
+        df_empty = Daru::DataFrame.new([], index: @multi_index, order: @order_mi)
+        df_empty[:c, :one, :bar] = 1..12
+
+        expect(df_empty[:c, :one, :bar].name).to eq "conebar"
+      end
     end
   end
 
@@ -848,6 +890,16 @@ describe Daru::DataFrame do
       expect(cloned[:a].object_id).to eq(@data_frame[:a].object_id)
       expect(cloned[:b].object_id).to eq(@data_frame[:b].object_id)
     end
+
+    it "original dataframe remains unaffected when operations are applied
+      on cloned data frame" do
+      original = @data_frame.dup
+      cloned = @data_frame.clone
+      cloned.delete_vector :a
+
+      expect(@data_frame).to eq(original)
+    end
+
   end
 
   context "#clone_structure" do
@@ -923,6 +975,17 @@ describe Daru::DataFrame do
       @ans_rows = Daru::DataFrame.new({b: [121, 144, 169, 196, 225], a: [1,4,9,16,25],
         c: [121, 484, 1089, 1936, 3025]}, order: [:a, :b, :c],
         index: [:one, :two, :three, :four, :five])
+
+      @data_frame_date_time = @data_frame.dup
+      @data_frame_date_time.index = Daru::DateTimeIndex.date_range(start:"2016-02-11", periods:5)
+
+      @ans_vector_date_time = Daru::DataFrame.new({b: [21,22,23,24,25], a: [11,12,13,14,15],
+        c: [21,32,43,54,65]}, order: [:a, :b, :c],
+        index: Daru::DateTimeIndex.date_range(start:"2016-02-11", periods:5))
+
+      @ans_rows_date_time = Daru::DataFrame.new({b: [121, 144, 169, 196, 225], a: [1,4,9,16,25],
+        c: [121, 484, 1089, 1936, 3025]}, order: [:a, :b, :c],
+        index: Daru::DateTimeIndex.date_range(start:"2016-02-11", periods:5))
     end
 
     it "maps over the vectors of a DataFrame and returns a DataFrame" do
@@ -941,6 +1004,24 @@ describe Daru::DataFrame do
 
       expect(ret).to eq(@ans_rows)
     end
+
+    it "maps over the vectors of a DataFrame with DateTimeIndex and returns a DataFrame with DateTimeIndex" do
+      ret = @data_frame_date_time.recode do |vector|
+        vector.map! { |e| e += 10}
+      end
+
+      expect(ret).to eq(@ans_vector_date_time)
+    end
+
+    it "maps over the rows of a DataFrame with DateTimeIndex and returns a DataFrame with DateTimeIndex" do
+      ret = @data_frame_date_time.recode(:row) do |row|
+        expect(row.class).to eq(Daru::Vector)
+        row.map! { |e| e*e }
+      end
+
+      expect(ret).to eq(@ans_rows_date_time)
+    end
+
   end
 
   context "#collect" do
@@ -1053,6 +1134,18 @@ describe Daru::DataFrame do
     end
   end
 
+  context "#delete_vectors" do
+    context Daru::Index do
+      it "deletes the specified vectors" do
+        @data_frame.delete_vectors :a, :b
+
+        expect(@data_frame).to eq(Daru::DataFrame.new({
+                c: [11,22,33,44,55]}, order: [:c],
+                index: [:one, :two, :three, :four, :five]))
+      end
+    end
+  end
+
   context "#delete_row" do
     it "deletes the specified row" do
       @data_frame.delete_row :three
@@ -1158,9 +1251,9 @@ describe Daru::DataFrame do
     end
   end
 
-  context "#to_hash" do
+  context "#to_h" do
     it "converts to a hash" do
-      expect(@data_frame.to_hash).to eq(
+      expect(@data_frame.to_h).to eq(
         {
           a: Daru::Vector.new([1,2,3,4,5],
             index: [:one, :two, :three, :four, :five]),
@@ -1187,15 +1280,16 @@ describe Daru::DataFrame do
     context Daru::Index do
       before :each do
         @df = Daru::DataFrame.new({a: [5,1,-6,7,5,5], b: [-2,-1,5,3,9,1], c: ['a','aa','aaa','aaaa','aaaaa','aaaaaa']})
+        @df[:a].metadata = { cdc_type: 2 }
       end
 
       it "sorts according to given vector order (bang)" do
-        a_sorter = lambda { |a,b| a <=> b }
+        a_sorter = lambda { |a| a }
         ans = @df.sort([:a], by: { a: a_sorter })
 
         expect(ans).to eq(
-          Daru::DataFrame.new({a: [-6,1,5,5,5,7], b: [5,-1,9,1,-2,3], c: ['aaa','aa','aaaaa','aaaaaa','a','aaaa']},
-            index: [2,1,4,5,0,3])
+          Daru::DataFrame.new({a: [-6,1,5,5,5,7], b: [5,-1,-2,9,1,3], c: ['aaa','aa','a','aaaaa','aaaaaa','aaaa']},
+            index: [2,1,0,4,5,3])
           )
         expect(ans).to_not eq(@df)
       end
@@ -1208,6 +1302,12 @@ describe Daru::DataFrame do
           )
         expect(ans).to_not eq(@df)
       end
+
+      it "retains the vector metadata from the original dataframe" do
+        ans = @df.sort([:a])
+        expect(ans[:a].metadata).to eq({ cdc_type: 2 })
+      end
+
     end
 
     context Daru::MultiIndex do
@@ -1223,11 +1323,11 @@ describe Daru::DataFrame do
       end
 
       it "sorts according to given vector order (bang)" do
-        a_sorter = lambda { |a,b| a <=> b }
+        a_sorter = lambda { |a| a }
 
         expect(@df.sort!([:a], by: { a: a_sorter })).to eq(
-          Daru::DataFrame.new({a: [-6,1,5,5,5,7], b: [5,-1,9,1,-2,3],
-            c: ['aaa','aa','aaaaa','aaaaaa','a','aaaa']}, index: [2,1,4,5,0,3])
+          Daru::DataFrame.new({a: [-6,1,5,5,5,7], b: [5,-1,-2,9,1,3],
+            c: ['aaa','aa','a','aaaaa','aaaaaa','aaaa']}, index: [2,1,0,4,5,3])
           )
       end
 
@@ -1260,6 +1360,68 @@ describe Daru::DataFrame do
             index: [7,3,4,6,5,0,1,2])
           )
       end
+
+      it "places nils at the beginning when sorting ascedingly" do
+        d = Daru::DataFrame.new({a: [1,1,1,nil,44,5,5,nil], b: [44,44,333,222,111,554,22,3], c: [3,2,5,3,3,1,5,5]})
+
+        expect(d.sort!([:a, :b, :c], ascending: [true, true, false])).to eq(
+          Daru::DataFrame.new({a: [nil,nil,1,1,1,5,5,44], b: [3,222,44,44,333,22,554,111], c: [5,3,3,2,5,5,1,3]},
+            index: [7,3,0,1,2,6,5,4])
+          )
+      end
+
+      it "places nils at the beginning when sorting decendingly" do
+        d = Daru::DataFrame.new({a: [1,1,1,nil,44,5,5,nil], b: [44,44,333,222,111,554,22,3], c: [3,2,5,3,3,1,5,5]})
+
+        expect(d.sort!([:a, :b, :c], ascending: [false, true, false])).to eq(
+          Daru::DataFrame.new({a: [nil,nil,44,5,5,1,1,1], b: [3,222,111,22,554,44,44,333], c: [5,3,3,5,1,3,2,5]},
+            index: [7,3,4,6,5,0,1,2])
+          )
+      end
+
+      it "sorts vectors of non-numeric types with nils in ascending order" do
+        non_numeric = Daru::DataFrame.new({a: [5,1,-6,7,5,5], b: [nil,-1,1,nil,-1,1],
+          c: ['aaa','aaa',nil,'baaa','xxx',nil]})
+
+        expect(non_numeric.sort!([:c], ascending: [true])).to eq(
+          Daru::DataFrame.new({a: [-6, 5, 5, 1, 7, 5], b: [1, 1, nil, -1, nil, -1],
+            c: [nil, nil, "aaa", "aaa", "baaa", "xxx"]},
+            index: [2, 5, 0, 1, 3, 4])
+          )
+      end
+
+      it "sorts vectors of non-numeric types with nils in descending order" do
+        non_numeric = Daru::DataFrame.new({a: [5,1,-6,7,5,5], b: [nil,-1,1,nil,-1,1],
+          c: ['aaa','aaa',nil,'baaa','xxx',nil]})
+
+        expect(non_numeric.sort!([:c], ascending: [false])).to eq(
+          Daru::DataFrame.new({a: [-6, 5, 5, 7, 5, 1], b: [1, 1, -1, nil, nil, -1],
+            c: [nil, nil, "xxx", "baaa", "aaa", "aaa"]},
+            index: [2, 5, 4, 3, 0, 1])
+          )
+      end
+
+      it "sorts vectors with block provided and handle nils automatically" do
+        non_numeric = Daru::DataFrame.new({a: [5,1,-6,7,5,5], b: [nil,-1,1,nil,-1,1],
+          c: ['aaa','aaa',nil,'baaa','xxx',nil]})
+
+        expect(non_numeric.sort!([:b], by: {b: lambda { |a| a.abs } }, handle_nils: true)).to eq(
+          Daru::DataFrame.new({a: [5, 7, 1, -6, 5, 5], b: [nil, nil, -1, 1, -1, 1],
+            c: ["aaa", "baaa", "aaa", nil, "xxx", nil]},
+            index: [0, 3, 1, 2, 4, 5])
+          )
+      end
+
+      it "sorts vectors with block provided and nils handled manually" do
+        non_numeric = Daru::DataFrame.new({a: [5,1,-6,7,5,5], b: [nil,-1,1,nil,-1,1],
+          c: ['aaa','aaa',nil,'baaa','xxx',nil]})
+
+      expect(non_numeric.sort!([:b], by: {b: lambda { |a| (a.nil?)?[1]:[0, a.abs]} }, handle_nils: false)).to eq(
+        Daru::DataFrame.new({a: [1, -6, 5, 5, 5, 7], b: [-1, 1, -1, 1, nil, nil],
+          c: ["aaa", nil, "xxx", nil, "aaa", "baaa"]},
+          index: [1, 2, 4, 5, 0, 3])
+        )
+      end
     end
 
     context Daru::MultiIndex do
@@ -1316,6 +1478,37 @@ describe Daru::DataFrame do
     end
   end
 
+  context "#rename_vectors" do
+    before do
+      @df = Daru::DataFrame.new({
+        a: [1,2,3,4,5],
+        b: [11,22,33,44,55],
+        c: %w(a b c d e)
+      })
+    end
+
+    it "renames vectors using a hash map" do
+      @df.rename_vectors :a => :alpha, :c => :gamma
+      expect(@df.vectors.to_a).to eq([:alpha, :b, :gamma])
+    end
+
+    it "overwrites vectors if the new name already exists" do
+      saved_vector = @df[:a].dup
+
+      @df.rename_vectors :a => :b
+      expect(@df.vectors.to_a).to eq([:b, :c])
+      expect(@df[:b]).to eq saved_vector
+    end
+
+    it "makes no changes if the old and new names are the same" do
+      saved_vector = @df[:a].dup
+
+      @df.rename_vectors :a => :a
+      expect(@df.vectors.to_a).to eq([:a, :b, :c])
+      expect(@df[:a]).to eq saved_vector
+    end
+  end
+
   context "#reindex" do
     it "re indexes and aligns accordingly" do
       df = Daru::DataFrame.new({
@@ -1603,6 +1796,56 @@ describe Daru::DataFrame do
         @df.pivot_table
       }.to raise_error
     end
+
+    it "aggregates when nils are present in value vector" do
+      df = Daru::DataFrame.new({
+        a: ['foo'  ,  'foo',  'foo',  'foo',  'foo',  'bar',  'bar',  'bar',  'ice'],
+        b: ['one'  ,  'one',  'one',  'two',  'two',  'one',  'one',  'two',  'two'],
+        c: ['small','large','large','small','small','large','small','large','small'],
+        d: [1,2,2,3,3,4,5,6,7],
+        e: [2,nil,4,6,6,8,10,12,nil]
+      })
+
+      expect(df.pivot_table index: [:a]).to eq(
+        Daru::DataFrame.new({
+          d:  [5.0, 2.2, 7],
+          e:  [10.0, 4.5, nil]
+        }, index: Daru::Index.new(['bar', 'foo', 'ice'])))
+    end
+
+    it "works when nils are present in value vector" do
+      df = Daru::DataFrame.new({
+        a: ['foo'  ,  'foo',  'foo',  'foo',  'foo',  'bar',  'bar',  'bar',  'ice'],
+        b: ['one'  ,  'one',  'one',  'two',  'two',  'one',  'one',  'two',  'two'],
+        c: ['small','large','large','small','small','large','small','large','small'],
+        d: [1,2,2,3,3,4,5,6,7],
+        e: [2,nil,4,6,6,8,10,12,nil]
+      })
+
+      agg_vectors = Daru::MultiIndex.from_tuples(
+        [
+          [:e, 'one'],
+          [:e, 'two']
+        ]
+      )
+
+      agg_index = Daru::MultiIndex.from_tuples(
+        [
+          ['bar'],
+          ['foo'],
+          ['ice']
+        ]
+      )
+
+      expect(df.pivot_table index: [:a], vectors: [:b], values: :e).to eq(
+        Daru::DataFrame.new(
+          [
+            [9, 3,  nil],
+            [12, 6, nil]
+          ], order: agg_vectors, index: agg_index
+        )
+      )
+    end
   end
 
   context "#shape" do
@@ -2092,11 +2335,13 @@ describe Daru::DataFrame do
   context "#concat" do
     before do
       @df1 = Daru::DataFrame.new({
-        a: [1, 2, 3]
+        a: [1, 2, 3],
+        b: [1, 2, 3]
       })
 
       @df2 = Daru::DataFrame.new({
-        a: [4, 5, 6]
+        a: [4, 5, 6],
+        c: [4, 5, 6]
       })
     end
 
@@ -2117,5 +2362,14 @@ describe Daru::DataFrame do
       expect(df_concat[:a].to_a).to eq df1_a + df2_a
     end
 
+    it 'fills in missing vectors with nils' do
+      df1_b = @df1[:b].to_a.dup
+      df2_c = @df2[:c].to_a.dup
+
+      df_concat = @df1.concat @df2
+      expect(df_concat[:b].to_a).to eq df1_b + [nil] * @df2.size
+      expect(df_concat[:c].to_a).to eq [nil] * @df1.size + df2_c
+    end
+
   end
 end if mri?