daru 0.1.5 → 0.3

data/spec/index/multi_index_spec.rb

@@ -35,6 +35,54 @@ describe Daru::MultiIndex do
           labels: [[0,0,1,1,2,2]])
       }.to raise_error
     end
+
+    context "create an MultiIndex with name" do
+      context 'if no name is set' do
+        subject { Daru::MultiIndex.new(
+                    levels: [[:a,:b,:c], [:one, :two]],
+                    labels: [[0,0,1,1,2,2], [0,1,0,1,0,1]]) }
+        its(:name) { is_expected.to be_nil }
+      end
+
+      context 'correctly return the MultiIndex name' do
+        subject { Daru::MultiIndex.new(
+                  levels: [[:a,:b,:c], [:one, :two]],
+                  labels: [[0,0,1,1,2,2], [0,1,0,1,0,1]], name: ['n1', 'n2']) }
+        its(:name) { is_expected.to eq ['n1', 'n2'] }
+      end
+
+      context "set new MultiIndex name" do
+        subject {
+          Daru::MultiIndex.new(
+                  levels: [[:a,:b,:c], [:one, :two]],
+                  labels: [[0,0,1,1,2,2], [0,1,0,1,0,1]], name: ['n1', 'n2']) }
+        before(:each) { subject.name = ['k1', 'k2'] }
+        its(:name) { is_expected.to eq ['k1', 'k2'] }
+      end
+
+      context "set new MultiIndex name having empty string" do
+        subject {
+          Daru::MultiIndex.new(
+                  levels: [[:a,:b,:c], [:one, :two]],
+                  labels: [[0,0,1,1,2,2], [0,1,0,1,0,1]], name: ['n1', 'n2']) }
+        before { subject.name = ['k1', ''] }
+        its(:name) { is_expected.to eq ['k1', ''] }
+      end
+
+      it "raises SizeError for wrong number of name" do
+        error_msg = "\'names\' and \'levels\' should be of same size. Size of the \'name\' array is 2 and size of the MultiIndex \'levels\' and \'labels\' is 3.\nIf you don\'t want to set name for particular level (say level \'i\') then put empty string on index \'i\' of the \'name\' Array."
+        expect { @multi_mi.name = ['n1', 'n2'] }.to raise_error(SizeError, error_msg)
+
+        error_msg = "'names' and 'levels' should be of same size. Size of the 'name' array is 0 and size of the MultiIndex 'levels' and 'labels' is 3.\nIf you don\'t want to set name for particular level (say level 'i') then put empty string on index 'i' of the 'name' Array."
+        expect { @multi_mi.name = [ ] }.to raise_error(SizeError, error_msg)
+
+        error_msg = "'names' and 'levels' should be of same size. Size of the 'name' array is 1 and size of the MultiIndex 'levels' and 'labels' is 3.\nIf you don\'t want to set name for particular level (say level 'i') then put empty string on index 'i' of the 'name' Array."
+        expect { @multi_mi.name = [''] }.to raise_error(SizeError, error_msg)
+
+        error_msg = "'names' and 'levels' should be of same size. Size of the 'name' array is 4 and size of the MultiIndex 'levels' and 'labels' is 3."
+        expect { @multi_mi.name = ['n1', 'n2', 'n3', 'n4'] }.to raise_error(SizeError, error_msg)
+      end
+    end
   end
 
   context ".from_tuples" do
@@ -154,8 +202,16 @@ describe Daru::MultiIndex do
       expect(@multi_mi.include?([:a, :one])).to eq(true)
     end
 
-    it "checks for non-existence of a tuple" do
-      expect(@multi_mi.include?([:boo])).to eq(false)
+    it "checks for non-existence of completely specified tuple" do
+      expect(@multi_mi.include?([:b, :two, :foo])).to eq(false)
+    end
+
+    it "checks for non-existence of a top layer incomplete tuple" do
+      expect(@multi_mi.include?([:d])).to eq(false)
+    end
+
+    it "checks for non-existence of a middle layer incomplete tuple" do
+      expect(@multi_mi.include?([:c, :three])).to eq(false)
     end
   end
 
@@ -255,6 +311,42 @@ describe Daru::MultiIndex do
         }.unindent
       }
     end
+
+    context 'multi index with name' do
+      subject {
+        Daru::MultiIndex.new(
+          levels: [[:a,:b,:c],[:one,:two],[:bar, :baz, :foo]],
+          labels: [
+            [0,0,0,0,1,1,1,1,2,2,2,2],
+            [0,0,1,1,0,1,1,0,0,0,1,1],
+            [0,1,0,1,0,0,1,2,0,1,2,0]], name: ['n1', 'n2', 'n3'])
+      }
+
+      its(:inspect) { is_expected.to start_with %Q{
+        |#<Daru::MultiIndex(12x3)>
+        |  n1  n2  n3
+        }.unindent
+      }
+    end
+
+    context 'multi index with name having empty string' do
+      subject {
+        mi= Daru::MultiIndex.new(
+                  levels: [[:a,:b,:c],[:one,:two],[:bar, :baz, :foo]],
+                  labels: [
+                    [0,0,0,0,1,1,1,1,2,2,2,2],
+                    [0,0,1,1,0,1,1,0,0,0,1,1],
+                    [0,1,0,1,0,0,1,2,0,1,2,0]], name: ['n1', 'n2', 'n3'])
+      }
+      before { subject.name = ['n1', '', 'n3'] }
+
+      its(:inspect) { is_expected.to start_with %Q{
+        |#<Daru::MultiIndex(12x3)>
+        |  n1      n3
+        }.unindent
+      }
+    end
+
   end
 
   context "#==" do
@@ -410,31 +502,31 @@ describe Daru::MultiIndex do
         [:b,:one,:foo]
       ])
     end
-    
+
     context "single index" do
       it { expect(idx.pos :b, :one, :bar).to eq 0 }
     end
-    
+
     context "multiple indexes" do
       subject { idx.pos :b, :one }
-      
+
       it { is_expected.to be_a Array }
       its(:size) { is_expected.to eq 2 }
       it { is_expected.to eq [0, 3] }
     end
-    
+
     context "single positional index" do
       it { expect(idx.pos 0).to eq 0 }
     end
-    
+
     context "multiple positional indexes" do
       subject { idx.pos 0, 1 }
-      
+
       it { is_expected.to be_a Array }
       its(:size) { is_expected.to eq 2 }
       it { is_expected.to eq [0, 1] }
     end
-    
+
     # TODO: Add specs for IndexError
   end
 
@@ -447,23 +539,23 @@ describe Daru::MultiIndex do
         [:b, :one, :foo]
       ])
     end
-    
+
     context "multiple indexes" do
       subject { idx.subset :b, :one }
-      
+
       it { is_expected.to be_a described_class }
       its(:size) { is_expected.to eq 2 }
       its(:to_a) { is_expected.to eq [[:bar], [:foo]] }
     end
-    
+
     context "multiple positional indexes" do
       subject { idx.subset 0, 1 }
-      
+
       it { is_expected.to be_a described_class }
       its(:size) { is_expected.to eq 2 }
       its(:to_a) { is_expected.to eq [[:b, :one, :bar], [:b, :two, :bar]] }
     end
-    
+
     # TODO: Checks for invalid indexes
   end
 
@@ -476,53 +568,53 @@ describe Daru::MultiIndex do
         [:b, :one, :foo]
       ])
     end
-    
+
     context "single position" do
       it { expect(idx.at 2).to eq [:b, :two, :baz] }
     end
-    
+
     context "multiple positions" do
       subject { idx.at 1, 2 }
-      
+
       it { is_expected.to be_a described_class }
       its(:size) { is_expected.to eq 2 }
       its(:to_a) { is_expected.to eq [[:b, :two, :bar],
         [:b, :two, :baz]] }
     end
-    
+
     context "range" do
       subject { idx.at 1..2 }
-      
+
       it { is_expected.to be_a described_class }
       its(:size) { is_expected.to eq 2 }
       its(:to_a) { is_expected.to eq [[:b, :two, :bar],
-        [:b, :two, :baz]] }    
+        [:b, :two, :baz]] }
     end
-    
+
     context "range with negative integers" do
       subject { idx.at 1..-2 }
-      
+
       it { is_expected.to be_a described_class }
       its(:size) { is_expected.to eq 2 }
       its(:to_a) { is_expected.to eq [[:b, :two, :bar],
-        [:b, :two, :baz]] }    
-    end    
-    
+        [:b, :two, :baz]] }
+    end
+
     context "rangle with single element" do
       subject { idx.at 1..1 }
-      
+
       it { is_expected.to be_a described_class }
       its(:size) { is_expected.to eq 1 }
       its(:to_a) { is_expected.to eq [[:b, :two, :bar]] }
     end
-    
+
     context "invalid position" do
       it { expect { idx.at 4 }.to raise_error IndexError }
     end
-    
+
     context "invalid positions" do
       it { expect { idx.at 2, 4 }.to raise_error IndexError }
-    end    
+    end
   end
 
   context "#add" do
@@ -534,7 +626,7 @@ describe Daru::MultiIndex do
         [:b, :one, :foo]
       ]
     end
-    
+
     context "single index" do
       subject { idx.add :b, :two, :baz }
 
@@ -556,15 +648,33 @@ describe Daru::MultiIndex do
         [:b, :one, :foo]
       ]
     end
-    
+
     context "single index" do
       it { expect(idx.valid? :a, :one, :bar).to eq true }
       it { expect(idx.valid? :b, :two, :three).to eq false }
     end
-    
+
     context "multiple indexes" do
       it { expect(idx.valid? :a, :one).to eq true }
       it { expect(idx.valid? :a, :three).to eq false }
     end
-  end    
+  end
+
+  context '#to_df' do
+    let(:idx) do
+      described_class.from_tuples([
+        %w[a one bar],
+        %w[a two bar],
+        %w[b two baz],
+        %w[b one foo]
+      ]).tap { |idx| idx.name = %w[col1 col2 col3] }
+    end
+
+    subject { idx.to_df }
+    it { is_expected.to eq Daru::DataFrame.new(
+           'col1' => %w[a a b b],
+           'col2' => %w[one two two one],
+           'col3' => %w[bar bar baz foo]
+    )}
+  end
 end

data/spec/io/io_spec.rb

@@ -1,12 +1,21 @@
+# -*- coding: utf-8 -*-
 describe Daru::IO do
   describe Daru::DataFrame do
     context ".from_csv" do
+      before do
+        %w[matrix_test repeated_fields scientific_notation sales-funnel].each do |file|
+          WebMock
+            .stub_request(:get,"http://dummy-remote-url/#{file}.csv")
+            .to_return(status: 200, body: File.read("spec/fixtures/#{file}.csv"))
+        end
+      end
+
       it "loads from a CSV file" do
         df = Daru::DataFrame.from_csv('spec/fixtures/matrix_test.csv',
           col_sep: ' ', headers: true)
 
-        df.vectors = [:image_resolution, :mls, :true_transform].to_index
-        expect(df.vectors).to eq([:image_resolution, :mls, :true_transform].to_index)
+        df.vectors = [:image_resolution, :true_transform, :mls].to_index
+        expect(df.vectors).to eq([:image_resolution, :true_transform, :mls].to_index)
         expect(df[:image_resolution].first).to eq(6.55779)
         expect(df[:true_transform].first).to eq("-0.2362347,0.6308649,0.7390552,0,0.6523478,-0.4607318,0.6018043,0,0.7201635,0.6242881,-0.3027024,4262.65,0,0,0,1")
       end
@@ -32,6 +41,34 @@ describe Daru::IO do
         df = Daru::DataFrame.from_csv 'spec/fixtures/sales-funnel.csv'
         expect(df.vectors.to_a).to eq(%W[Account Name Rep Manager Product Quantity Price Status])
       end
+
+      it "handles empty rows in the CSV" do
+        df = Daru::DataFrame.from_csv 'spec/fixtures/empty_rows_test.csv'
+        expect(df.nrows).to eq(13)
+      end
+
+      it "uses the custom boolean converter correctly" do
+        df = Daru::DataFrame.from_csv 'spec/fixtures/boolean_converter_test.csv', converters: [:boolean]
+        expect(df['Domestic'].to_a).to all be_boolean
+      end
+
+      it "uses the custom string converter correctly" do
+        df = Daru::DataFrame.from_csv 'spec/fixtures/string_converter_test.csv', converters: [:string]
+        expect(df['Case Number'].to_a.all? {|x| String === x }).to be_truthy
+      end
+
+      it "allow symbol to converters option" do
+        df = Daru::DataFrame.from_csv 'spec/fixtures/boolean_converter_test.csv', converters: :boolean
+        expect(df['Domestic'].to_a).to all be_boolean
+      end
+
+      it "checks for equal parsing of local CSV files and remote CSV files" do
+        %w[matrix_test repeated_fields scientific_notation sales-funnel].each do |file|
+          df_local  = Daru::DataFrame.from_csv("spec/fixtures/#{file}.csv")
+          df_remote = Daru::DataFrame.from_csv("http://dummy-remote-url/#{file}.csv")
+          expect(df_local).to eq(df_remote)
+        end
+      end
     end
 
     context "#write_csv" do
@@ -86,6 +123,38 @@ describe Daru::IO do
       end
     end
 
+    context "#from_excel with row_id" do
+      before do
+        id   = Daru::Vector.new(['id', 1.0, 2.0, 3.0, 4.0, 5.0, 6.0])
+        name = Daru::Vector.new(%w(name Alex Claude Peter Franz George Fernand))
+        age  = Daru::Vector.new(['age', 20.0, 23.0, 25.0, nil, 5.5, nil])
+        city = Daru::Vector.new(['city', 'New York', 'London', 'London', 'Paris', 'Tome', nil])
+        a1   = Daru::Vector.new(['a1', 'a,b', 'b,c', 'a', nil, 'a,b,c', nil])
+        @expected_1 = Daru::DataFrame.new({:id2 => id, :name2 => name, :age2 => age}, order: [:id2, :name2, :age2])
+        @expected_2 = Daru::DataFrame.new({
+          :id => id, :name => name, :age => age, :city => city, :a1 => a1
+          }, order: [:id, :name, :age, :city, :a1])
+      end
+
+      it "loads DataFrame from test_xls_2.xls" do
+        df = Daru::DataFrame.from_excel 'spec/fixtures/test_xls_2.xls'
+
+        expect(df.nrows).to eq(7)
+        expect(df.vectors.to_a).to eq([:id2, :name2, :age2])
+        expect(df[:age2][6]).to eq(nil)
+        expect(@expected_1).to eq(df)
+      end
+
+      it "loads DataFrame from test_xls_2.xls with row_id" do
+        df = Daru::DataFrame.from_excel 'spec/fixtures/test_xls_2.xls', {row_id: 1}
+
+        expect(df.nrows).to eq(7)
+        expect(df.vectors.to_a).to eq([:id, :name, :age, :city, :a1])
+        expect(df[:age][6]).to eq(nil)
+        expect(@expected_2).to eq(df)
+      end
+    end
+
     context "#write_excel" do
       before do
         a   = Daru::Vector.new(100.times.map { rand(100) })
@@ -269,6 +338,181 @@ describe Daru::IO do
         expect(a).to eq(@data_frame)
       end
     end
+
+    context "#from_html" do
+      context "in wiki info table" do
+        let(:path) {  "file://#{Dir.pwd}/spec/fixtures/wiki_table_info.html" }
+        let(:order) { ["FName", "LName", "Age"] }
+        let(:index) { ["One", "Two", "Three", "Four", "Five", "Six", "Seven"] }
+        let(:name) { "Wikipedia Information Table" }
+
+        context "returns default dataframe" do
+          subject { Daru::DataFrame.from_html(path) }
+
+          it { is_expected.to be_an(Array).and all be_a(Daru::DataFrame) }
+          its(:first) { is_expected.to eq (Daru::DataFrame.new(
+                [["Tinu", "Blaszczyk", "Lily", "Olatunkboh", "Adrienne", "Axelia", "Jon-Kabat"],
+                ["Elejogun", "Kostrzewski", "McGarrett", "Chijiaku", "Anthoula", "Athanasios", "Zinn"],
+                ["14", "25", "16", "22", "22", "22", "22"]],
+                order: ["First name","Last name","Age"]
+              )
+            )
+          }
+        end
+
+        context "returns user-modified dataframe" do
+          subject { Daru::DataFrame.from_html(path, order: order, index: index, name: name) }
+
+          it { is_expected.to be_an(Array).and all be_a(Daru::DataFrame) }
+          its(:first) { is_expected.to eq(Daru::DataFrame.new(
+                [["Tinu", "Blaszczyk", "Lily", "Olatunkboh", "Adrienne", "Axelia", "Jon-Kabat"],
+                ["Elejogun", "Kostrzewski", "McGarrett", "Chijiaku", "Anthoula", "Athanasios", "Zinn"],
+                ["14", "25", "16", "22", "22", "22", "22"]],
+                order: ["FName","LName", "Age"],
+                index: ["One", "Two", "Three", "Four", "Five", "Six", "Seven"],
+                name: "Wikipedia Information Table"
+              )
+            )
+          }
+        end
+      end
+
+      context "in wiki climate data" do
+        let(:path) { "file://#{Dir.pwd}/spec/fixtures/wiki_climate.html" }
+
+        context "returns default dataframe" do
+          subject { Daru::DataFrame.from_html(path) }
+
+          it { is_expected.to be_an(Array).and all be_a(Daru::DataFrame) }
+          its('first.index') { is_expected.to eq(Daru::Index.new(
+                ["Record high °C (°F)", "Average high °C (°F)", "Daily mean °C (°F)", "Average low °C (°F)", "Record low °C (°F)", "Average rainfall mm (inches)", "Average rainy days", "Average relative humidity (%)", "Mean monthly sunshine hours", "Mean daily sunshine hours"]
+              )
+            )
+          }
+
+        end
+      end
+
+      context "with valid html table markups" do
+        let(:path) { "file://#{Dir.pwd}/spec/fixtures/valid_markup.html" }
+        let(:index) { ["W","X","Y","Z"] }
+        let(:name) { "Small HTML table with index" }
+
+        context "returns user-modified dataframe" do
+          subject { Daru::DataFrame.from_html(path, index: index, name: name) }
+
+          it { is_expected.to be_an(Array).and all be_a(Daru::DataFrame) }
+          its(:first) { is_expected.to eq(Daru::DataFrame.new(
+                [["6", "4","9","7"],["7","0","4","0"]],
+                order: ["a","b"],
+                index: ["W","X","Y","Z"],
+                name: "Small HTML table with index"
+              )
+            )
+          }
+        end
+      end
+
+      context "in year-wise passengers figure" do
+        let(:path) { "file://#{Dir.pwd}/spec/fixtures/macau.html" }
+        let(:match) { "2001" }
+        let(:name) { "Year-wise Passengers Figure" }
+
+        context "returns matching dataframes with index" do
+          subject { Daru::DataFrame.from_html(path, match: match, name: name) }
+
+          it { is_expected.to be_an(Array).and all be_a(Daru::DataFrame) }
+          its('first.index') { is_expected.to eq(Daru::Index.new(
+                ["January","February","March","April","May","June","July","August","September","October","November","December","Total"]
+              )
+            )
+          }
+          its(:first) { is_expected.to eq(Daru::DataFrame.new(
+                [
+                  ["265,603","184,381","161,264","161,432","117,984",""],
+                  ["249,259","264,066","209,569","168,777","150,772",""],
+                  ["312,319","226,483","186,965","172,060","149,795",""],
+                  ["351,793","296,541","237,449","180,241","179,049",""],
+                  ["338,692","288,949","230,691","172,391","189,925",""],
+                  ["332,630","271,181","231,328","157,519","175,402",""],
+                  ["344,658","304,276","243,534","205,595","173,103",""],
+                  ["360,899","300,418","257,616","241,140","178,118",""],
+                  ["291,817","280,803","210,885","183,954","163,385",""],
+                  ["327,232","298,873","231,251","205,726","176,879",""],
+                  ["315,538","265,528","228,637","181,677","146,804",""],
+                  ["314,866","257,929","210,922","183,975","151,362",""],
+                  ["3,805,306","3,239,428","2,640,111","2,214,487","1,952,578","0"]
+                ].transpose,
+                order: ["2001","2000","1999","1998","1997","1996"],
+                index: ["January","February","March","April","May","June","July","August","September","October","November","December","Total"],
+                name: "Year-wise Passengers Figure"
+              )
+            )
+          }
+        end
+      end
+
+      context "in share market data" do
+        let(:path) { "file://#{Dir.pwd}/spec/fixtures/moneycontrol.html" }
+        let(:match) { "Sun Pharma" }
+        let(:index) { ["Alpha", "Beta", "Gamma", "Delta", "Misc"] }
+        let(:name) { "Share Market Analysis" }
+
+        context "returns matching dataframes" do
+          subject { Daru::DataFrame.from_html(path, match: match) }
+
+          it { is_expected.to be_an(Array).and all be_a(Daru::DataFrame) }
+          its(:first) { is_expected.to eq(Daru::DataFrame.new(
+              [
+                ["Sun Pharma","502.60","-65.05","2,117.87"],
+                ["Reliance","1356.90","19.60","745.10"],
+                ["Tech Mahindra","379.45","-49.70","650.22"],
+                ["ITC","315.85","6.75","621.12"],
+                ["HDFC","1598.85","50.95","553.91"]
+              ].transpose,
+              order: ["Company","Price","Change","Value (Rs Cr.)"]
+              )
+            )
+          }
+        end
+
+        context "returns user-modified matching dataframes" do
+          subject { Daru::DataFrame.from_html(path, match: match, index: index, name: name) }
+
+          it { is_expected.to be_an(Array).and all be_a(Daru::DataFrame) }
+          its(:last) { is_expected.to eq(Daru::DataFrame.new(
+                [
+                  ["Sun Pharma","502.60","-65.05","2,117.87"],
+                  ["Reliance","1356.90","19.60","745.10"],
+                  ["Tech Mahindra","379.45","-49.70","650.22"],
+                  ["ITC","315.85","6.75","621.12"],
+                  ["HDFC","1598.85","50.95","553.91"]
+                ].transpose,
+                order: ["Company","Price","Change","Value (Rs Cr.)"],
+                index: ["Alpha", "Beta", "Gamma", "Delta", "Misc"],
+                name: "Share Market Analysis"
+              )
+            )
+          }
+        end
+
+      end
+
+      context "in election results data" do
+        let(:path) { "file://#{Dir.pwd}/spec/fixtures/eciresults.html" }
+
+        context "returns default dataframes" do
+          subject { Daru::DataFrame.from_html(path) }
+
+          it { is_expected.to be_an(Array).and all be_a(Daru::DataFrame) }
+          its('first.vectors') { is_expected.to eq(Daru::Index.new(
+                ["PartyName", "Votes Wise(%)"]
+              )
+            )
+          }
+        end
+      end
+    end
   end
 
   describe Daru::Vector do