daru 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b584977ade16d5cfbd3bdd9da7202bbc2adae894
-  data.tar.gz: 3de8ef80fc55c91c0f504543fda1f23b40b4db6b
+  metadata.gz: 10338f8d554cc2c70b2dcc2d8fd029e73446f4de
+  data.tar.gz: b848b5923eebe90577ef93d4c9e988d7aa09fe9b
 SHA512:
-  metadata.gz: 3f498f5caa4dbc14c2ca884c45492e474442d3f067ecf673a0235f01ec1781bb96c6f6e764006dcbf349a68ea011ccd977d69fdeb324a202ffa4fc5351a3a7b9
-  data.tar.gz: d209e60b566e0e900d9c62fca6a1fb9023cd09cd81194e9b331c59682f557d8917ca347978bd4ed9b614c8d1ba441126211f3dc15a691752733f5dca0200be5d
+  metadata.gz: ad6c0de4217a65a2c6245c4b969e98d970da47a08ce8128b060cd06a13ae415df6d5e7fdc7bba4c84115fc8559e80bea0a39c87efb1c3401df358f4df5d43117
+  data.tar.gz: 34abe4afd0c88c24d3d0bbd6f11df5efe554f50b0d3d045a8a969f4eae2232d8d1cfeead814e336da507ca94b3126447631699258fe1a5dc5ea736de77c587f8

data/.rubocop.yml

@@ -2,6 +2,10 @@ AllCops:
   Include:
     - 'lib/**/*'
   Exclude:
+    - 'daru.gemspec'
+    - 'Rakefile'
+    - 'Gemfile'
+    - 'Guardfile'
     - '**/*.erb'
     - 'spec/*'
     - 'spec/**/*'
@@ -12,7 +16,7 @@ AllCops:
   TargetRubyVersion: 2.0
 
 # Preferred codebase style ---------------------------------------------
-Style/ExtraSpacing:
+Layout/ExtraSpacing:
   AllowForAlignment: true
 
 Style/FormatString:
@@ -21,16 +25,16 @@ Style/FormatString:
 Style/AndOr:
   EnforcedStyle: conditionals
 
-Style/SpaceAroundEqualsInParameterDefault:
+Layout/SpaceAroundEqualsInParameterDefault:
   EnforcedStyle: no_space
 
-Style/SpaceInsideBlockBraces:
+Layout/SpaceInsideBlockBraces:
   EnforcedStyle: space
 
-Style/SpaceInsideHashLiteralBraces:
+Layout/SpaceInsideHashLiteralBraces:
   EnforcedStyle: no_space
 
-Style/AlignParameters:
+Layout/AlignParameters:
   EnforcedStyle: with_fixed_indentation
 
 Style/EmptyElse:
@@ -57,10 +61,10 @@ Style/SingleLineBlockParams:
 Style/PerlBackrefs:
   Enabled: false
 
-Style/SpaceAfterComma:
+Layout/SpaceAfterComma:
   Enabled: false
 
-Style/SpaceAroundOperators:
+Layout/SpaceAroundOperators:
   Enabled: false
 
 Style/EmptyCaseCondition:
@@ -69,6 +73,15 @@ Style/EmptyCaseCondition:
 Style/MultilineBlockChain:
   Enabled: false
 
+# See https://github.com/bbatsov/rubocop/issues/4429
+Style/YodaCondition:
+  Enabled: false
+
+Style/PercentLiteralDelimiters:
+  PreferredDelimiters:
+    '%i': '[]'
+    '%w': '[]'
+
 # Neither of prefered styles are good enough :(
 Style/BlockDelimiters:
   Enabled: false

data/CONTRIBUTING.md

@@ -31,7 +31,7 @@ While preparing your pull requests, don't forget to check your code with Rubocop
 ## Basic Development Flow
 
 1. Create a new branch with `git checkout -b <branch_name>`.
-2. Make your changes.
+2. Make your changes. Write tests covering every case how your feature will be used. If creating new files for tests, refer to the 'Testing' section [below](#Testing).
 3. Try out these changes with `rake pry`.
 4. Run the test suite with `rake spec`. (Alternatively you can use `guard` as described [here](https://github.com/SciRuby/daru/blob/master/CONTRIBUTING.md#testing). Also run Rubocop coding style guidelines with `rake cop`.
 5. Commit the changes with `git commit -am "briefly describe what you did"` and submit pull request.

data/History.md

@@ -1,3 +1,50 @@
+# 0.1.6 (04 August 2017)
+* Major Enhancements
+  - Add support for reading HTML tables into DataFrames (@athityakumar)
+  - Add support for importing remote CSVs (@athityakumar, @anshuman23)
+  - Allow named indexes (@Shekharrajak)
+  - DataFrame GroupBy returns MultiIndex DataFrame (@Shekharrajak)
+  - Add new functions to Vector: max, min, index_of_max, index_of_min, max_by, min_by, index_of_max_by, index_of_min_by (@athityakumar)
+  - Add summary to DataFrame and Vector without reportbuilder (@ananyo2012)
+  - Add support for missing data for where clause (@athityakumar)
+
+* Minor Enhancements
+  - Allow inserting or updating DataFrame vectors with single values (@baarkerlounger)
+  - Add a boolean converter to the CSV importer (@baarkerlounger)
+  - Fix documentation of replace_values method (@kojix2)
+  - Improve HTML table code of DataFrame and Vector (@Shekharrajak  )
+  - Support CSV files with empty rows (@baarkerlounger)
+  - Better DataFrame and Vector to_s methods (@baarkerlounger)
+  - Add support for histogram to Vector moving average convergence-divergence (@parthm)
+  - Add support for negative arguments to Vector.lag (@parthm)
+  - Return Nyaplot instance instead of nil for Nyaplot Vector, Category and DataFrame (@Shekharrajak)
+  - Add global configurable error stream which allows error stream to be silenced (@sivagollapalli)
+  - Rubocop update and cleanup (@zverok)
+  - Improve performance of DataFrame covariance (@genya0407)
+  - Index [] to only take index value as argument (@ananyo2012)
+  - Better error raised when Vector is missing from DataFrame (@sivagollapalli)
+  - Add default order for DataFrame (@athityakumar)
+  - Add is_values to Index (@Shekharrajak)
+  - Improve spec style in IO/SQL data source spec (@dshvimer)
+  - Open SQLite databases by bath (@dshvimer)
+  - Remove unnecessary whitespace (@Shekharrajak)
+  - Remove the .svg from Travis CI build link (@athityakumar)
+  - Fix Travis CI icon in README (@athityakumar)
+  - Replace is_nil?, not_nil? with is_values (@lokeshh)
+  - Update contributing documentation (@v0dro)
+
+* Fixes
+  - Fix missing axis labels for categorized scatter plot with Gruff (@xprazak2)
+  - Fix NMatrix Vector initialization when Vector has nils and no nm_type is given (@baarkerlounger)
+  - Fix head/tail methods on DataFrames with DateTime indexes and on Vector_at splat calls (@baarkerlounger)
+  - Fix empty DateTime Index (@zverok)
+  - Fix where clause when data contains missing/undefined values (@Shekharrajak)
+  - Fix apply_scalar_operator spec (@athityakumar)
+  - Change nil check to respond_to operator check for apply_scalar_operator (@athityakumar)
+  - Make where compatible with is_values (@athityakumar)
+  - Fix vector is_values method (@athityakumar)
+
+
 # 0.1.5 (30 January 2017)
 * Major Enhancements
   - Add Daru::Vector#group_by (@lokeshh).
@@ -19,7 +66,7 @@
   - `DataFrame#access_row` with integer index. (Yusuke Sangenya)
   - Add method alias for comparison operator. (Yusuke Sangenya)
   - Update Nokogiri version. (Yusuke Sangenya)
-  - Return `Daru::Vector` for multiple modal values for `Daru::Vector#mode`. (baarkerlounger)
+  - Return `Daru::Vector` for multiple modal values for `Daru::Vector#mode`. (@baarkerlounger)
 
 * Fixes
   - Fix many to one joins. The prior version was shifting values in the left dataframe before checking whether values in the right dataframe should be shifted.  They both need to be checked at the same time before shifting either. (@gnilrets)

data/README.md

@@ -1,7 +1,7 @@
 # daru - Data Analysis in RUby
 
 [![Gem Version](https://badge.fury.io/rb/daru.svg)](http://badge.fury.io/rb/daru)
-[![Build Status](https://travis-ci.org/v0dro/daru.svg)](https://travis-ci.org/v0dro/daru)
+[![Build Status](https://travis-ci.org/SciRuby/daru.svg?branch=master)](https://travis-ci.org/SciRuby/daru)
 [![Gitter](https://badges.gitter.im/v0dro/daru.svg)](https://gitter.im/v0dro/daru?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 
 ## Introduction
@@ -168,7 +168,7 @@ data_frame.where(
 ```
 ![con1](images/con1.png)
 
-*Plotting* 
+*Plotting*
 
 Daru supports plotting of interactive graphs with [nyaplot](https://github.com/domitry/nyaplot). You can easily create a plot with the `#plot` method. Here we plot the gallons sold on the Y axis and name of the brand on the X axis in a bar graph.
 ``` ruby
@@ -196,7 +196,7 @@ For details see [CONTRIBUTING](https://github.com/v0dro/daru/blob/master/CONTRIB
 
 ## Acknowledgements
 
-* Google and the Ruby Science Foundation for the Google Summer of Code 2016 grant for speed enhancements and implementation of support for categorical data. Special thanks to [@lokeshh](https://github.com/lokeshh), [@zverok](https://github.com/zverok) and [@agisga](https://github.com/agisga) for their efforts. 
+* Google and the Ruby Science Foundation for the Google Summer of Code 2016 grant for speed enhancements and implementation of support for categorical data. Special thanks to [@lokeshh](https://github.com/lokeshh), [@zverok](https://github.com/zverok) and [@agisga](https://github.com/agisga) for their efforts.
 * Google and the Ruby Science Foundation for the Google Summer of Code 2015 grant for further developing daru and integrating it with other ruby gems.
 * Thank you [last.fm](http://www.last.fm/) for making user data accessible to the public.
 

data/benchmarks/statistics.rb

@@ -5,26 +5,26 @@ vector = Daru::Vector.new(
   (10**6).times.map.to_a.shuffle,
   missing_values: 100.times.map.to_a.shuffle
   )
-  
+
 vector_gsl = Daru::Vector.new(
   10000.times.map.to_a.shuffle,
   missing_values: 100.times.map.to_a.shuffle,
   dtype: :gsl
-  )  
-  
+  )
+
 Benchmark.bm do |x|
   x.report("Mean of a vector") do
     vector.mean
   end
-  
+
   x.report("Minimum of a vector") do
     vector.min
   end
-  
+
   x.report("Mean of a vector with data type gsl") do
     vector_gsl.mean
   end
-  
+
   x.report "Minimum of a vector with data type gsl" do
     vector_gsl.min
   end

data/benchmarks/where_clause.rb

@@ -35,7 +35,7 @@ Benchmark.bm do |x|
 end
 
 #                     ====== Benchmarks ======
-# 
+#
 # Benchmarking DataFrame#where
 #
 #                             user     system      total      real

data/benchmarks/where_vs_filter.rb

@@ -22,7 +22,7 @@ Benchmark.bm do |x|
 end
 
 #             ===== Benchmarks =====
-# 
+#
 #                user     system      total        real
 # where        0.000000   0.000000   0.000000 (  0.002575)
 # filter_rows  0.210000   0.000000   0.210000 (  0.205403)

data/daru.gemspec

@@ -52,7 +52,6 @@ EOF
 
   spec.add_runtime_dependency 'backports'
 
-  spec.add_development_dependency 'reportbuilder', '~> 1.4'
   spec.add_development_dependency 'spreadsheet', '~> 1.1.1'
   spec.add_development_dependency 'bundler', '~> 1.10'
   spec.add_development_dependency 'rake', '~>10.5'
@@ -69,11 +68,13 @@ EOF
   spec.add_development_dependency 'dbd-sqlite3'
   spec.add_development_dependency 'dbi'
   spec.add_development_dependency 'activerecord', '~> 4.0'
+  spec.add_development_dependency 'mechanize'
   spec.add_development_dependency 'sqlite3'
-  spec.add_development_dependency 'rubocop', '>= 0.40.0'
+  spec.add_development_dependency 'rubocop', '~> 0.49.0'
   spec.add_development_dependency 'ruby-prof'
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'gruff'
+  spec.add_development_dependency 'webmock'
   if RUBY_VERSION < '2.1.0'
     spec.add_development_dependency 'nokogiri', '<= 1.6.8.1'
   else

data/lib/daru.rb

@@ -38,11 +38,13 @@ module Daru
 
   @plotting_library = :nyaplot
 
+  @error_stream = $stderr
+
   class << self
     # A variable which will set whether Vector metadata is updated immediately or lazily.
     # Call the #update method every time a values are set or removed in order to update
     # metadata like positions of missing values.
-    attr_accessor :lazy_update
+    attr_accessor :lazy_update, :error_stream
     attr_reader :plotting_library
 
     def create_has_library(library)
@@ -72,6 +74,10 @@ module Daru
         raise ArgumentError, "Unsupported library #{lib}"
       end
     end
+
+    def error msg
+      error_stream.puts msg if error_stream
+    end
   end
 
   create_has_library :gsl
@@ -80,13 +86,13 @@ module Daru
   create_has_library :gruff
 end
 
-[['reportbuilder', '~>1.4'], ['spreadsheet', '~>1.1.1']].each do |lib|
+{'spreadsheet' => '~>1.1.1', 'mechanize' => '~>2.7.5'}.each do |name, version|
   begin
-    gem lib[0], lib[1]
-    require lib[0]
+    gem name, version
+    require name
   rescue LoadError
-    STDERR.puts "\nInstall the #{lib[0]} gem version #{lib[1]} for using"\
-    " #{lib[0]} functions."
+    Daru.error "\nInstall the #{name} gem version #{version} for using"\
+    " #{name} functions."
   end
 end
 
@@ -98,6 +104,8 @@ require 'date'
 
 require 'daru/version.rb'
 
+require 'open-uri'
+
 require 'daru/index/index.rb'
 require 'daru/index/multi_index.rb'
 require 'daru/index/categorical_index.rb'

data/lib/daru/accessors/gsl_wrapper.rb

@@ -65,7 +65,7 @@ if Daru.has_gsl?
           ::GSL::Vector.alloc(@data.to_a - [Float::NAN])
         end
 
-        [:mean, :min, :max, :prod, :sum].each do |method|
+        %i[mean min max prod sum].each do |method|
           define_method(method) do
             compact.send(method.to_sym) rescue nil
           end

data/lib/daru/accessors/nmatrix_wrapper.rb

@@ -25,6 +25,8 @@ if Daru.has_nmatrix?
         attr_reader :size, :data, :nm_dtype
 
         def initialize vector, context, nm_dtype=:int32
+          # To avoid arrays with nils throwing TypeError for nil nm_dtype
+          nm_dtype = :object if nm_dtype.nil? && vector.any?(&:nil?)
           @size = vector.size
           @data = NMatrix.new [@size*2], vector.to_a, dtype: nm_dtype
           @context = context

data/lib/daru/category.rb

@@ -461,7 +461,7 @@ module Daru
       @coding_scheme = scheme
     end
 
-    CODING_SCHEMES = [:dummy, :deviation, :helmert, :simple].freeze
+    CODING_SCHEMES = %i[dummy deviation helmert simple].freeze
 
     # Contrast code the vector acording to the coding scheme set.
     # @note To set the coding scheme use #coding_scheme=

data/lib/daru/core/group_by.rb

@@ -1,7 +1,7 @@
 module Daru
   module Core
     class GroupBy
-      attr_reader :groups
+      attr_reader :groups, :df
 
       # Iterate over each group created by group_by. A DataFrame is yielded in
       # block.
@@ -32,12 +32,7 @@ module Daru
         #   #  => {4=>["test"], 2=>["me"], 6=>["please"]}
         #
         # - zverok, 2016-09-12
-        keys    = tuples.uniq.sort(&TUPLE_SORTER)
-
-        keys.each do |key|
-          @groups[key] = all_indices_for(tuples, key)
-        end
-        @groups.freeze
+        init_groups_df tuples, names
       end
 
       # Get a Daru::Vector of the size of each group.
@@ -201,15 +196,8 @@ module Daru
         transpose = elements.transpose
         rows      = indexes.each.map { |idx| transpose[idx] }
 
-        new_index =
-          begin
-            @context.index[indexes]
-          rescue IndexError
-            indexes
-          end
-
         Daru::DataFrame.rows(
-          rows, index: new_index, order: @context.vectors
+          rows, index: indexes, order: @context.vectors
         )
       end
 
@@ -251,8 +239,26 @@ module Daru
         Daru::Vector.new(result_hash.values, index: index)
       end
 
+      def inspect
+        @df.inspect
+      end
+
       private
 
+      def init_groups_df tuples, names
+        multi_index_tuples = []
+        keys = tuples.uniq.sort(&TUPLE_SORTER)
+        keys.each do |key|
+          indices = all_indices_for(tuples, key)
+          @groups[key] = indices
+          indices.each do |indice|
+            multi_index_tuples << key + [indice]
+          end
+        end
+        @groups.freeze
+        @df = resultant_context(multi_index_tuples, names) unless multi_index_tuples.empty?
+      end
+
       def select_groups_from method, quantity
         selection     = @context
         rows, indexes = [], []
@@ -293,6 +299,17 @@ module Daru
         end
       end
 
+      def resultant_context(multi_index_tuples, names)
+        multi_index = Daru::MultiIndex.from_tuples(multi_index_tuples)
+        context_tmp = @context.dup.delete_vectors(*names)
+        rows_tuples = context_tmp.access_row_tuples_by_indexs(
+          *@groups.values.flatten!
+        )
+        context_new = Daru::DataFrame.rows(rows_tuples, index: multi_index)
+        context_new.vectors = context_tmp.vectors
+        context_new
+      end
+
       def all_indices_for arry, element
         found, index, indexes = -1, -1, []
         while found

data/lib/daru/core/query.rb

@@ -9,13 +9,13 @@ module Daru
         end
 
         def & other
-          BoolArray.new @barry.zip(other.barry).map { |b, o| b && o }
+          BoolArray.new(@barry.zip(other.barry).map { |b, o| b && o })
         end
 
         alias :and :&
 
         def | other
-          BoolArray.new @barry.zip(other.barry).map { |b, o| b || o }
+          BoolArray.new(@barry.zip(other.barry).map { |b, o| b || o })
         end
 
         alias :or :|
@@ -39,11 +39,11 @@ module Daru
 
       class << self
         def apply_scalar_operator operator, data, other
-          BoolArray.new data.map { |d| !!d.send(operator, other) }
+          BoolArray.new(data.map { |d| !!d.send(operator, other) if d.respond_to?(operator) })
         end
 
         def apply_vector_operator operator, vector, other
-          BoolArray.new vector.zip(other).map { |d, o| !!d.send(operator, o) }
+          BoolArray.new(vector.zip(other).map { |d, o| !!d.send(operator, o) })
         end
 
         def df_where data_frame, bool_array