daru 0.1.5 → 0.1.6

data/lib/daru/dataframe.rb

@@ -20,7 +20,7 @@ module Daru
       #
       # == Arguments
       #
-      # * path - Path of the file to load specified as a String.
+      # * path - Local path / Remote URL of the file to load specified as a String.
       #
       # == Options
       #
@@ -63,7 +63,7 @@ module Daru
 
       # Read a database query and returns a Dataset
       #
-      # @param dbh [DBI::DatabaseHandle] A DBI connection to be used to run the query
+      # @param dbh [DBI::DatabaseHandle, String] A DBI connection OR Path to a SQlite3 database.
       # @param query [String] The query to be executed
       #
       # @return A dataframe containing the data resulting from the query
@@ -72,6 +72,11 @@ module Daru
       #
       #  dbh = DBI.connect("DBI:Mysql:database:localhost", "user", "password")
       #  Daru::DataFrame.from_sql(dbh, "SELECT * FROM test")
+      #
+      #  #Alternatively
+      #
+      #  require 'dbi'
+      #  Daru::DataFrame.from_sql("path/to/sqlite.db", "SELECT * FROM test")
       def from_sql dbh, query
         Daru::IO.from_sql dbh, query
       end
@@ -112,6 +117,49 @@ module Daru
         Daru::IO.from_plaintext path, fields
       end
 
+      # Read the table data from a remote html file. Please note that this module
+      # works only for static table elements on a HTML page, and won't work in
+      # cases where the data is being loaded into the HTML table by Javascript.
+      #
+      # By default - all <th> tag elements in the first proper row are considered
+      # as the order, and all the <th> tag elements in the first column are
+      # considered as the index.
+      #
+      # == Arguments
+      #
+      # * path [String] - URL of the target HTML file.
+      # * fields [Hash] -
+      #
+      #   +:match+ - A *String* to match and choose a particular table(s) from multiple tables of a HTML page.
+      #
+      #   +:order+ - An *Array* which would act as the user-defined order, to override the parsed *Daru::DataFrame*.
+      #
+      #   +:index+ - An *Array* which would act as the user-defined index, to override the parsed *Daru::DataFrame*.
+      #
+      #   +:name+ - A *String* that manually assigns a name to the scraped *Daru::DataFrame*, for user's preference.
+      #
+      # == Returns
+      # An Array of +Daru::DataFrame+s, with each dataframe corresponding to a
+      # HTML table on that webpage.
+      #
+      # == Usage
+      #   dfs = Daru::DataFrame.from_html("http://www.moneycontrol.com/", match: "Sun Pharma")
+      #   dfs.count
+      #   # => 4
+      #
+      #   dfs.first
+      #   #
+      #   # => <Daru::DataFrame(5x4)>
+      #   #          Company      Price     Change Value (Rs
+      #   #     0 Sun Pharma     502.60     -65.05   2,117.87
+      #   #     1   Reliance    1356.90      19.60     745.10
+      #   #     2 Tech Mahin     379.45     -49.70     650.22
+      #   #     3        ITC     315.85       6.75     621.12
+      #   #     4       HDFC    1598.85      50.95     553.91
+      def from_html path, fields={}
+        Daru::IO.from_html path, fields
+      end
+
       # Create DataFrame by specifying rows as an Array of Arrays or Array of
       # Daru::Vector objects.
       def rows source, opts={}
@@ -239,6 +287,48 @@ module Daru
     #   #  b          7          2
     #   #  c          8          3
     #   #  d          9          4
+    #
+    #   df = Daru::DataFrame.new([[1,2,3,4],[6,7,8,9]], name: :bat_man)
+    #
+    #   # =>
+    #   # #<Daru::DataFrame: bat_man (4x2)>
+    #   #             0          1
+    #   #  0          1          6
+    #   #  1          2          7
+    #   #  2          3          8
+    #   #  3          4          9
+    #
+    #   # Dataframe having Index name
+    #
+    #   df = Daru::DataFrame.new({a: [1,2,3,4], b: [6,7,8,9]}, order: [:b, :a],
+    #     index: Daru::Index.new([:a, :b, :c, :d], name: 'idx_name'),
+    #     name: :spider_man)
+    #
+    #   # =>
+    #   # <Daru::DataFrame:80766980 @name = spider_man @size = 4>
+    #   # idx_name            b          a
+    #   #        a          6          1
+    #   #        b          7          2
+    #   #        c          8          3
+    #   #        d          9          4
+    #
+    #
+    #   idx = Daru::Index.new [100, 99, 101, 1, 2], name: "s1"
+    #   => #<Daru::Index(5): s1 {100, 99, 101, 1, 2}>
+    #
+    #   df = Daru::DataFrame.new({b: [11,12,13,14,15], a: [1,2,3,4,5],
+    #     c: [11,22,33,44,55]},
+    #     order: [:a, :b, :c],
+    #     index: idx)
+    #    # =>
+    #    #<Daru::DataFrame(5x3)>
+    #    #   s1   a   b   c
+    #    #  100   1  11  11
+    #    #   99   2  12  22
+    #    #  101   3  13  33
+    #    #    1   4  14  44
+    #    #    2   5  15  55
+
     def initialize source, opts={} # rubocop:disable Metrics/MethodLength
       vectors, index = opts[:order], opts[:index] # FIXME: just keyword arges after Ruby 2.1
       @data = []
@@ -457,7 +547,7 @@ module Daru
     def dup vectors_to_dup=nil
       vectors_to_dup = @vectors.to_a unless vectors_to_dup
 
-      src = vectors_to_dup.map { |vec| @data[@vectors[vec]].dup }
+      src = vectors_to_dup.map { |vec| @data[@vectors.pos(vec)].dup }
       new_order = Daru::Index.new(vectors_to_dup)
 
       Daru::DataFrame.new src, order: new_order, index: @index.dup, name: @name, clone: true
@@ -544,7 +634,7 @@ module Daru
     #     b: [:a,  :b,          nil, Float::NAN, nil,        3,   5,   8],
     #     c: ['a',  Float::NAN, 3,   4,          3,          5,   nil, 7]
     #   }, index: 11..18)
-    #   df
+    #   df.replace_values nil, Float::NAN
     #   # => #<Daru::DataFrame(8x3)>
     #   #       a   b   c
     #   #   11   1   a   a
@@ -679,7 +769,7 @@ module Daru
     # * +axis+ - The axis to map over. Can be :vector (or :column) or :row.
     # Default to :vector.
     def map! axis=:vector, &block
-      if axis == :vector || axis == :column
+      if %i[vector column].include?(axis)
         map_vectors!(&block)
       elsif axis == :row
         map_rows!(&block)
@@ -913,7 +1003,7 @@ module Daru
 
     # creates a new vector with the data of a given field which the block returns true
     def filter_vector vec, &block
-      Daru::Vector.new each_row.select(&block).map { |row| row[vec] }
+      Daru::Vector.new(each_row.select(&block).map { |row| row[vec] })
     end
 
     # Iterates over each row and retains it in a new DataFrame if the block returns
@@ -1031,7 +1121,7 @@ module Daru
     alias :vector_missing_values :missing_values_rows
 
     def has_missing_data?
-      !!@data.any? { |vec| vec.include_values?(*Daru::MISSING_VALUES) }
+      @data.any? { |vec| vec.include_values?(*Daru::MISSING_VALUES) }
     end
     alias :flawed? :has_missing_data?
     deprecate :has_missing_data?, :include_values?, 2016, 10
@@ -1119,7 +1209,7 @@ module Daru
     #     row[:a] < 3 and row[:b] == 'b'
     #   end #=> true
     def any? axis=:vector, &block
-      if axis == :vector || axis == :column
+      if %i[vector column].include?(axis)
         @data.any?(&block)
       elsif axis == :row
         each_row do |row|
@@ -1141,7 +1231,7 @@ module Daru
     #     row[:a] < 10
     #   end #=> true
     def all? axis=:vector, &block
-      if axis == :vector || axis == :column
+      if %i[vector column].include?(axis)
         @data.all?(&block)
       elsif axis == :row
         each_row.all?(&block)
@@ -1377,7 +1467,7 @@ module Daru
     #   df.rename_vectors :a => :alpha, :c => :gamma
     #   df.vectors.to_a #=> [:alpha, :b, :gamma]
     def rename_vectors name_map
-      existing_targets = name_map.select { |k,v| k != v }.values & vectors.to_a
+      existing_targets = name_map.reject { |k,v| k == v }.values & vectors.to_a
       delete_vectors(*existing_targets)
 
       new_names = vectors.to_a.map { |v| name_map[v] ? name_map[v] : v }
@@ -1408,19 +1498,16 @@ module Daru
       Daru::DataFrame.new(arry, clone: cln, order: order, index: @index)
     end
 
-    # Generate a summary of this DataFrame with ReportBuilder.
-    def summary(method=:to_text)
-      ReportBuilder.new(no_title: true).add(self).send(method)
-    end
-
-    def report_building(b) # :nodoc: #
-      b.section(name: @name) do |g|
-        g.text "Number of rows: #{nrows}"
-        @vectors.each do |v|
-          g.text "Element:[#{v}]"
-          g.parse_element(self[v])
-        end
+    # Generate a summary of this DataFrame based on individual vectors in the DataFrame
+    # @return [String] String containing the summary of the DataFrame
+    def summary
+      summary = "= #{name}"
+      summary << "\n  Number of rows: #{nrows}"
+      @vectors.each do |v|
+        summary << "\n  Element:[#{v}]\n"
+        summary << self[v].summary(1)
       end
+      summary
     end
 
     # Sorts a dataframe (ascending/descending) in the given pripority sequence of
@@ -1783,7 +1870,9 @@ module Daru
     end
 
     # Convert to html for IRuby.
-    def to_html threshold=30
+    def to_html(threshold=30)
+      table_thead = to_html_thead
+      table_tbody = to_html_tbody(threshold)
       path = if index.is_a?(MultiIndex)
                File.expand_path('../iruby/templates/dataframe_mi.html.erb', __FILE__)
              else
@@ -1792,8 +1881,28 @@ module Daru
       ERB.new(File.read(path).strip).result(binding)
     end
 
+    def to_html_thead
+      table_thead_path =
+        if index.is_a?(MultiIndex)
+          File.expand_path('../iruby/templates/dataframe_mi_thead.html.erb', __FILE__)
+        else
+          File.expand_path('../iruby/templates/dataframe_thead.html.erb', __FILE__)
+        end
+      ERB.new(File.read(table_thead_path).strip).result(binding)
+    end
+
+    def to_html_tbody(threshold=30)
+      table_tbody_path =
+        if index.is_a?(MultiIndex)
+          File.expand_path('../iruby/templates/dataframe_mi_tbody.html.erb', __FILE__)
+        else
+          File.expand_path('../iruby/templates/dataframe_tbody.html.erb', __FILE__)
+        end
+      ERB.new(File.read(table_tbody_path).strip).result(binding)
+    end
+
     def to_s
-      to_html
+      "#<#{self.class}#{': ' + @name.to_s if @name}(#{nrows}x#{ncols})>"
     end
 
     # Method for updating the metadata (i.e. missing value positions) of the
@@ -1900,14 +2009,13 @@ module Daru
 
     # Pretty print in a nice table format for the command line (irb/pry/iruby)
     def inspect spacing=10, threshold=15
-      row_headers = index.is_a?(MultiIndex) ? index.sparse_tuples : index.to_a
       name_part = @name ? ": #{@name} " : ''
 
       "#<#{self.class}#{name_part}(#{nrows}x#{ncols})>\n" +
         Formatters::Table.format(
           each_row.lazy,
           row_headers: row_headers,
-          headers: vectors,
+          headers: headers,
           threshold: threshold,
           spacing: spacing
         )
@@ -2002,8 +2110,32 @@ module Daru
       end
     end
 
+    # returns array of row tuples at given index(s)
+    def access_row_tuples_by_indexs *indexes
+      positions = @index.pos(*indexes)
+
+      return populate_row_for(positions) if positions.is_a? Numeric
+
+      res = []
+      new_rows = @data.map { |vec| vec[*indexes] }
+      indexes.each do |index|
+        tuples = []
+        new_rows.map { |row| tuples += [row[index]] }
+        res << tuples
+      end
+      res
+    end
+
     private
 
+    def headers
+      Daru::Index.new(Array(index.name) + @vectors.to_a)
+    end
+
+    def row_headers
+      index.is_a?(MultiIndex) ? index.sparse_tuples : index.to_a
+    end
+
     def convert_categorical_vectors names
       names.map do |n|
         next unless self[n].category?
@@ -2034,7 +2166,7 @@ module Daru
     end
 
     def dispatch_to_axis(axis, method, *args, &block)
-      if axis == :vector || axis == :column
+      if %i[vector column].include?(axis)
         send("#{method}_vector", *args, &block)
       elsif axis == :row
         send("#{method}_row", *args, &block)
@@ -2044,7 +2176,7 @@ module Daru
     end
 
     def dispatch_to_axis_pl(axis, method, *args, &block)
-      if axis == :vector || axis == :column
+      if %i[vector column].include?(axis)
         send("#{method}_vectors", *args, &block)
       elsif axis == :row
         send("#{method}_rows", *args, &block)
@@ -2053,7 +2185,7 @@ module Daru
       end
     end
 
-    AXES = [:row, :vector].freeze
+    AXES = %i[row vector].freeze
 
     def extract_axis names, default=:vector
       if AXES.include?(names.last)
@@ -2065,7 +2197,7 @@ module Daru
 
     def access_vector *names
       if names.first.is_a?(Range)
-        dup(@vectors[names.first])
+        dup(@vectors.subset(names.first))
       elsif @vectors.is_a?(MultiIndex)
         access_vector_multi_index(*names)
       else
@@ -2087,14 +2219,18 @@ module Daru
 
     def access_vector_single_index *names
       if names.count < 2
-        pos = @vectors[names.first]
+        begin
+          pos = @vectors.is_a?(Daru::DateTimeIndex) ? @vectors[names.first] : @vectors.pos(names.first)
+        rescue IndexError
+          raise IndexError, "Specified vector #{names.first} does not exist"
+        end
 
         return @data[pos] if pos.is_a?(Numeric)
 
         names = pos
       end
 
-      new_vectors = names.map { |name| [name, @data[@vectors[name]]] }.to_h
+      new_vectors = names.map { |name| [name, @data[@vectors.pos(name)]] }.to_h
 
       order = names.is_a?(Array) ? Daru::Index.new(names) : names
       Daru::DataFrame.new(new_vectors, order: order,
@@ -2126,7 +2262,7 @@ module Daru
       if @index.empty?
         insert_vector_in_empty name, vector
       else
-        vec = prepare_vector_for_insert name, vector
+        vec = prepare_for_insert name, vector
 
         assign_or_add_vector name, vec
       end
@@ -2173,25 +2309,35 @@ module Daru
       @data.map! { |v| v.empty? ? v.reindex(@index) : v }
     end
 
-    def prepare_vector_for_insert name, vector
-      if vector.is_a?(Daru::Vector)
-        # so that index-by-index assignment is avoided when possible.
-        return vector.dup if vector.index == @index
-
-        Daru::Vector.new([], name: coerce_name(name), index: @index).tap { |v|
-          @index.each do |idx|
-            v[idx] = vector.index.include?(idx) ? vector[idx] : nil
-          end
-        }
+    def prepare_for_insert name, arg
+      if arg.is_a? Daru::Vector
+        prepare_vector_for_insert name, arg
+      elsif arg.respond_to?(:to_a)
+        prepare_enum_for_insert name, arg
       else
-        # FIXME: No spec checks this case... And SizeError is not a thing - zverok, 2016-05-08
-        if @size != vector.size
-          raise SizeError,
-            "Specified vector of length #{vector.size} cannot be inserted in DataFrame of size #{@size}"
+        prepare_value_for_insert name, arg
+      end
+    end
+
+    def prepare_vector_for_insert name, vector
+      # so that index-by-index assignment is avoided when possible.
+      return vector.dup if vector.index == @index
+      Daru::Vector.new([], name: coerce_name(name), index: @index).tap { |v|
+        @index.each do |idx|
+          v[idx] = vector.index.include?(idx) ? vector[idx] : nil
         end
+      }
+    end
 
-        Daru::Vector.new(vector, name: coerce_name(name), index: @index)
+    def prepare_enum_for_insert name, enum
+      if @size != enum.size
+        raise "Specified vector of length #{enum.size} cannot be inserted in DataFrame of size #{@size}"
       end
+      Daru::Vector.new(enum, name: coerce_name(name), index: @index)
+    end
+
+    def prepare_value_for_insert name, value
+      Daru::Vector.new(Array(value) * @size, name: coerce_name(name), index: @index)
     end
 
     def insert_or_modify_row indexes, vector
@@ -2276,8 +2422,10 @@ module Daru
 
       case source.first
       when Array
+        vectors ||= (0..source.size-1).to_a
         initialize_from_array_of_arrays source, vectors, index, opts
       when Vector
+        vectors ||= (0..source.size-1).to_a
         initialize_from_array_of_vectors source, vectors, index, opts
       when Hash
         initialize_from_array_of_hashes source, vectors, index, opts

data/lib/daru/date_time/index.rb

@@ -226,7 +226,7 @@ module Daru
       to_a.each(&block)
     end
 
-    attr_reader :frequency, :offset, :periods
+    attr_reader :frequency, :offset, :periods, :keys
 
     # Create a DateTimeIndex with or without a frequency in data. The constructor
     # should be used for creating DateTimeIndex by directly passing in DateTime
@@ -253,6 +253,7 @@ module Daru
     #     DateTime.new(2012,4,11), DateTime.new(2012,4,12)], freq: :infer)
     #   #=>#<DateTimeIndex:84198340 offset=D periods=8 data=[2012-04-05T00:00:00+00:00...2012-04-12T00:00:00+00:00]>
     def initialize data, opts={freq: nil}
+      super data
       Helper.possibly_convert_to_date_time data
 
       @offset =
@@ -405,7 +406,7 @@ module Daru
         @data
       else
         @data.sort_by(&:last)
-      end.transpose.first
+      end.transpose.first || []
     end
 
     # Size of index.
@@ -419,6 +420,7 @@ module Daru
 
     def inspect
       meta = [@periods, @frequency ? "frequency=#{@frequency}" : nil].compact.join(', ')
+      return "#<#{self.class}(#{meta})>" if @data.empty?
       "#<#{self.class}(#{meta}) " \
          "#{@data.first[0]}...#{@data.last[0]}>"
     end
@@ -490,7 +492,7 @@ module Daru
     #   @return [Array<Integer>] Array containing minutes of each index.
     # @!method sec
     #   @return [Array<Integer>] Array containing seconds of each index.
-    [:year, :month, :day, :hour, :min, :sec].each do |meth|
+    %i[year month day hour min sec].each do |meth|
       define_method(meth) do
         each_with_object([]) do |d, arr|
           arr << d.send(meth)
@@ -528,7 +530,7 @@ module Daru
       slice first, last
     end
 
-    def slice_between_dates first, last # rubocop:disable Metrics/AbcSize
+    def slice_between_dates first, last # rubocop:disable Metrics/AbcSize,Metrics/PerceivedComplexity
       # about that ^ disable: I'm waiting for cleaner understanding
       # of offsets logic. Reference: https://github.com/v0dro/daru/commit/7e1c34aec9516a9ba33037b4a1daaaaf1de0726a#diff-a95ef410a8e1f4ea3cc48d231bb880faR250
       start    = @data.bsearch { |d| d[0] >= first }
@@ -542,7 +544,7 @@ module Daru
         st = @data.index(start)
         en = after_en ? @data.index(after_en) - 1 : Helper.last_date(@data)[1]
         return start[1] if st == en
-        DateTimeIndex.new(@data[st..en].transpose[0])
+        DateTimeIndex.new(@data[st..en].transpose[0] || []) # empty slice guard
       end
     end