daru 0.1.5 → 0.1.6

data/lib/daru/formatters/table.rb

@@ -10,6 +10,7 @@ module Daru
         @data = data || []
         @headers = (headers || []).to_a
         @row_headers = (row_headers || []).to_a
+        @row_headers = [''] * @data.to_a.size if @row_headers.empty?
       end
 
       DEFAULT_SPACING = 10

data/lib/daru/index/index.rb

@@ -1,5 +1,5 @@
 module Daru
-  class Index
+  class Index # rubocop:disable Metrics/ClassLength
     include Enumerable
     # It so happens that over riding the .new method in a super class also
     # tampers with the default .new method for class that inherit from the
@@ -44,24 +44,31 @@ module Daru
     end
 
     attr_reader :relation_hash, :size
+    attr_accessor :name
 
-    def initialize index
-      index =
-        case index
-        when nil
-          []
-        when Integer
-          index.times.to_a
-        when Enumerable
-          index.to_a
-        else
-          raise ArgumentError,
-            "Cannot create index from #{index.class} #{index.inspect}"
-        end
-
+    # @example
+    #
+    #   idx = Daru::Index.new [:one, 'one', 1, 2, :two]
+    #   => #<Daru::Index(5): {one, one, 1, 2, two}>
+    #
+    #   # set the name
+    #
+    #   idx.name = "index_name"
+    #   => "index_name"
+    #
+    #   idx
+    #   => #<Daru::Index(5): index_name {one, one, 1, 2, two}>
+    #
+    #   # set the name during initialization
+    #
+    #   idx = Daru::Index.new [:one, 'one', 1, 2, :two], name: "index_name"
+    #   => #<Daru::Index(5): index_name {one, one, 1, 2, two}>
+    def initialize index, opts={}
+      index = guess_index index
       @relation_hash = index.each_with_index.to_h.freeze
       @keys = @relation_hash.keys
       @size = @relation_hash.size
+      @name = opts[:name]
     end
 
     def ==(other)
@@ -106,21 +113,24 @@ module Daru
       indexes = preprocess_range(indexes.first) if indexes.first.is_a? Range
 
       if indexes.size == 1
-        self[indexes.first]
+        numeric_pos indexes.first
       else
-        indexes.map { |index| by_single_key index }
+        indexes.map { |index| numeric_pos index }
       end
     end
 
     def subset *indexes
       if indexes.first.is_a? Range
-        slice indexes.first.begin, indexes.first.end
+        start = indexes.first.begin
+        en = indexes.first.end
+
+        subset_slice start, en
       elsif include? indexes.first
         # Assume 'indexes' contain indexes not positions
         Daru::Index.new indexes
       else
         # Assume 'indexes' contain positions not indexes
-        Daru::Index.new indexes.map { |k| key k }
+        Daru::Index.new(indexes.map { |k| key k })
       end
     end
 
@@ -143,10 +153,11 @@ module Daru
     end
 
     def inspect threshold=20
+      name_part = @name ? "#{@name} " : ''
       if size <= threshold
-        "#<#{self.class}(#{size}): {#{to_a.join(', ')}}>"
+        "#<#{self.class}(#{size}): #{name_part}{#{to_a.join(', ')}}>"
       else
-        "#<#{self.class}(#{size}): {#{to_a.first(threshold).join(', ')} ... #{to_a.last}}>"
+        "#<#{self.class}(#{size}): #{name_part}{#{to_a.first(threshold).join(', ')} ... #{to_a.last}}>"
       end
     end
 
@@ -154,12 +165,27 @@ module Daru
       start = args[0]
       en = args[1]
 
+      start_idx = @relation_hash[start]
+      en_idx    = @relation_hash[en]
+
+      if start_idx.nil?
+        nil
+      elsif en_idx.nil?
+        Array(start_idx..size-1)
+      else
+        Array(start_idx..en_idx)
+      end
+    end
+
+    def subset_slice *args
+      start = args[0]
+      en = args[1]
+
       if start.is_a?(Integer) && en.is_a?(Integer)
         Index.new @keys[start..en]
       else
         start_idx = @relation_hash[start]
         en_idx    = @relation_hash[en]
-
         Index.new @keys[start_idx..en_idx]
       end
     end
@@ -187,6 +213,27 @@ module Daru
       @relation_hash.key? index
     end
 
+    # @note Do not use it to check for Float::NAN as
+    #   Float::NAN == Float::NAN is false
+    # Return vector of booleans with value at ith position is either
+    # true or false depending upon whether index value at position i is equal to
+    # any of the values passed in the argument or not
+    # @param [Array] *indexes values to equate with
+    # @return [Daru::Vector] vector of boolean values
+    # @example
+    #   dv = Daru::Index.new [1, 2, 3, :one, 'one']
+    #   dv.is_values 1, 'one'
+    #   # => #<Daru::Vector(5)>
+    #   #     0  true
+    #   #     1  false
+    #   #     2  false
+    #   #     3  false
+    #   #     4  true
+    def is_values(*indexes) # rubocop:disable Style/PredicateName
+      bool_array = @relation_hash.keys.map { |r| indexes.include?(r) }
+      Daru::Vector.new(bool_array)
+    end
+
     def empty?
       @relation_hash.empty?
     end
@@ -222,8 +269,47 @@ module Daru
       self.class.new(new_order.map { |i| from[i] })
     end
 
+    # Sorts a `Index`, according to its values. Defaults to ascending order
+    # sorting.
+    #
+    # @param [Hash] opts the options for sort method.
+    # @option opts [Boolean] :ascending False, to get descending order.
+    #
+    # @return [Index] sorted `Index` according to its values.
+    #
+    # @example
+    #   di = Daru::Index.new [100, 99, 101, 1, 2]
+    #   # Say you want to sort in descending order
+    #   di.sort(ascending: false) #=> Daru::Index.new [101, 100, 99, 2, 1]
+    #   # Say you want to sort in ascending order
+    #   di.sort #=> Daru::Index.new [1, 2, 99, 100, 101]
+    def sort opts={}
+      opts = {ascending: true}.merge(opts)
+      if opts[:ascending]
+        new_index, = @relation_hash.sort.transpose
+      else
+        new_index, = @relation_hash.sort.reverse.transpose
+      end
+
+      self.class.new(new_index)
+    end
+
     private
 
+    def guess_index index
+      case index
+      when nil
+        []
+      when Integer
+        index.times.to_a
+      when Enumerable
+        index.to_a
+      else
+        raise ArgumentError,
+          "Cannot create index from #{index.class} #{index.inspect}"
+      end
+    end
+
     def preprocess_range rng
       start   = rng.begin
       en      = rng.end
@@ -243,22 +329,14 @@ module Daru
     end
 
     def by_multi_key *key
-      if include? key[0]
-        Daru::Index.new key.map { |k| k }
-      else
-        # Assume the user is specifing values for index not keys
-        # Return index object having keys corresponding to values provided
-        Daru::Index.new key.map { |k| key k }
-      end
+      key.map { |k| by_single_key k }
     end
 
     def by_single_key key
       if @relation_hash.key?(key)
         @relation_hash[key]
-      elsif key.is_a?(Numeric) && key < size
-        key
       else
-        raise IndexError, "Specified index #{key.inspect} does not exist"
+        nil
       end
     end
 
@@ -266,7 +344,7 @@ module Daru
     def validate_positions *positions
       positions = [positions] if positions.is_a? Integer
       positions.each do |pos|
-        raise IndexError, "#{pos} is not a valid position." if pos >= size
+        raise IndexError, "#{pos} is not a valid position." if pos >= size || pos < -size
       end
     end
 
@@ -285,5 +363,15 @@ module Daru
         positions
       end
     end
+
+    def numeric_pos key
+      if @relation_hash.key?(key)
+        @relation_hash[key]
+      elsif key.is_a?(Numeric) && (key < size && key >= -size)
+        key
+      else
+        raise IndexError, "Specified index #{key.inspect} does not exist"
+      end
+    end
   end
 end

data/lib/daru/index/multi_index.rb

@@ -1,5 +1,5 @@
 module Daru
-  class MultiIndex < Index
+  class MultiIndex < Index # rubocop:disable Metrics/ClassLength
     def each(&block)
       to_a.each(&block)
     end
@@ -14,6 +14,68 @@ module Daru
       @levels.map(&:keys)
     end
 
+    # names and levels should be of same size. If size of Array `name` is less
+    # or greater than size of array `levels` then it raises `SizeError`.
+    # If user don't want to put name for particular level then user must put
+    # empty string in that index of Array `name`.
+    # For example there is multi_index of 3 levels and user don't want to name
+    # level 0, then do mulit_index.name = ['', 'level1_name1', 'level2_name']
+    #
+    # @example
+    #
+    #   # set the name during initialization
+    #
+    #   mi = Daru::MultiIndex.new(
+    #       levels: [[:a,:b,:c], [:one, :two]],
+    #       labels: [[0,0,1,1,2,2], [0,1,0,1,0,1]], name: ['s1', 's2'])
+    #
+    #   # =>
+    #   # <Daru::MultiIndex(6x2)>
+    #   #   s1  s2
+    #   #    a one
+    #   #      two
+    #   #    b one
+    #   #      two
+    #   #    c one
+    #   #      two
+    #
+    #   # set new name
+    #
+    #   mi.name = ['k1', 'k2']
+    #   => ["k1", "k2"]
+    #
+    #   mi
+    #   =>
+    #   #   #<Daru::MultiIndex(6x2)>
+    #   #   k1  k2
+    #   #    a one
+    #   #      two
+    #   #    b one
+    #   #      two
+    #   #    c one
+    #   #      two
+    #
+    #   # access the name
+    #
+    #   mi.name
+    #   => ["k1", "k2"]
+    #
+    #   # If you don't want to name level 0
+    #
+    #   mi.name = ['', 'k2']
+    #   => ["", "k2"]
+    #
+    #   mi
+    #   =>
+    #   #<Daru::MultiIndex(6x2)>
+    #   #       k2
+    #   #    a one
+    #   #      two
+    #   #    b one
+    #   #      two
+    #   #    c one
+    #   #      two
+    #
     def initialize opts={}
       labels = opts[:labels]
       levels = opts[:levels]
@@ -24,6 +86,12 @@ module Daru
 
       @labels = labels
       @levels = levels.map { |e| e.map.with_index.to_h }
+      self.name = opts[:name] unless opts[:name].nil?
+    end
+
+    def name=(names)
+      validate_name names, @labels
+      @name = names
     end
 
     def incorrect_fields?(_labels, levels)
@@ -171,8 +239,20 @@ module Daru
       end
     end
 
+    # Array `name` must have same length as levels and labels.
+    def validate_name names, levels
+      error_msg = "'names' and 'levels' should be of same size. Size of the "\
+      "'name' array is #{names.size} and size of the MultiIndex 'levels' and "\
+      "'labels' is #{labels.size}."
+      suggestion_msg = "If you don\'t want to set name for particular level " \
+      "(say level 'i') then put empty string on index 'i' of the 'name' Array."
+
+      raise SizeError, error_msg if names.size > levels.size
+      raise SizeError, [error_msg, suggestion_msg].join("\n") if names.size < levels.size
+    end
+
     private :find_all_indexes, :multi_index_from_multiple_selections,
-      :retrieve_from_range, :retrieve_from_tuples
+      :retrieve_from_range, :retrieve_from_tuples, :validate_name
 
     def key index
       raise ArgumentError, "Key #{index} is too large" if index >= @labels[0].size
@@ -232,7 +312,7 @@ module Daru
 
     def inspect threshold=20
       "#<Daru::MultiIndex(#{size}x#{width})>\n" +
-        Formatters::Table.format([], row_headers: sparse_tuples, threshold: threshold)
+        Formatters::Table.format([], headers: @name, row_headers: sparse_tuples, threshold: threshold)
     end
 
     def to_html

data/lib/daru/io/csv/converters.rb

@@ -0,0 +1,18 @@
+module Daru
+  module IO
+    module CSV
+      CONVERTERS = {
+        boolean: lambda { |f, _|
+          case f.downcase.strip
+          when 'true'
+            true
+          when 'false'
+            false
+          else
+            f
+          end
+        }
+      }.freeze
+    end
+  end
+end

data/lib/daru/io/io.rb

@@ -1,4 +1,5 @@
 module Daru
+  require_relative 'csv/converters.rb'
   module IOHelpers
     class << self
       def process_row(row,empty)
@@ -76,7 +77,6 @@ module Daru
       # Functions for loading/writing CSV files
       def from_csv path, opts={}
         daru_options, opts = from_csv_prepare_opts opts
-
         # Preprocess headers for detecting and correcting repetition in
         # case the :headers option is not specified.
         hsh =
@@ -86,7 +86,6 @@ module Daru
             from_csv_hash(path, opts)
               .tap { |hash| daru_options[:order] = hash.keys }
           end
-
         Daru::DataFrame.new(hsh,daru_options)
       end
 
@@ -111,11 +110,10 @@ module Daru
 
       # Execute a query and create a data frame from the result
       #
-      # @param dbh [DBI::DatabaseHandle] A DBI connection to be used to run the query
+      # @param dbh [DBI::DatabaseHandle, String] A DBI connection OR Path to a SQlite3 database.
       # @param query [String] The query to be executed
       #
       # @return A dataframe containing the data resulting from the query
-
       def from_sql(db, query)
         require 'daru/io/sql_data_source'
         SqlDataSource.make_dataframe(db, query)
@@ -186,13 +184,28 @@ module Daru
         end
       end
 
+      def from_html path, opts
+        page = Mechanize.new.get(path)
+        page.search('table').map { |table| html_parse_table table }
+            .keep_if { |table| html_search table, opts[:match] }
+            .compact
+            .map { |table| html_decide_values table, opts }
+            .map { |table| html_table_to_dataframe table }
+      rescue LoadError
+        raise 'Install the mechanize gem version 2.7.5 with `gem install mechanize`,'\
+        ' for using the from_html function.'
+      end
+
       private
 
-      DARU_OPT_KEYS = [:clone, :order, :index, :name].freeze
+      DARU_OPT_KEYS = %i[clone order index name].freeze
 
       def from_csv_prepare_opts opts
         opts[:col_sep]           ||= ','
-        opts[:converters]        ||= :numeric
+        opts[:skip_blanks]       ||= true
+        opts[:converters]        ||= [:numeric]
+
+        opts[:converters] = from_csv_prepare_converters(opts[:converters])
 
         daru_options = opts.keys.each_with_object({}) do |k, hash|
           hash[k] = opts.delete(k) if DARU_OPT_KEYS.include?(k)
@@ -200,11 +213,22 @@ module Daru
         [daru_options, opts]
       end
 
+      def from_csv_prepare_converters(converters)
+        converters.flat_map do |c|
+          if ::CSV::Converters[c]
+            ::CSV::Converters[c]
+          elsif Daru::IO::CSV::CONVERTERS[c]
+            Daru::IO::CSV::CONVERTERS[c]
+          else
+            c
+          end
+        end
+      end
+
       def from_csv_hash_with_headers(path, opts)
         opts[:header_converters] ||= :symbol
-
         ::CSV
-          .read(path, 'rb',opts)
+          .parse(open(path), opts)
           .tap { |c| yield c if block_given? }
           .by_col.map { |col_name, values| [col_name, values] }.to_h
       end
@@ -212,15 +236,60 @@ module Daru
       def from_csv_hash(path, opts)
         csv_as_arrays =
           ::CSV
-          .open(path, 'rb', opts)
+          .parse(open(path), opts)
           .tap { |c| yield c if block_given? }
           .to_a
-
         headers       = ArrayHelper.recode_repeated(csv_as_arrays.shift)
         csv_as_arrays = csv_as_arrays.transpose
-
         headers.each_with_index.map { |h, i| [h, csv_as_arrays[i]] }.to_h
       end
+
+      def html_parse_table(table)
+        headers, headers_size = html_scrape_tag(table,'th')
+        data, size = html_scrape_tag(table, 'td')
+        data = data.keep_if { |x| x.count == size }
+        order, indice = html_parse_hash(headers, size, headers_size) if headers_size >= size
+        return unless (indice.nil? || indice.count == data.count) && !order.nil? && order.count>0
+        {data: data.compact, index: indice, order: order}
+      end
+
+      def html_scrape_tag(table, tag)
+        arr  = table.search('tr').map { |row| row.search(tag).map { |val| val.text.strip } }
+        size = arr.map(&:count).max
+        [arr, size]
+      end
+
+      # Splits headers (all th tags) into order and index. Wherein,
+      # Order : All <th> tags on first proper row of HTML table
+      # index : All <th> tags on first proper column of HTML table
+      def html_parse_hash(headers, size, headers_size)
+        headers_index = headers.find_index { |x| x.count == headers_size }
+        order = headers[headers_index]
+        order_index = order.count - size
+        order = order[order_index..-1]
+        indice = headers[headers_index+1..-1].flatten
+        indice = nil if indice.to_a.empty?
+        [order, indice]
+      end
+
+      def html_search(table, match=nil)
+        match.nil? ? true : (table.to_s.include? match)
+      end
+
+      # Allows user to override the scraped order / index / data
+      def html_decide_values(scraped_val={}, user_val={})
+        %I[data index name order].each do |key|
+          user_val[key] ||= scraped_val[key]
+        end
+        user_val
+      end
+
+      def html_table_to_dataframe(table)
+        Daru::DataFrame.rows table[:data],
+          index: table[:index],
+          order: table[:order],
+          name: table[:name]
+      end
     end
   end
 end