daru 0.1.5 → 0.3

data/lib/daru/index/multi_index.rb

@@ -1,5 +1,5 @@
 module Daru
-  class MultiIndex < Index
+  class MultiIndex < Index # rubocop:disable Metrics/ClassLength
     def each(&block)
       to_a.each(&block)
     end
@@ -9,11 +9,74 @@ module Daru
     end
 
     attr_reader :labels
+    attr_reader :name
 
     def levels
       @levels.map(&:keys)
     end
 
+    # names and levels should be of same size. If size of Array `name` is less
+    # or greater than size of array `levels` then it raises `SizeError`.
+    # If user don't want to put name for particular level then user must put
+    # empty string in that index of Array `name`.
+    # For example there is multi_index of 3 levels and user don't want to name
+    # level 0, then do multi_index.name = ['', 'level1_name1', 'level2_name']
+    #
+    # @example
+    #
+    #   # set the name during initialization
+    #
+    #   mi = Daru::MultiIndex.new(
+    #       levels: [[:a,:b,:c], [:one, :two]],
+    #       labels: [[0,0,1,1,2,2], [0,1,0,1,0,1]], name: ['s1', 's2'])
+    #
+    #   # =>
+    #   # <Daru::MultiIndex(6x2)>
+    #   #   s1  s2
+    #   #    a one
+    #   #      two
+    #   #    b one
+    #   #      two
+    #   #    c one
+    #   #      two
+    #
+    #   # set new name
+    #
+    #   mi.name = ['k1', 'k2']
+    #   => ["k1", "k2"]
+    #
+    #   mi
+    #   =>
+    #   #   #<Daru::MultiIndex(6x2)>
+    #   #   k1  k2
+    #   #    a one
+    #   #      two
+    #   #    b one
+    #   #      two
+    #   #    c one
+    #   #      two
+    #
+    #   # access the name
+    #
+    #   mi.name
+    #   => ["k1", "k2"]
+    #
+    #   # If you don't want to name level 0
+    #
+    #   mi.name = ['', 'k2']
+    #   => ["", "k2"]
+    #
+    #   mi
+    #   =>
+    #   #<Daru::MultiIndex(6x2)>
+    #   #       k2
+    #   #    a one
+    #   #      two
+    #   #    b one
+    #   #      two
+    #   #    c one
+    #   #      two
+    #
     def initialize opts={}
       labels = opts[:labels]
       levels = opts[:levels]
@@ -24,6 +87,12 @@ module Daru
 
       @labels = labels
       @levels = levels.map { |e| e.map.with_index.to_h }
+      self.name = opts[:name] unless opts[:name].nil?
+    end
+
+    def name=(names)
+      validate_name names, @labels
+      @name = names
     end
 
     def incorrect_fields?(_labels, levels)
@@ -84,7 +153,7 @@ module Daru
     # Returns positions given indexes or positions
     # @note If the arugent is both a valid index and a valid position,
     #   it will treated as valid index
-    # @param [Array<object>] *indexes indexes or positions
+    # @param indexes [Array<object>] indexes or positions
     # @example
     #   idx = Daru::MultiIndex.from_tuples [[:a, :one], [:a, :two], [:b, :one], [:b, :two]]
     #   idx.pos :a
@@ -109,7 +178,7 @@ module Daru
 
     # Takes positional values and returns subset of the self
     #   capturing the indexes at mentioned positions
-    # @param [Array<Integer>] positional values
+    # @param positions [Array<Integer>] positional values
     # @return [object] index object
     # @example
     #   idx = Daru::MultiIndex.from_tuples [[:a, :one], [:a, :two], [:b, :one], [:b, :two]]
@@ -128,12 +197,12 @@ module Daru
     end
 
     def add *indexes
-      Daru::MultiIndex.from_tuples to_a << indexes
+      Daru::MultiIndex.from_tuples(to_a + [indexes])
     end
 
     def reorder(new_order)
       from = to_a
-      self.class.from_tuples(new_order.map { |i| from[i] })
+      MultiIndex.from_tuples(new_order.map { |i| from[i] })
     end
 
     def try_retrieve_from_integer int
@@ -171,8 +240,42 @@ module Daru
       end
     end
 
+    def remove_layer layer_index
+      @levels.delete_at(layer_index)
+      @labels.delete_at(layer_index)
+      @name.delete_at(layer_index) unless @name.nil?
+
+      coerce_index
+    end
+
+    def coerce_index
+      if @levels.size == 1
+        elements = to_a.flatten
+
+        if elements.uniq.length == elements.length
+          Daru::Index.new(elements)
+        else
+          Daru::CategoricalIndex.new(elements)
+        end
+      else
+        self
+      end
+    end
+
+    # Array `name` must have same length as levels and labels.
+    def validate_name names, levels
+      error_msg = "'names' and 'levels' should be of same size. Size of the "\
+      "'name' array is #{names.size} and size of the MultiIndex 'levels' and "\
+      "'labels' is #{labels.size}."
+      suggestion_msg = "If you don\'t want to set name for particular level " \
+      "(say level 'i') then put empty string on index 'i' of the 'name' Array."
+
+      raise SizeError, error_msg if names.size > levels.size
+      raise SizeError, [error_msg, suggestion_msg].join("\n") if names.size < levels.size
+    end
+
     private :find_all_indexes, :multi_index_from_multiple_selections,
-      :retrieve_from_range, :retrieve_from_tuples
+      :retrieve_from_range, :retrieve_from_tuples, :validate_name
 
     def key index
       raise ArgumentError, "Key #{index} is too large" if index >= @labels[0].size
@@ -183,7 +286,7 @@ module Daru
     end
 
     def dup
-      MultiIndex.new levels: levels.dup, labels: labels
+      MultiIndex.new levels: levels.dup, labels: labels.dup, name: (@name.nil? ? nil : @name.dup)
     end
 
     def drop_left_level by=1
@@ -204,8 +307,9 @@ module Daru
 
     def include? tuple
       return false unless tuple.is_a? Enumerable
-      tuple.flatten.each_with_index
-           .all? { |tup, i| @levels[i][tup] }
+      @labels[0...tuple.flatten.size]
+        .transpose
+        .include?(tuple.flatten.each_with_index.map { |e, i| @levels[i][e] })
     end
 
     def size
@@ -232,7 +336,7 @@ module Daru
 
     def inspect threshold=20
       "#<Daru::MultiIndex(#{size}x#{width})>\n" +
-        Formatters::Table.format([], row_headers: sparse_tuples, threshold: threshold)
+        Formatters::Table.format([], headers: @name, row_headers: sparse_tuples, threshold: threshold)
     end
 
     def to_html
@@ -262,5 +366,9 @@ module Daru
         [nil] * (cur.size - left.size) + left.map(&:first)
       }
     end
+
+    def to_df
+      Daru::DataFrame.new(@name.zip(to_a.transpose).to_h)
+    end
   end
 end

data/lib/daru/io/csv/converters.rb

@@ -0,0 +1,21 @@
+module Daru
+  module IO
+    module CSV
+      CONVERTERS = {
+        boolean: lambda { |f, _|
+          case f.downcase.strip
+          when 'true'
+            true
+          when 'false'
+            false
+          else
+            f
+          end
+        },
+        string: lambda { |f, _|
+          f
+        }
+      }.freeze
+    end
+  end
+end

data/lib/daru/io/io.rb

@@ -1,4 +1,5 @@
 module Daru
+  require_relative 'csv/converters.rb'
   module IOHelpers
     class << self
       def process_row(row,empty)
@@ -39,14 +40,11 @@ module Daru
 
       def from_excel path, opts={}
         opts = {
-          worksheet_id: 0
+          worksheet_id: 0,
+          row_id: 0
         }.merge opts
 
-        worksheet_id = opts[:worksheet_id]
-        book         = Spreadsheet.open path
-        worksheet    = book.worksheet worksheet_id
-        headers      = ArrayHelper.recode_repeated(worksheet.row(0)).map(&:to_sym)
-
+        worksheet, headers = read_from_excel(path, opts)
         df = Daru::DataFrame.new({})
         headers.each_with_index do |h,i|
           col = worksheet.column(i).to_a
@@ -57,6 +55,18 @@ module Daru
         df
       end
 
+      def read_from_excel path, opts
+        optional_gem 'spreadsheet', '~>1.1.1'
+
+        worksheet_id = opts[:worksheet_id]
+        row_id       = opts[:row_id]
+        book         = Spreadsheet.open path
+        worksheet    = book.worksheet worksheet_id
+        headers      = ArrayHelper.recode_repeated(worksheet.row(row_id)).map(&:to_sym)
+
+        [worksheet, headers]
+      end
+
       def dataframe_write_excel dataframe, path, _opts={}
         book   = Spreadsheet::Workbook.new
         sheet  = book.create_worksheet
@@ -76,7 +86,6 @@ module Daru
       # Functions for loading/writing CSV files
       def from_csv path, opts={}
         daru_options, opts = from_csv_prepare_opts opts
-
         # Preprocess headers for detecting and correcting repetition in
         # case the :headers option is not specified.
         hsh =
@@ -86,7 +95,6 @@ module Daru
             from_csv_hash(path, opts)
               .tap { |hash| daru_options[:order] = hash.keys }
           end
-
         Daru::DataFrame.new(hsh,daru_options)
       end
 
@@ -111,11 +119,10 @@ module Daru
 
       # Execute a query and create a data frame from the result
       #
-      # @param dbh [DBI::DatabaseHandle] A DBI connection to be used to run the query
+      # @param db [DBI::DatabaseHandle, String] A DBI connection OR Path to a SQlite3 database.
       # @param query [String] The query to be executed
       #
       # @return A dataframe containing the data resulting from the query
-
       def from_sql(db, query)
         require 'daru/io/sql_data_source'
         SqlDataSource.make_dataframe(db, query)
@@ -135,23 +142,11 @@ module Daru
       #
       # @return A dataframe containing the data in the given relation
       def from_activerecord(relation, *fields)
-        if fields.empty?
-          records = relation.map do |record|
-            record.attributes.symbolize_keys
-          end
-          return Daru::DataFrame.new(records)
-        else
-          fields = fields.map(&:to_sym)
-        end
-
-        vectors = fields.map { |name| [name, Daru::Vector.new([], name: name)] }.to_h
+        fields = relation.klass.column_names if fields.empty?
+        fields = fields.map(&:to_sym)
 
-        Daru::DataFrame.new(vectors, order: fields).tap do |df|
-          relation.pluck(*fields).each do |record|
-            df.add_row(Array(record))
-          end
-          df.update
-        end
+        result = relation.pluck(*fields).transpose
+        Daru::DataFrame.new(result, order: fields).tap(&:update)
       end
 
       # Loading data from plain text files
@@ -186,13 +181,34 @@ module Daru
         end
       end
 
+      def from_html path, opts
+        optional_gem 'mechanize', '~>2.7.5'
+        page = Mechanize.new.get(path)
+        page.search('table').map { |table| html_parse_table table }
+            .keep_if { |table| html_search table, opts[:match] }
+            .compact
+            .map { |table| html_decide_values table, opts }
+            .map { |table| html_table_to_dataframe table }
+      end
+
       private
 
-      DARU_OPT_KEYS = [:clone, :order, :index, :name].freeze
+      def optional_gem(name, version)
+        gem name, version
+        require name
+      rescue LoadError
+        Daru.error "\nInstall the #{name} gem version #{version} for using"\
+        " #{name} functions."
+      end
+
+      DARU_OPT_KEYS = %i[clone order index name].freeze
 
       def from_csv_prepare_opts opts
         opts[:col_sep]           ||= ','
-        opts[:converters]        ||= :numeric
+        opts[:skip_blanks]       ||= true
+        opts[:converters]        ||= [:numeric]
+
+        opts[:converters] = from_csv_prepare_converters(opts[:converters])
 
         daru_options = opts.keys.each_with_object({}) do |k, hash|
           hash[k] = opts.delete(k) if DARU_OPT_KEYS.include?(k)
@@ -200,11 +216,22 @@ module Daru
         [daru_options, opts]
       end
 
+      def from_csv_prepare_converters(converters)
+        Array(converters).flat_map do |c|
+          if ::CSV::Converters[c]
+            ::CSV::Converters[c]
+          elsif Daru::IO::CSV::CONVERTERS[c]
+            Daru::IO::CSV::CONVERTERS[c]
+          else
+            c
+          end
+        end
+      end
+
       def from_csv_hash_with_headers(path, opts)
         opts[:header_converters] ||= :symbol
-
         ::CSV
-          .read(path, 'rb',opts)
+          .parse(open(path), opts)
           .tap { |c| yield c if block_given? }
           .by_col.map { |col_name, values| [col_name, values] }.to_h
       end
@@ -212,15 +239,60 @@ module Daru
       def from_csv_hash(path, opts)
         csv_as_arrays =
           ::CSV
-          .open(path, 'rb', opts)
+          .parse(open(path), **opts)
           .tap { |c| yield c if block_given? }
           .to_a
-
         headers       = ArrayHelper.recode_repeated(csv_as_arrays.shift)
         csv_as_arrays = csv_as_arrays.transpose
-
         headers.each_with_index.map { |h, i| [h, csv_as_arrays[i]] }.to_h
       end
+
+      def html_parse_table(table)
+        headers, headers_size = html_scrape_tag(table,'th')
+        data, size = html_scrape_tag(table, 'td')
+        data = data.keep_if { |x| x.count == size }
+        order, indice = html_parse_hash(headers, size, headers_size) if headers_size >= size
+        return unless (indice.nil? || indice.count == data.count) && !order.nil? && order.count>0
+        {data: data.compact, index: indice, order: order}
+      end
+
+      def html_scrape_tag(table, tag)
+        arr  = table.search('tr').map { |row| row.search(tag).map { |val| val.text.strip } }
+        size = arr.map(&:count).max
+        [arr, size]
+      end
+
+      # Splits headers (all th tags) into order and index. Wherein,
+      # Order : All <th> tags on first proper row of HTML table
+      # index : All <th> tags on first proper column of HTML table
+      def html_parse_hash(headers, size, headers_size)
+        headers_index = headers.find_index { |x| x.count == headers_size }
+        order = headers[headers_index]
+        order_index = order.count - size
+        order = order[order_index..-1]
+        indice = headers[headers_index+1..-1].flatten
+        indice = nil if indice.to_a.empty?
+        [order, indice]
+      end
+
+      def html_search(table, match=nil)
+        match.nil? ? true : (table.to_s.include? match)
+      end
+
+      # Allows user to override the scraped order / index / data
+      def html_decide_values(scraped_val={}, user_val={})
+        %I[data index name order].each do |key|
+          user_val[key] ||= scraped_val[key]
+        end
+        user_val
+      end
+
+      def html_table_to_dataframe(table)
+        Daru::DataFrame.rows table[:data],
+          index: table[:index],
+          order: table[:order],
+          name: table[:name]
+      end
     end
   end
 end