daru-io 0.1.0

data/lib/daru/io/importers/excel.rb

@@ -0,0 +1,99 @@
+require 'daru/io/importers/base'
+
+module Daru
+  module IO
+    module Importers
+      # Excel Importer Class, that extends `read_excel` method to `Daru::DataFrame`
+      #
+      # @see Daru::IO::Importers::Excelx For .xlsx format
+      class Excel < Base
+        Daru::DataFrame.register_io_module :read_excel do |*args, &io_block|
+          if args.first.end_with?('.xlsx')
+            require 'daru/io/importers/excelx'
+            Daru::IO::Importers::Excelx.new(*args[1..-1], &io_block).read(*args[0])
+          else
+            Daru::IO::Importers::Excel.new(*args[1..-1], &io_block).read(*args[0])
+          end
+        end
+
+        # Checks for required gem dependencies of Excel Importer
+        def initialize
+          optional_gem 'spreadsheet', '~> 1.1.1'
+        end
+
+        # Reads from an excel (.xls) file
+        #
+        # @!method self.read(path)
+        #
+        # @param path [String] Path of Excel file, where the DataFrame is to be imported from.
+        #
+        # @return [Daru::IO::Importers::Excel]
+        #
+        # @example Reading from an excel file
+        #   instance = Daru::IO::Importers::Excel.read("test_xls.xls")
+        def read(path)
+          @file_data = Spreadsheet.open(path)
+          self
+        end
+
+        # Imports a `Daru::DataFrame` from an Excel Importer instance
+        #
+        # @param worksheet_id [Integer] The index of the worksheet in the excel file,
+        #   from where the `Daru::DataFrame` will be imported. By default, the first
+        #   worksheet has `:worksheet_id` as 0. In general, the n-th worksheet has
+        #   its worksheet_id as n-1.
+        #
+        #   If worksheet_id option is not given, it is taken as 0 by default and the
+        #   `Daru::DataFrame` will be imported from the first worksheet in the excel file.
+        # @param headers [Boolean] Defaults to true. When set to true, first row of the
+        #   given worksheet_id is used as the order of the Daru::DataFrame and data of
+        #   the Dataframe consists of the remaining rows.
+        #
+        # @return [Daru::DataFrame]
+        #
+        #   default_instance = Daru::IO::Importers::Excel.new
+        #
+        # @example Importing from a default worksheet
+        #   df = instance.call
+        #
+        #   #=> #<Daru::DataFrame(6x5)>
+        #   #            id     name      age     city       a1
+        #   #    0        1     Alex       20 New York      a,b
+        #   #    1        2   Claude       23   London      b,c
+        #   #    2        3    Peter       25   London        a
+        #   #    3        4    Franz      nil    Paris      nil
+        #   #    4        5   George      5.5     Tome    a,b,c
+        #   #    5        6  Fernand      nil      nil      nil
+        #
+        # @example Importing from a specific worksheet
+        #   df = instance.call(worksheet_id: 0)
+        #
+        #   #=> #<Daru::DataFrame(6x5)>
+        #   #            id     name      age     city       a1
+        #   #    0        1     Alex       20 New York      a,b
+        #   #    1        2   Claude       23   London      b,c
+        #   #    2        3    Peter       25   London        a
+        #   #    3        4    Franz      nil    Paris      nil
+        #   #    4        5   George      5.5     Tome    a,b,c
+        #   #    5        6  Fernand      nil      nil      nil
+        def call(worksheet_id: 0, headers: true)
+          worksheet = @file_data.worksheet(worksheet_id)
+          headers   = if headers
+                        ArrayHelper.recode_repeated(worksheet.row(0)).map(&:to_sym)
+                      else
+                        (0..worksheet.row(0).to_a.size-1).to_a
+                      end
+
+          df = Daru::DataFrame.new({})
+          headers.each_with_index do |h,i|
+            col = worksheet.column(i).to_a
+            col.delete_at(0) if headers
+            df[h] = col
+          end
+
+          df
+        end
+      end
+    end
+  end
+end

data/lib/daru/io/importers/excelx.rb

@@ -0,0 +1,138 @@
+require 'daru/io/importers/base'
+
+module Daru
+  module IO
+    module Importers
+      # Excelx Importer Class, that handles .xlsx files in the Excel Importer
+      #
+      # @see Daru::IO::Importers::Excel For .xls format
+      class Excelx < Base
+        # Checks for required gem dependencies of Excelx Importer
+        def initialize
+          optional_gem 'roo', '~> 2.7.0'
+        end
+
+        # Reads from an excelx (xlsx) file
+        #
+        # @!method self.read(path)
+        #
+        # @param path [String] Local / Remote path of xlsx file, where the DataFrame is
+        #   to be imported from.
+        #
+        # @return [Daru::IO::Importers::Excelx]
+        #
+        # @example Reading from a local xlsx file
+        #   local_instance = Daru::IO::Importers::Excelx.read("Stock-counts-sheet.xlsx")
+        #
+        # @example Reading from a remote xlsx file
+        #   url = "https://www.exact.com/uk/images/downloads/getting-started-excel-sheets/Stock-counts-sheet.xlsx"
+        #   remote_instance = Daru::IO::Importers::Excelx.read(url)
+        def read(path)
+          @file_data = Roo::Excelx.new(path)
+          self
+        end
+
+        # Imports a `Daru::DataFrame` from an Excelx Importer instance
+        #
+        # @param sheet [Integer or String] Imports from a specific sheet
+        # @param skiprows [Integer] Skips the first `:skiprows` number of rows from the
+        #   sheet being parsed.
+        # @param skipcols [Integer] Skips the first `:skipcols` number of columns from the
+        #   sheet being parsed.
+        # @param order [Boolean] Defaults to true. When set to true, first row of the
+        #   given sheet is used as the order of the Daru::DataFrame and data of
+        #   the Dataframe consists of the remaining rows.
+        # @param index [Boolean] Defaults to false. When set to true, first column of the
+        #   given sheet is used as the index of the Daru::DataFrame and data of
+        #   the Dataframe consists of the remaining columns.
+        #
+        #   When set to false, a default order (0 to n-1) is chosen for the DataFrame,
+        #   and the data of the DataFrame consists of all rows in the sheet.
+        #
+        # @return [Daru::DataFrame]
+        #
+        # @example Importing from specific sheet
+        #   df = local_instance.call(sheet: 'Example Stock Counts')
+        #
+        #   #=> <Daru::DataFrame(15x7)>
+        #   #           Status Stock coun  Item code        New Descriptio Stock coun Offset G/L
+        #   #     0          H          1        nil        nil New stock  2014-08-01        nil
+        #   #     1        nil          1  IND300654          2 New stock  2014-08-01      51035
+        #   #     2        nil          1   IND43201          5 New stock  2014-08-01      51035
+        #   #     3        nil          1   OUT30045          3 New stock  2014-08-01      51035
+        #   #    ...       ...        ...     ...           ...     ...       ...           ...
+        #
+        # @example Importing from a remote URL and default sheet
+        #   df = remote_instance.call
+        #
+        #   #=> <Daru::DataFrame(15x7)>
+        #   #           Status Stock coun  Item code        New Descriptio Stock coun Offset G/L
+        #   #     0          H          1        nil        nil New stock  2014-08-01        nil
+        #   #     1        nil          1  IND300654          2 New stock  2014-08-01      51035
+        #   #     2        nil          1   IND43201          5 New stock  2014-08-01      51035
+        #   #     3        nil          1   OUT30045          3 New stock  2014-08-01      51035
+        #   #    ...       ...        ...     ...           ...     ...       ...           ...
+        #
+        # @example Importing without headers
+        #   df = local_instance.call(sheet: 'Example Stock Counts', headers: false)
+        #
+        #   #=> <Daru::DataFrame(16x7)>
+        #   #                0           1          2          3          4          5        6
+        #   #     0      Status Stock coun  Item code        New Descriptio Stock coun Offset G/L
+        #   #     1          H          1        nil        nil New stock  2014-08-01        nil
+        #   #     2        nil          1  IND300654          2 New stock  2014-08-01      51035
+        #   #     3        nil          1   IND43201          5 New stock  2014-08-01      51035
+        #   #     4        nil          1   OUT30045          3 New stock  2014-08-01      51035
+        #   #    ...       ...        ...     ...           ...     ...       ...           ...
+        def call(sheet: 0, skiprows: 0, skipcols: 0, order: true, index: false)
+          @order    = order
+          @index    = index
+          worksheet = @file_data.sheet(sheet)
+          @data     = strip_html_tags(skip_data(worksheet.to_a, skiprows, skipcols))
+          @index    = process_index
+          @order    = process_order || (0..@data.first.length-1)
+          @data     = process_data
+
+          Daru::DataFrame.rows(@data, order: @order, index: @index)
+        end
+
+        private
+
+        def process_data
+          return skip_data(@data, 1, 1) if @order && @index
+          return skip_data(@data, 1, 0) if @order
+          return skip_data(@data, 0, 1) if @index
+          @data
+        end
+
+        def process_index
+          return nil unless @index
+          @index = @data.transpose.first
+          @index = skip_data(@index, 1) if @order
+          @index
+        end
+
+        def process_order
+          return nil unless @order
+          @order = @data.first
+          @order = skip_data(@order, 1) if @index
+          @order
+        end
+
+        def skip_data(data, rows, cols=nil)
+          return data[rows..-1].map { |row| row[cols..-1] } unless cols.nil?
+          data[rows..-1]
+        end
+
+        def strip_html_tags(data)
+          data.map do |row|
+            row.map do |ele|
+              next ele unless ele.is_a?(String)
+              ele.gsub(/<[^>]+>/, '')
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/daru/io/importers/html.rb

@@ -0,0 +1,144 @@
+require 'daru/io/importers/base'
+
+module Daru
+  module IO
+    module Importers
+      # HTML Importer Class, that extends `read_html` method to `Daru::DataFrame`
+      #
+      # @note
+      #   Please note that this module works only for static table elements on a
+      #   HTML page, and won't work in cases where the data is being loaded into
+      #   the HTML table by inline Javascript.
+      class HTML < Base
+        Daru::DataFrame.register_io_module :read_html, self
+
+        # Checks for required gem dependencies of HTML Importer
+        def initialize
+          require 'open-uri'
+          optional_gem 'nokogiri'
+        end
+
+        # Reads from a html file / website
+        #
+        # @!method self.read(path)
+        #
+        # @param path [String] Website URL / path to HTML file, where the
+        #   DataFrame is to be imported from.
+        #
+        # @return [Daru::IO::Importers::HTML]
+        #
+        # @example Reading from a website url file
+        #   instance = Daru::IO::Importers::HTML.read('http://www.moneycontrol.com/')
+        def read(path)
+          @file_data = Nokogiri.parse(open(path).read)
+          self
+        end
+
+        # Imports Array of `Daru::DataFrame`s from a HTML Importer instance
+        #
+        # @param match [String] A `String` to match and choose a particular table(s)
+        #   from multiple tables of a HTML page.
+        # @param index [Array or Daru::Index or Daru::MultiIndex] If given, it
+        #   overrides the parsed index. Have a look at `:index` option, at
+        #   {http://www.rubydoc.info/gems/daru/0.1.5/Daru%2FDataFrame:initialize
+        #   Daru::DataFrame#initialize}
+        # @param order [Array or Daru::Index or Daru::MultiIndex] If given, it
+        #   overrides the parsed order. Have a look at `:order` option
+        #   [here](http://www.rubydoc.info/gems/daru/0.1.5/Daru%2FDataFrame:initialize)
+        # @param name [String] As `name` of the imported `Daru::DataFrame` isn't
+        #   parsed automatically by the module, users can set the name attribute to
+        #   their `Daru::DataFrame` manually, through this option.
+        #
+        #   See `:name` option
+        #   [here](http://www.rubydoc.info/gems/daru/0.1.5/Daru%2FDataFrame:initialize)
+        #
+        # @return [Array<Daru::DataFrame>]
+        #
+        # @example Importing with matching tables
+        #   list_of_dfs = instance.call(match: 'Sun Pharma')
+        #   list_of_dfs.count
+        #   #=> 4
+        #
+        #   df = list_of_dfs.first
+        #
+        #   # As the website keeps changing everyday, the output might not be exactly
+        #   # the same as the one obtained below. Nevertheless, a Daru::DataFrame
+        #   # should be obtained (as long as 'Sun Pharma' is there on the website).
+        #
+        #   #=> <Daru::DataFrame(5x4)>
+        #   #        Company      Price     Change Value (Rs
+        #   #   0 Sun Pharma     502.60     -65.05   2,117.87
+        #   #   1   Reliance    1356.90      19.60     745.10
+        #   #   2 Tech Mahin     379.45     -49.70     650.22
+        #   #   3        ITC     315.85       6.75     621.12
+        #   #   4       HDFC    1598.85      50.95     553.91
+        def call(match: nil, order: nil, index: nil, name: nil)
+          @match   = match
+          @options = {name: name, index: index, order: order}
+
+          @file_data
+            .search('table')
+            .map { |table| parse_table(table) }
+            .compact
+            .keep_if { |table| satisfy_dimension(table) && search(table) }
+            .map { |table| decide_values(table, @options) }
+            .map { |table| table_to_dataframe(table) }
+        end
+
+        private
+
+        # Allows user to override the scraped order / index / data
+        def decide_values(scraped_val, user_val)
+          scraped_val.merge(user_val) { |_key, scraped, user| user || scraped }
+        end
+
+        # Splits headers (all th tags) into order and index. Wherein,
+        # Order : All <th> tags on first proper row of HTML table
+        # index : All <th> tags on first proper column of HTML table
+        def parse_hash(headers, size, headers_size)
+          headers_index = headers.find_index { |x| x.count == headers_size }
+          order = headers[headers_index]
+          order_index = order.count - size
+          order = order[order_index..-1]
+          indice = headers[headers_index+1..-1].flatten
+          indice = nil if indice.to_a.empty?
+          [order, indice]
+        end
+
+        def parse_table(table)
+          headers, headers_size = scrape_tag(table,'th')
+          data, size = scrape_tag(table, 'td')
+          data = data.keep_if { |x| x.count == size }
+          order, indice = parse_hash(headers, size, headers_size) if headers_size >= size
+          return unless (indice.nil? || indice.count == data.count) && !order.nil? && order.count>0
+          {data: data.compact, index: indice, order: order}
+        end
+
+        def scrape_tag(table, tag)
+          arr  = table.search('tr').map { |row| row.search(tag).map { |val| val.text.strip } }
+          size = arr.map(&:count).max
+          [arr, size]
+        end
+
+        def satisfy_dimension(table)
+          return false if @options[:order] && table[:data].first.size != @options[:order].size
+          return false if @options[:index] && table[:data].size != @options[:index].size
+          true
+        end
+
+        def search(table)
+          @match.nil? ? true : table.to_s.include?(@match)
+        end
+
+        def table_to_dataframe(table)
+          Daru::DataFrame.rows(
+            table[:data],
+            index: table[:index],
+            order: table[:order],
+            name: table[:name]
+          )
+        end
+      end
+    end
+  end
+end

data/lib/daru/io/importers/json.rb

@@ -0,0 +1,152 @@
+require 'daru/io/importers/base'
+
+module Daru
+  module IO
+    module Importers
+      # JSON Importer Class, that extends `from_json` and `read_json` methods
+      # to `Daru::DataFrame`
+      class JSON < Base
+        Daru::DataFrame.register_io_module :from_json, self
+        Daru::DataFrame.register_io_module :read_json, self
+
+        # Checks for required gem dependencies of JSON Importer
+        def initialize
+          require 'open-uri'
+          require 'json'
+          optional_gem 'jsonpath'
+        end
+
+        # Reads data from a json file / remote json response
+        #
+        # @!method self.read(path)
+        #
+        # @param path [String] Local / Remote path to json file, where the dataframe is to be imported
+        #   from.
+        #
+        # @return [Daru::IO::Importers::JSON]
+        #
+        # @example Reading from simply nested remote json response
+        #   url = 'https://data.nasa.gov/resource/2vr3-k9wn.json'
+        #   simple_read_instance = Daru::IO::Importers::JSON.read(url)
+        #
+        # @example Reading from complexy nested remote json response
+        #   url = 'http://api.tvmaze.com/singlesearch/shows?q=game-of-thrones&embed=episodes'
+        #   complex_read_instance = Daru::IO::Importers::JSON.read(url)
+        def read(path)
+          @file_data = ::JSON.parse(open(path).read)
+          @json      = @file_data
+          self
+        end
+
+        # Loads from a Ruby structure of Hashes and / or Arrays
+        #
+        # @!method self.from(instance)
+        #
+        # @param instance [Hash or Array] A simple / complexly nested JSON structure
+        #
+        # @return [Daru::IO::Importers::JSON]
+        #
+        # @example Loading from Ruby Hash of Arrays
+        #   from_instance = Daru::IO::Importers::JSON.from({x: [1,4], y: [2,5], z: [3, 6]})
+        def from(instance)
+          @file_data = instance
+          @json      = @file_data.is_a?(String) ? ::JSON.parse(@file_data) : @file_data
+          self
+        end
+
+        # Imports a `Daru::DataFrame` from a JSON Importer instance
+        #
+        # @param columns [Array] JSON-path slectors to select specific fields
+        #   from the JSON input.
+        # @param order [String or Array] Either a JSON-path selector string, or
+        #   an array containing the order of the `Daru::DataFrame`.
+        # @param index [String or Array] Either a JSON-path selector string, or
+        #   an array containing the order of the `Daru::DataFrame`.
+        # @param named_columns [Hash] JSON-path slectors to select specific fields
+        #   from the JSON input.
+        #
+        # @return [Daru::DataFrame]
+        #
+        # @note For more information on using JSON-path selectors, have a look at
+        #   the explanations {http://www.rubydoc.info/gems/jsonpath/0.5.8 here}
+        #   and {http://goessner.net/articles/JsonPath/ here}.
+        #
+        # @example Importing without jsonpath selectors
+        #   df = simple_read_instance.call
+        #
+        #   #=> #<Daru::DataFrame(202x10)>
+        #   #    designation discovery_      h_mag      i_deg    moid_au orbit_clas  period_yr ...
+        #   #  0 419880 (20 2011-01-07       19.7       9.65      0.035     Apollo       4.06  ...
+        #   #  1 419624 (20 2010-09-17       20.5      14.52      0.028     Apollo          1  ...
+        #   #  2 414772 (20 2010-07-28         19      23.11      0.333     Apollo       1.31  ...
+        #   # ...        ...        ...        ...        ...        ...        ...       ...  ...
+        #
+        # @example Importing with jsonpath selectors
+        #   df = complex_read_instance.call(
+        #     "$.._embedded..episodes..name",
+        #     "$.._embedded..episodes..season",
+        #     "$.._embedded..episodes..number",
+        #     index: (10..70).to_a,
+        #     RunTime: "$.._embedded..episodes..runtime"
+        #   )
+        #
+        #   #=> #<Daru::DataFrame(61x4)>
+        #   #         name           season     number    RunTime
+        #   #   10 Winter is           1          1         60
+        #   #   11 The Kingsr          1          2         60
+        #   #   12  Lord Snow          1          3         60
+        #   #  ...        ...        ...        ...        ...
+        #
+        # @example Importing from `from` method
+        #   df = from_instance.call
+        #
+        #   #=> #<Daru::DataFrame(2x3)>
+        #   #       x   y   z
+        #   #   0   1   2   3
+        #   #   1   4   5   6
+        def call(*columns, order: nil, index: nil, **named_columns)
+          init_opts(*columns, order: order, index: index, **named_columns)
+          @data    = fetch_data
+          @index   = at_jsonpath(@index)
+          @order   = at_jsonpath(@order)
+          @order ||= Array.new(@columns.count) { |x| x } + @named_columns.keys
+
+          Daru::DataFrame.new(@data, order: @order, index: @index)
+        end
+
+        private
+
+        def at_jsonpath(jsonpath)
+          jsonpath.is_a?(String) ? JsonPath.on(@json, jsonpath) : jsonpath
+        end
+
+        def fetch_data
+          return @json if @columns.empty? && @named_columns.empty?
+
+          # If only one unnamed column is provided without any named_columns,
+          # entire dataset is assumed to reside in that JSON-path.
+          return at_jsonpath(@columns.first) if @columns.size == 1 && @named_columns.empty?
+          data_columns = @columns + @named_columns.values
+          data_columns.map { |col| at_jsonpath(col) }
+        end
+
+        def init_opts(*columns, order: nil, index: nil, **named_columns)
+          @columns       = columns
+          @order         = order
+          @index         = index
+          @named_columns = named_columns
+
+          validate_params
+        end
+
+        def validate_params
+          return if @order.nil? || @named_columns.empty?
+
+          raise ArgumentError,
+            'Do not pass on order and named columns together, at the same '\
+            'function call. Please use only order or only named_columns.'
+        end
+      end
+    end
+  end
+end