daru_lite 0.1

data/lib/daru_lite/index/multi_index.rb

@@ -0,0 +1,374 @@
+module DaruLite
+  class MultiIndex < Index # rubocop:disable Metrics/ClassLength
+    def each(&block)
+      to_a.each(&block)
+    end
+
+    def map(&block)
+      to_a.map(&block)
+    end
+
+    attr_reader :labels, :name
+
+    def levels
+      @levels.map(&:keys)
+    end
+
+    # names and levels should be of same size. If size of Array `name` is less
+    # or greater than size of array `levels` then it raises `SizeError`.
+    # If user don't want to put name for particular level then user must put
+    # empty string in that index of Array `name`.
+    # For example there is multi_index of 3 levels and user don't want to name
+    # level 0, then do multi_index.name = ['', 'level1_name1', 'level2_name']
+    #
+    # @example
+    #
+    #   # set the name during initialization
+    #
+    #   mi = DaruLite::MultiIndex.new(
+    #       levels: [[:a,:b,:c], [:one, :two]],
+    #       labels: [[0,0,1,1,2,2], [0,1,0,1,0,1]], name: ['s1', 's2'])
+    #
+    #   # =>
+    #   # <DaruLite::MultiIndex(6x2)>
+    #   #   s1  s2
+    #   #    a one
+    #   #      two
+    #   #    b one
+    #   #      two
+    #   #    c one
+    #   #      two
+    #
+    #   # set new name
+    #
+    #   mi.name = ['k1', 'k2']
+    #   => ["k1", "k2"]
+    #
+    #   mi
+    #   =>
+    #   #   #<DaruLite::MultiIndex(6x2)>
+    #   #   k1  k2
+    #   #    a one
+    #   #      two
+    #   #    b one
+    #   #      two
+    #   #    c one
+    #   #      two
+    #
+    #   # access the name
+    #
+    #   mi.name
+    #   => ["k1", "k2"]
+    #
+    #   # If you don't want to name level 0
+    #
+    #   mi.name = ['', 'k2']
+    #   => ["", "k2"]
+    #
+    #   mi
+    #   =>
+    #   #<DaruLite::MultiIndex(6x2)>
+    #   #       k2
+    #   #    a one
+    #   #      two
+    #   #    b one
+    #   #      two
+    #   #    c one
+    #   #      two
+    #
+    def initialize(opts = {})
+      labels = opts[:labels]
+      levels = opts[:levels]
+
+      raise ArgumentError, 'Must specify both labels and levels' unless labels && levels
+      raise ArgumentError, 'Labels and levels should be same size' if labels.size != levels.size
+      raise ArgumentError, 'Incorrect labels and levels' if incorrect_fields?(labels, levels)
+
+      @labels = labels
+      @levels = levels.map { |e| e.map.with_index.to_h }
+      self.name = opts[:name] unless opts[:name].nil?
+    end
+
+    def name=(names)
+      validate_name names, @labels
+      @name = names
+    end
+
+    def incorrect_fields?(_labels, levels)
+      levels[0].size # FIXME: without this exact call some specs are failing
+
+      levels.any? { |e| e.uniq.size != e.size }
+    end
+
+    private :incorrect_fields?
+
+    def self.from_arrays(arrays)
+      levels = arrays.map { |e| e.uniq.sort_by(&:to_s) }
+
+      labels = arrays.each_with_index.map do |arry, level_index|
+        level = levels[level_index]
+        arry.map { |lvl| level.index(lvl) }
+      end
+
+      MultiIndex.new labels: labels, levels: levels
+    end
+
+    def self.from_tuples(tuples)
+      from_arrays tuples.transpose
+    end
+
+    def self.try_from_tuples(tuples)
+      from_tuples(tuples) if tuples.respond_to?(:first) && tuples.first.is_a?(Array)
+    end
+
+    def [](*key)
+      key.flatten!
+      if key[0].is_a?(Range)
+        retrieve_from_range(key[0])
+      elsif key[0].is_a?(Integer) && key.size == 1
+        try_retrieve_from_integer(key[0])
+      else
+        begin
+          retrieve_from_tuples key
+        rescue NoMethodError
+          raise IndexError, "Specified index #{key.inspect} do not exist"
+        end
+      end
+    end
+
+    def valid?(*indexes)
+      # FIXME: This is perhaps not a good method
+      pos(*indexes)
+      true
+    rescue IndexError
+      false
+    end
+
+    # Returns positions given indexes or positions
+    # @note If the arugent is both a valid index and a valid position,
+    #   it will treated as valid index
+    # @param indexes [Array<object>] indexes or positions
+    # @example
+    #   idx = DaruLite::MultiIndex.from_tuples [[:a, :one], [:a, :two], [:b, :one], [:b, :two]]
+    #   idx.pos :a
+    #   # => [0, 1]
+    def pos(*indexes)
+      if indexes.first.is_a? Integer
+        return indexes.first if indexes.size == 1
+
+        return indexes
+      end
+      res = self[indexes]
+      return res if res.is_a? Integer
+
+      res.map { |i| self[i] }
+    end
+
+    def subset(*indexes)
+      if indexes.first.is_a? Integer
+        MultiIndex.from_tuples(indexes.map { |index| key(index) })
+      else
+        self[indexes].conform indexes
+      end
+    end
+
+    # Takes positional values and returns subset of the self
+    #   capturing the indexes at mentioned positions
+    # @param positions [Array<Integer>] positional values
+    # @return [object] index object
+    # @example
+    #   idx = DaruLite::MultiIndex.from_tuples [[:a, :one], [:a, :two], [:b, :one], [:b, :two]]
+    #   idx.at 0, 1
+    #   # => #<DaruLite::MultiIndex(2x2)>
+    #   #   a one
+    #   #     two
+    def at(*positions)
+      positions = preprocess_positions(*positions)
+      validate_positions(*positions)
+      if positions.is_a? Integer
+        key(positions)
+      else
+        DaruLite::MultiIndex.from_tuples(positions.map { |v| key(v) })
+      end
+    end
+
+    def add(*indexes)
+      DaruLite::MultiIndex.from_tuples(to_a + [indexes])
+    end
+
+    def reorder(new_order)
+      from = to_a
+      MultiIndex.from_tuples(new_order.map { |i| from[i] })
+    end
+
+    def try_retrieve_from_integer(int)
+      @levels[0].key?(int) ? retrieve_from_tuples([int]) : int
+    end
+
+    def retrieve_from_range(range)
+      MultiIndex.from_tuples(range.map { |index| key(index) })
+    end
+
+    def retrieve_from_tuples(key)
+      chosen = []
+
+      key.each_with_index do |k, depth|
+        level_index = @levels[depth][k]
+        raise IndexError, "Specified index #{key.inspect} do not exist" if level_index.nil?
+
+        label = @labels[depth]
+        chosen = find_all_indexes label, level_index, chosen
+      end
+
+      return chosen[0] if chosen.size == 1 && key.size == @levels.size
+
+      multi_index_from_multiple_selections(chosen)
+    end
+
+    def multi_index_from_multiple_selections(chosen)
+      MultiIndex.from_tuples(chosen.map { |e| key(e) })
+    end
+
+    def find_all_indexes(label, level_index, chosen)
+      if chosen.empty?
+        label.each_with_index
+             .select { |lbl, _| lbl == level_index }.map(&:last)
+      else
+        chosen.keep_if { |c| label[c] == level_index }
+      end
+    end
+
+    def remove_layer(layer_index)
+      @levels.delete_at(layer_index)
+      @labels.delete_at(layer_index)
+      @name&.delete_at(layer_index)
+
+      coerce_index
+    end
+
+    def coerce_index
+      if @levels.size == 1
+        elements = to_a.flatten
+
+        if elements.uniq.length == elements.length
+          DaruLite::Index.new(elements)
+        else
+          DaruLite::CategoricalIndex.new(elements)
+        end
+      else
+        self
+      end
+    end
+
+    # Array `name` must have same length as levels and labels.
+    def validate_name(names, levels)
+      error_msg = "'names' and 'levels' should be of same size. Size of the " \
+                  "'name' array is #{names.size} and size of the MultiIndex 'levels' and " \
+                  "'labels' is #{labels.size}."
+      suggestion_msg = 'If you do not want to set name for particular level ' \
+                       "(say level 'i') then put empty string on index 'i' of the 'name' Array."
+
+      raise SizeError, error_msg if names.size > levels.size
+      raise SizeError, [error_msg, suggestion_msg].join("\n") if names.size < levels.size
+    end
+
+    private :find_all_indexes, :multi_index_from_multiple_selections,
+            :retrieve_from_range, :retrieve_from_tuples, :validate_name
+
+    def key(index)
+      raise ArgumentError, "Key #{index} is too large" if index >= @labels[0].size
+
+      @labels
+        .each_with_index
+        .map { |label, i| @levels[i].keys[label[index]] }
+    end
+
+    def dup
+      MultiIndex.new levels: levels.dup, labels: labels.dup, name: @name&.dup
+    end
+
+    def drop_left_level(by = 1)
+      MultiIndex.from_arrays to_a.transpose[by..]
+    end
+
+    def |(other)
+      MultiIndex.from_tuples(to_a | other.to_a)
+    end
+
+    def &(other)
+      MultiIndex.from_tuples(to_a & other.to_a)
+    end
+
+    def empty?
+      @labels.flatten.empty? && @levels.all?(&:empty?)
+    end
+
+    def include?(tuple)
+      return false unless tuple.is_a? Enumerable
+
+      @labels[0...tuple.flatten.size]
+        .transpose
+        .include?(tuple.flatten.each_with_index.map { |e, i| @levels[i][e] })
+    end
+
+    def size
+      @labels[0].size
+    end
+
+    def width
+      @levels.size
+    end
+
+    def ==(other)
+      self.class == other.class  &&
+        labels   == other.labels &&
+        levels   == other.levels
+    end
+
+    def to_a
+      (0...size).map { |e| key(e) }
+    end
+
+    def values
+      Array.new(size) { |i| i }
+    end
+
+    def inspect(threshold = 20)
+      "#<DaruLite::MultiIndex(#{size}x#{width})>\n" +
+        Formatters::Table.format([], headers: @name, row_headers: sparse_tuples, threshold: threshold)
+    end
+
+    def to_html
+      path = File.expand_path('../iruby/templates/multi_index.html.erb', __dir__)
+      ERB.new(File.read(path).strip).result(binding)
+    end
+
+    # Provide a MultiIndex for sub vector produced
+    #
+    # @param input_indexes [Array] the input by user to index the vector
+    # @return [Object] the MultiIndex object for sub vector produced
+    def conform(input_indexes)
+      return self if input_indexes[0].is_a? Range
+
+      drop_left_level input_indexes.size
+    end
+
+    # Return tuples with nils in place of repeating values, like this:
+    #
+    # [:a , :bar, :one]
+    # [nil, nil , :two]
+    # [nil, :foo, :one]
+    #
+    def sparse_tuples
+      tuples = to_a
+      [tuples.first] + each_cons(2).map do |prev, cur|
+        left = cur.zip(prev).drop_while { |c, p| c == p }
+        Array.new(cur.size - left.size) + left.map(&:first)
+      end
+    end
+
+    def to_df
+      DaruLite::DataFrame.new(@name.zip(to_a.transpose).to_h)
+    end
+  end
+end

data/lib/daru_lite/io/csv/converters.rb

@@ -0,0 +1,21 @@
+module DaruLite
+  module IO
+    module CSV
+      CONVERTERS = {
+        boolean: lambda { |f, _|
+          case f.downcase.strip
+          when 'true'
+            true
+          when 'false'
+            false
+          else
+            f
+          end
+        },
+        string: lambda { |f, _|
+          f
+        }
+      }.freeze
+    end
+  end
+end

data/lib/daru_lite/io/io.rb

@@ -0,0 +1,294 @@
+module DaruLite
+  require_relative 'csv/converters'
+  module IOHelpers
+    class << self
+      def process_row(row, empty)
+        row.to_a.map do |c|
+          if empty.include?(c)
+            # FIXME: As far as I can guess, it will never work.
+            # It is called only inside `from_plaintext`, and there
+            # data is splitted by `\s+` -- there is no chance that
+            # "empty" (currently just '') will be between data?..
+            nil
+          else
+            try_string_to_number(c)
+          end
+        end
+      end
+
+      def open_local_or_remote_file(path)
+        uri = URI.parse(path)
+        uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS) ? uri.open : File.open(uri.path)
+      end
+
+      private
+
+      INT_PATTERN = /^[-+]?\d+$/.freeze
+      FLOAT_PATTERN = /^[-+]?\d+[,.]?\d*(e-?\d+)?$/.freeze
+
+      def try_string_to_number(s)
+        case s
+        when INT_PATTERN
+          s.to_i
+        when FLOAT_PATTERN
+          s.tr(',', '.').to_f
+        else
+          s
+        end
+      end
+    end
+  end
+
+  module IO
+    class << self
+      # Functions for loading/writing Excel files.
+
+      def from_excel(path, opts = {})
+        opts = {
+          worksheet_id: 0,
+          row_id: 0
+        }.merge opts
+
+        worksheet, headers = read_from_excel(path, opts)
+        df = DaruLite::DataFrame.new({})
+        headers.each_with_index do |h, i|
+          col = worksheet.column(i).to_a
+          col.delete_at 0
+          df[h] = col
+        end
+
+        df
+      end
+
+      def read_from_excel(path, opts)
+        optional_gem 'spreadsheet', '~>1.3.0'
+
+        worksheet_id = opts[:worksheet_id]
+        row_id       = opts[:row_id]
+        book         = Spreadsheet.open path
+        worksheet    = book.worksheet worksheet_id
+        headers      = ArrayHelper.recode_repeated(worksheet.row(row_id)).map(&:to_sym)
+
+        [worksheet, headers]
+      end
+
+      def dataframe_write_excel(dataframe, path, _opts = {})
+        book   = Spreadsheet::Workbook.new
+        sheet  = book.create_worksheet
+        format = Spreadsheet::Format.new color: :blue, weight: :bold
+
+        sheet.row(0).concat(dataframe.vectors.to_a.map(&:to_s)) # Unfreeze strings
+        sheet.row(0).default_format = format
+        i = 1
+        dataframe.each_row do |row|
+          sheet.row(i).concat(row.to_a)
+          i += 1
+        end
+
+        book.write(path)
+      end
+
+      # Functions for loading/writing CSV files
+      def from_csv(path, opts = {})
+        daru_options, opts = from_csv_prepare_opts opts
+        # Preprocess headers for detecting and correcting repetition in
+        # case the :headers option is not specified.
+        hsh =
+          if opts[:headers]
+            from_csv_hash_with_headers(path, opts)
+          else
+            from_csv_hash(path, opts)
+              .tap { |hash| daru_options[:order] = hash.keys }
+          end
+        DaruLite::DataFrame.new(hsh, daru_options)
+      end
+
+      def dataframe_write_csv(dataframe, path, opts = {})
+        options = {
+          converters: :numeric
+        }.merge(opts)
+
+        writer = ::CSV.open(path, 'w', **options)
+        writer << dataframe.vectors.to_a unless options[:headers] == false
+
+        dataframe.each_row do |row|
+          writer << if options[:convert_comma]
+                      row.map { |v| v.to_s.tr('.', ',') }
+                    else
+                      row.to_a
+                    end
+        end
+
+        writer.close
+      end
+
+      # Execute a query and create a data frame from the result
+      #
+      # @param db [DBI::DatabaseHandle, String] A DBI connection OR Path to a SQlite3 database.
+      # @param query [String] The query to be executed
+      #
+      # @return A dataframe containing the data resulting from the query
+      def from_sql(db, query)
+        require 'daru_lite/io/sql_data_source'
+        SqlDataSource.make_dataframe(db, query)
+      end
+
+      def dataframe_write_sql(ds, dbh, table)
+        require 'dbi'
+        query = "INSERT INTO #{table} (#{ds.vectors.to_a.join(',')}) VALUES (#{(['?'] * ds.vectors.size).join(',')})"
+        sth   = dbh.prepare(query)
+        ds.each_row { |c| sth.execute(*c.to_a) }
+        true
+      end
+
+      # Load dataframe from AR::Relation
+      #
+      # @param relation [ActiveRecord::Relation] A relation to be used to load the contents of dataframe
+      #
+      # @return A dataframe containing the data in the given relation
+      def from_activerecord(relation, *fields)
+        fields = relation.klass.column_names if fields.empty?
+        fields = fields.map(&:to_sym)
+
+        result = relation.pluck(*fields).transpose
+        DaruLite::DataFrame.new(result, order: fields).tap(&:update)
+      end
+
+      # Loading data from plain text files
+
+      def from_plaintext(filename, fields)
+        ds = DaruLite::DataFrame.new({}, order: fields)
+        fp = File.open(filename, 'r')
+        fp.each_line do |line|
+          row = DaruLite::IOHelpers.process_row(line.strip.split(/\s+/), [''])
+          next if row == ["\x1A"]
+
+          ds.add_row(row)
+        end
+        ds.update
+        fields.each { |f| ds[f].rename f }
+        ds
+      end
+
+      # Loading and writing Marshalled DataFrame/Vector
+      def save(klass, filename)
+        fp = File.open(filename, 'w')
+        Marshal.dump(klass, fp)
+        fp.close
+      end
+
+      def load(filename)
+        if File.exist? filename
+          o = false
+          File.open(filename, 'r') { |fp| o = Marshal.load(fp) }
+          o
+        else
+          false
+        end
+      end
+
+      private
+
+      def optional_gem(name, version)
+        gem name, version
+        require name
+      rescue LoadError
+        DaruLite.error "\nInstall the #{name} gem version #{version} for using  #{name} functions."
+      end
+
+      DARU_OPT_KEYS = %i[clone order index name].freeze
+
+      def from_csv_prepare_opts(opts)
+        opts[:col_sep]           ||= ','
+        opts[:skip_blanks]       ||= true
+        opts[:converters]        ||= [:numeric]
+
+        opts[:converters] = from_csv_prepare_converters(opts[:converters])
+
+        daru_options = opts.keys.each_with_object({}) do |k, hash|
+          hash[k] = opts.delete(k) if DARU_OPT_KEYS.include?(k)
+        end
+        [daru_options, opts]
+      end
+
+      def from_csv_prepare_converters(converters)
+        Array(converters).flat_map do |c|
+          if ::CSV::Converters[c]
+            ::CSV::Converters[c]
+          elsif DaruLite::IO::CSV::CONVERTERS[c]
+            DaruLite::IO::CSV::CONVERTERS[c]
+          else
+            c
+          end
+        end
+      end
+
+      def from_csv_hash_with_headers(path, opts)
+        opts[:header_converters] ||= :symbol
+        ::CSV
+          .parse(DaruLite::IOHelpers.open_local_or_remote_file(path), **opts)
+          .tap { |c| yield c if block_given? }
+          .by_col.to_h { |col_name, values| [col_name, values] }
+      end
+
+      def from_csv_hash(path, opts)
+        csv_as_arrays =
+          ::CSV
+          .parse(DaruLite::IOHelpers.open_local_or_remote_file(path), **opts)
+          .tap { |c| yield c if block_given? }
+          .to_a
+        headers       = ArrayHelper.recode_repeated(csv_as_arrays.shift)
+        csv_as_arrays = csv_as_arrays.transpose
+        headers.each_with_index.to_h { |h, i| [h, csv_as_arrays[i]] }
+      end
+
+      def html_parse_table(table)
+        headers, headers_size = html_scrape_tag(table, 'th')
+        data, size = html_scrape_tag(table, 'td')
+        data = data.keep_if { |x| x.count == size }
+        order, indice = html_parse_hash(headers, size, headers_size) if headers_size >= size
+        return unless (indice.nil? || indice.count == data.count) && !order.nil? && order.count.positive?
+
+        { data: data.compact, index: indice, order: order }
+      end
+
+      def html_scrape_tag(table, tag)
+        arr  = table.search('tr').map { |row| row.search(tag).map { |val| val.text.strip } }
+        size = arr.map(&:count).max
+        [arr, size]
+      end
+
+      # Splits headers (all th tags) into order and index. Wherein,
+      # Order : All <th> tags on first proper row of HTML table
+      # index : All <th> tags on first proper column of HTML table
+      def html_parse_hash(headers, size, headers_size)
+        headers_index = headers.find_index { |x| x.count == headers_size }
+        order = headers[headers_index]
+        order_index = order.count - size
+        order = order[order_index..]
+        indice = headers[headers_index + 1..].flatten
+        indice = nil if indice.to_a.empty?
+        [order, indice]
+      end
+
+      def html_search(table, match = nil)
+        match.nil? ? true : (table.to_s.include? match)
+      end
+
+      # Allows user to override the scraped order / index / data
+      def html_decide_values(scraped_val = {}, user_val = {})
+        %I[data index name order].each do |key|
+          user_val[key] ||= scraped_val[key]
+        end
+        user_val
+      end
+
+      def html_table_to_dataframe(table)
+        DaruLite::DataFrame.rows table[:data],
+                                 index: table[:index],
+                                 order: table[:order],
+                                 name: table[:name]
+      end
+    end
+  end
+end