daru_lite 0.1

data/lib/daru_lite/configuration.rb

@@ -0,0 +1,34 @@
+module DaruLite
+  # Defines constants and methods related to configuration
+  module Configuration
+    INSPECT_OPTIONS_KEYS = [
+      :max_rows,
+      # Terminal
+      :spacing
+    ].freeze
+
+    # Jupyter
+    DEFAULT_MAX_ROWS = 30
+
+    # Terminal
+    DEFAULT_SPACING = 10
+
+    attr_accessor(*INSPECT_OPTIONS_KEYS)
+
+    def configure
+      yield self
+    end
+
+    def self.extended(base)
+      base.reset_options
+    end
+
+    def reset_options
+      self.max_rows  = DEFAULT_MAX_ROWS
+
+      self.spacing   = DEFAULT_SPACING
+    end
+  end
+
+  extend Configuration
+end

data/lib/daru_lite/core/group_by.rb

@@ -0,0 +1,403 @@
+module DaruLite
+  module Core
+    class GroupBy
+      class << self
+        extend Gem::Deprecate
+
+        # @private
+        def group_by_index_to_positions(indexes_with_positions, sort: false)
+          index_to_positions = {}
+
+          indexes_with_positions.each do |idx, position|
+            (index_to_positions[idx] ||= []) << position
+          end
+
+          if sort # TODO: maybe add a more "stable" sorting option?
+            sorted_keys = index_to_positions.keys.sort(&DaruLite::Core::GroupBy::TUPLE_SORTER)
+            index_to_positions = sorted_keys.to_h { |k| [k, index_to_positions[k]] }
+          end
+
+          index_to_positions
+        end
+        alias get_positions_group_map_on group_by_index_to_positions
+        deprecate :get_positions_group_map_on, :group_by_index_to_positions, 2019, 10
+
+        # @private
+        def get_positions_group_for_aggregation(multi_index, level = -1)
+          raise unless multi_index.is_a?(DaruLite::MultiIndex)
+
+          new_index = multi_index.dup
+          new_index.remove_layer(level) # TODO: recheck code of DaruLite::MultiIndex#remove_layer
+
+          group_by_index_to_positions(new_index.each_with_index)
+        end
+
+        # @private
+        def get_positions_group_map_for_df(df, group_by_keys, sort: true)
+          indexes_with_positions = df[*group_by_keys].to_df.each_row.map(&:to_a).each_with_index
+
+          group_by_index_to_positions(indexes_with_positions, sort: sort)
+        end
+
+        # @private
+        def group_map_from_positions_to_indexes(positions_group_map, index)
+          positions_group_map.transform_values { |positions| positions.map { |pos| index.at(pos) } }
+        end
+
+        # @private
+        def df_from_group_map(df, group_map, remaining_vectors, from_position: true)
+          return nil if group_map == {}
+
+          new_index = group_map.flat_map { |group, values| values.map { |val| group + [val] } }
+          new_index = DaruLite::MultiIndex.from_tuples(new_index)
+
+          return DaruLite::DataFrame.new({}, index: new_index) if remaining_vectors == []
+
+          new_rows_order = group_map.values.flatten
+          new_df = df[*remaining_vectors].to_df.get_sub_dataframe(new_rows_order, by_position: from_position)
+          new_df.index = new_index
+
+          new_df
+        end
+      end
+
+      # The group_by was done over the vectors in group_vectors; the remaining vectors are the non_group_vectors
+      attr_reader :group_vectors, :non_group_vectors
+
+      # lazy accessor/attr_reader for the attribute groups
+      def groups
+        @groups ||= GroupBy.group_map_from_positions_to_indexes(@groups_by_pos, @context.index)
+      end
+      alias groups_by_idx groups
+
+      # lazy accessor/attr_reader for the attribute df
+      def df
+        @df ||= GroupBy.df_from_group_map(@context, @groups_by_pos, @non_group_vectors)
+      end
+      alias grouped_df df
+
+      # Iterate over each group created by group_by. A DataFrame is yielded in
+      # block.
+      def each_group
+        return to_enum(:each_group) unless block_given?
+
+        groups.each_key do |k|
+          yield get_group(k)
+        end
+      end
+
+      TUPLE_SORTER = lambda do |left, right|
+        return -1 unless right
+        return 1 unless left
+
+        left = left.compact
+        right = right.compact
+        return left <=> right || 0 if left.length == right.length
+
+        left.length <=> right.length
+      end
+
+      def initialize(context, names)
+        @group_vectors     = names
+        @non_group_vectors = context.vectors.to_a - names
+
+        @context = context # TODO: maybe rename in @original_df
+
+        # FIXME: It feels like we don't want to sort here. Ruby's #group_by
+        # never sorts:
+        #
+        #   ['test', 'me', 'please'].group_by(&:size)
+        #   #  => {4=>["test"], 2=>["me"], 6=>["please"]}
+        #
+        # - zverok, 2016-09-12
+        @groups_by_pos = GroupBy.get_positions_group_map_for_df(@context, @group_vectors, sort: true)
+      end
+
+      # Get a DaruLite::Vector of the size of each group.
+      def size
+        index = get_grouped_index
+
+        values = @groups_by_pos.values.map(&:size)
+        DaruLite::Vector.new(values, index: index, name: :size)
+      end
+
+      # Get the first group
+      def first
+        head(1)
+      end
+
+      # Get the last group
+      def last
+        tail(1)
+      end
+
+      # Get the top 'n' groups
+      # @param quantity [Fixnum] (5) The number of groups.
+      # @example Usage of head
+      #   df = DaruLite::DataFrame.new({
+      #     a: %w{foo bar foo bar   foo bar foo foo},
+      #     b: %w{one one two three two two one three},
+      #     c:   [1  ,2  ,3  ,1    ,3  ,6  ,3  ,8],
+      #     d:   [11 ,22 ,33 ,44   ,55 ,66 ,77 ,88]
+      #   })
+      #   df.group_by([:a, :b]).head(1)
+      #   # =>
+      #   # #<DaruLite::DataFrame:82745170 @name = d7003f75-5eb9-4967-9303-c08dd9160224 @size = 6>
+      #   #                     a          b          c          d
+      #   #          1        bar        one          2         22
+      #   #          3        bar      three          1         44
+      #   #          5        bar        two          6         66
+      #   #          0        foo        one          1         11
+      #   #          7        foo      three          8         88
+      #   #          2        foo        two          3         33
+      def head(quantity = 5)
+        select_groups_from :first, quantity
+      end
+
+      # Get the bottom 'n' groups
+      # @param quantity [Fixnum] (5) The number of groups.
+      # @example Usage of tail
+      #   df = DaruLite::DataFrame.new({
+      #     a: %w{foo bar foo bar   foo bar foo foo},
+      #     b: %w{one one two three two two one three},
+      #     c:   [1  ,2  ,3  ,1    ,3  ,6  ,3  ,8],
+      #     d:   [11 ,22 ,33 ,44   ,55 ,66 ,77 ,88]
+      #   })
+      #   # df.group_by([:a, :b]).tail(1)
+      #   # =>
+      #   # #<DaruLite::DataFrame:82378270 @name = 0623db46-5425-41bd-a843-99baac3d1d9a @size = 6>
+      #   #                     a          b          c          d
+      #   #          1        bar        one          2         22
+      #   #          3        bar      three          1         44
+      #   #          5        bar        two          6         66
+      #   #          6        foo        one          3         77
+      #   #          7        foo      three          8         88
+      #   #          4        foo        two          3         55
+      def tail(quantity = 5)
+        select_groups_from :last, quantity
+      end
+
+      # Calculate mean of numeric groups, excluding missing values.
+      # @example Usage of mean
+      #   df = DaruLite::DataFrame.new({
+      #     a: %w{foo bar foo bar   foo bar foo foo},
+      #     b: %w{one one two three two two one three},
+      #     c:   [1  ,2  ,3  ,1    ,3  ,6  ,3  ,8],
+      #     d:   [11 ,22 ,33 ,44   ,55 ,66 ,77 ,88]
+      #   df.group_by([:a, :b]).mean
+      #   # =>
+      #   # #<DaruLite::DataFrame:81097450 @name = 0c32983f-3e06-451f-a9c9-051cadfe7371 @size = 6>
+      #   #                         c          d
+      #   # ["bar", "one"]          2         22
+      #   # ["bar", "three"]        1         44
+      #   # ["bar", "two"]          6         66
+      #   # ["foo", "one"]        2.0       44.0
+      #   # ["foo", "three"]        8         88
+      #   # ["foo", "two"]        3.0       44.0
+      def mean
+        apply_method :numeric, :mean
+      end
+
+      # Calculate the median of numeric groups, excluding missing values.
+      def median
+        apply_method :numeric, :median
+      end
+
+      # Calculate sum of numeric groups, excluding missing values.
+      def sum
+        apply_method :numeric, :sum
+      end
+
+      # Count groups, excludes missing values.
+      # @example Using count
+      #   df = DaruLite::DataFrame.new({
+      #     a: %w{foo bar foo bar   foo bar foo foo},
+      #     b: %w{one one two three two two one three},
+      #     c:   [1  ,2  ,3  ,1    ,3  ,6  ,3  ,8],
+      #     d:   [11 ,22 ,33 ,44   ,55 ,66 ,77 ,88]
+      #   })
+      #   df.group_by([:a, :b]).count
+      #   # =>
+      #   # #<DaruLite::DataFrame:76900210 @name = 7b9cf55d-17f8-48c7-b03a-2586c6e5ec5a @size = 6>
+      #   #                           c          d
+      #   # ["bar", "one"]            1          1
+      #   # ["bar", "two"]            1          1
+      #   # ["bar", "three"]          1          1
+      #   # ["foo", "one"]            2          2
+      #   # ["foo", "three"]          1          1
+      #   # ["foo", "two"]            2          2
+      def count
+        width = @non_group_vectors.size
+        DaruLite::DataFrame.new([size] * width, order: @non_group_vectors)
+      end
+
+      # Calculate sample standard deviation of numeric vector groups, excluding
+      # missing values.
+      def std
+        apply_method :numeric, :std
+      end
+
+      # Find the max element of each numeric vector group.
+      def max
+        apply_method :numeric, :max
+      end
+
+      # Find the min element of each numeric vector group.
+      def min
+        apply_method :numeric, :min
+      end
+
+      # Returns one of the selected groups as a DataFrame.
+      # @param group [Array] The group that is to be selected from those grouped.
+      #
+      # @example Getting a group
+      #
+      #   df = DaruLite::DataFrame.new({
+      #         a: %w{foo bar foo bar   foo bar foo foo},
+      #         b: %w{one one two three two two one three},
+      #         c:   [1  ,2  ,3  ,1    ,3  ,6  ,3  ,8],
+      #         d:   [11 ,22 ,33 ,44   ,55 ,66 ,77 ,88]
+      #       })
+      #   df.group_by([:a, :b]).get_group ['bar','two']
+      #   #=>
+      #   ##<DaruLite::DataFrame:83258980 @name = 687ee3f6-8874-4899-97fa-9b31d84fa1d5 @size = 1>
+      #   #                    a          b          c          d
+      #   #         5        bar        two          6         66
+      def get_group(group)
+        indexes   = groups_by_idx[group]
+        elements  = @context.each_vector.map(&:to_a)
+        transpose = elements.transpose
+        rows      = indexes.each.map { |idx| transpose[idx] }
+
+        DaruLite::DataFrame.rows(
+          rows, index: indexes, order: @context.vectors
+        )
+      end
+
+      # Iteratively applies a function to the values in a group and accumulates the result.
+      # @param init (nil) The initial value of the accumulator.
+      # @yieldparam block [Proc] A proc or lambda that accepts two arguments.  The first argument
+      #                          is the accumulated result.  The second argument is a DataFrame row.
+      # @example Usage of reduce
+      #   df = DaruLite::DataFrame.new({
+      #     a: ['a','b'] * 3,
+      #     b: [1,2,3] * 2,
+      #     c: 'A'..'F'
+      #   })
+      #   df.group_by([:a]).reduce('') { |result, row| result += row[:c]; result }
+      #   # =>
+      #   # #<DaruLite::Vector:70343147159900 @name = nil @size = 2 >
+      #   #     nil
+      #   #   a ACE
+      #   #   b BDF
+      def reduce(init = nil)
+        result_hash = groups_by_idx.each_with_object({}) do |(group, indices), h|
+          group_indices = indices.map { |v| @context.index.to_a[v] }
+
+          grouped_result = init
+          group_indices.each do |idx|
+            grouped_result = yield(grouped_result, @context.row[idx])
+          end
+
+          h[group] = grouped_result
+        end
+
+        index = get_grouped_index(result_hash.keys)
+
+        DaruLite::Vector.new(result_hash.values, index: index)
+      end
+
+      def inspect
+        grouped_df.inspect
+      end
+
+      # Function to use for aggregating the data.
+      # `group_by` is using DaruLite::DataFrame#aggregate
+      #
+      # @param options [Hash] options for column, you want in resultant dataframe
+      #
+      # @return [DaruLite::DataFrame]
+      #
+      # @example
+      #
+      #   df = DaruLite::DataFrame.new(
+      #     name: ['Ram','Krishna','Ram','Krishna','Krishna'],
+      #     visited: ['Hyderabad', 'Delhi', 'Mumbai', 'Raipur', 'Banglore'])
+      #
+      #   => #<DaruLite::DataFrame(5x2)>
+      #                   name   visited
+      #            0       Ram Hyderabad
+      #            1   Krishna     Delhi
+      #            2       Ram    Mumbai
+      #            3   Krishna    Raipur
+      #            4   Krishna  Banglore
+      #
+      #   df.group_by(:name)
+      #   => #<DaruLite::DataFrame(5x1)>
+      #                          visited
+      #      Krishna         1     Delhi
+      #                      3    Raipur
+      #                      4  Banglore
+      #          Ram         0 Hyderabad
+      #                      2    Mumbai
+      #
+      #   df.group_by(:name).aggregate(visited: -> (vec){vec.to_a.join(',')})
+      #   => #<DaruLite::DataFrame(2x1)>
+      #                  visited
+      #       Krishna Delhi,Raipur,Banglore
+      #           Ram Hyderabad,Mumbai
+      #
+      def aggregate(options = {})
+        new_index = get_grouped_index
+
+        @context.aggregate(options) { [@groups_by_pos.values, new_index] }
+      end
+
+      private
+
+      def select_groups_from(method, quantity)
+        selection = @context
+        rows = []
+        indexes = []
+
+        groups_by_idx.each_value do |index|
+          index.send(method, quantity).each do |idx|
+            rows << selection.row[idx].to_a
+            indexes << idx
+          end
+        end
+        indexes.flatten!
+
+        DaruLite::DataFrame.rows(rows, order: @context.vectors, index: indexes)
+      end
+
+      def select_numeric_non_group_vectors
+        @non_group_vectors.select { |ngvec| @context[ngvec].type == :numeric }
+      end
+
+      def apply_method(method_type, method)
+        raise 'To implement' if method_type != :numeric
+
+        aggregation_options = select_numeric_non_group_vectors.map { |k| [k, method] }.to_h
+
+        aggregate(aggregation_options)
+      end
+
+      def get_grouped_index(index_tuples = nil)
+        index_tuples = @groups_by_pos.keys if index_tuples.nil?
+
+        if multi_indexed_grouping?
+          DaruLite::MultiIndex.from_tuples(index_tuples)
+        else
+          DaruLite::Index.new(index_tuples.flatten)
+        end
+      end
+
+      def multi_indexed_grouping?
+        return false unless @groups_by_pos.keys[0]
+
+        @groups_by_pos.keys[0].size > 1
+      end
+    end
+  end
+end

data/lib/daru_lite/core/merge.rb

@@ -0,0 +1,270 @@
+module DaruLite
+  module Core
+    class MergeFrame
+      class NilSorter
+        include Comparable
+
+        def nil?
+          true
+        end
+
+        def ==(_other)
+          false
+        end
+
+        def <=>(other)
+          other.nil? ? 0 : -1
+        end
+      end
+
+      # quick-fix for issue #171
+      def initialize(left_df, right_df, opts = {})
+        init_opts(opts)
+        validate_on!(left_df, right_df)
+        key_sanitizer = ->(h) { sanitize_merge_keys(h.values_at(*on)) }
+
+        @left = df_to_a(left_df)
+        @left.sort! { |a, b| safe_compare(a.values_at(*on), b.values_at(*on)) }
+        @left_key_values = @left.map(&key_sanitizer)
+
+        @right = df_to_a(right_df)
+        @right.sort! { |a, b| safe_compare(a.values_at(*on), b.values_at(*on)) }
+        @right_key_values = @right.map(&key_sanitizer)
+
+        @left_keys, @right_keys = merge_keys(left_df, right_df, on)
+      end
+
+      def join
+        res = []
+
+        until left.empty? && right.empty?
+          lkey = first_left_key
+          rkey = first_right_key
+
+          row(lkey, rkey).tap { |r| res << r if r }
+        end
+
+        DaruLite::DataFrame.new(res, order: dataframe_vector_names)
+      end
+
+      private
+
+      attr_reader :on, :indicator,
+                  :left, :left_key_values, :keep_left, :left_keys,
+                  :right, :right_key_values, :keep_right, :right_keys
+
+      attr_accessor :merge_key
+
+      LEFT_RIGHT_COMBINATIONS = {
+        #       left   right
+        inner: [false, false],
+        left: [true, false],
+        right: [false, true],
+        outer: [true, true]
+      }.freeze
+
+      def init_opts(opts)
+        @on = opts[:on]
+        @keep_left, @keep_right = extract_left_right(opts[:how])
+        @indicator = opts[:indicator]
+      end
+
+      def dataframe_vector_names
+        left_keys.values + on + right_keys.values + Array(indicator)
+      end
+
+      def extract_left_right(how)
+        LEFT_RIGHT_COMBINATIONS[how] or
+          raise ArgumentError, "Unrecognized join option: #{how}"
+      end
+
+      def sanitize_merge_keys(merge_keys)
+        merge_keys.map { |v| v.nil? ? NilSorter.new : v }
+      end
+
+      def df_to_a(df)
+        # FIXME: much faster than "native" DataFrame#to_a. Should not be
+        h = df.to_h
+        keys = h.keys
+        h.values.map(&:to_a).transpose.map { |r| keys.zip(r).to_h }
+      end
+
+      def merge_keys(df1, df2, on)
+        duplicates =
+          (df1.vectors.to_a + df2.vectors.to_a - on)
+          .group_by(&:itself)
+          .select { |_, g| g.count == 2 }.map(&:first)
+
+        [
+          guard_keys(df1.vectors.to_a - on, duplicates, 1),
+          guard_keys(df2.vectors.to_a - on, duplicates, 2)
+        ]
+      end
+
+      def guard_keys(keys, duplicates, num)
+        keys.to_h { |v| [v, guard_duplicate(v, duplicates, num)] }
+      end
+
+      def guard_duplicate(val, duplicates, num)
+        duplicates.include?(val) ? :"#{val}_#{num}" : val
+      end
+
+      def row(lkey, rkey)
+        # :nocov:
+        # It's just an impossibility handler, can't be covered :)
+        raise 'Unexpected condition met during merge' if !lkey && !rkey
+
+        # :nocov:
+        if lkey == rkey
+          self.merge_key = lkey
+          add_indicator(merge_matching_rows, :both)
+        elsif !rkey || lt(lkey, rkey)
+          add_indicator(left_row_missing_right, :left_only)
+        else # !lkey || lt(rkey, lkey)
+          add_indicator(right_row_missing_left, :right_only)
+        end
+      end
+
+      def add_indicator(row, indicator_value)
+        return row unless indicator
+
+        row[indicator] = indicator_value
+        row
+      end
+
+      def merge_matching_rows
+        if one_to_one_merge?
+          merge_rows(one_to_one_left_row, one_to_one_right_row)
+        elsif one_to_many_merge?
+          result = merge_rows(left.first, right.first)
+          one_to_many_shift
+          result
+        else
+          result = cartesian_product.shift
+          end_cartesian_product if cartesian_product.empty?
+          result
+        end
+      end
+
+      def one_to_many_shift
+        shift_left = first_right_key != next_right_key
+        shift_right = first_left_key != next_left_key
+        one_to_one_left_row if shift_left
+        one_to_one_right_row if shift_right
+      end
+
+      def one_to_one_merge?
+        merge_key != next_left_key && merge_key != next_right_key
+      end
+
+      def one_to_many_merge?
+        !(merge_key == next_left_key && merge_key == next_right_key)
+      end
+
+      def one_to_one_left_row
+        left_key_values.shift
+        left.shift
+      end
+
+      def one_to_one_right_row
+        right_key_values.shift
+        right.shift
+      end
+
+      def left_row_missing_right
+        val = one_to_one_left_row
+        expand_row(val, left_keys) if keep_left
+      end
+
+      def right_row_missing_left
+        val = one_to_one_right_row
+        expand_row(val, right_keys) if keep_right
+      end
+
+      def lt(k1, k2)
+        (k1 <=> k2) == -1
+      end
+
+      def merge_rows(lrow, rrow)
+        left_keys
+          .to_h { |from, to| [to, lrow[from]] }
+          .merge(on.to_h { |col| [col, lrow[col]] })
+          .merge(indicator ? { indicator => nil } : {})
+          .merge(right_keys.to_h { |from, to| [to, rrow[from]] })
+      end
+
+      def expand_row(row, renamings)
+        renamings
+          .to_h { |from, to| [to, row[from]] }
+          .merge(on.to_h { |col| [col, row[col]] })
+          .merge(indicator ? { indicator => nil } : {})
+      end
+
+      def first_right_key
+        right_key_values.empty? ? nil : right_key_values.first
+      end
+
+      def next_right_key
+        right_key_values[1]
+      end
+
+      def first_left_key
+        left_key_values.empty? ? nil : left_key_values.first
+      end
+
+      def next_left_key
+        left_key_values[1]
+      end
+
+      def left_rows_at_merge_key
+        left.take_while { |arr| sanitize_merge_keys(arr.values_at(*on)) == merge_key }
+      end
+
+      def right_rows_at_merge_key
+        right.take_while { |arr| sanitize_merge_keys(arr.values_at(*on)) == merge_key }
+      end
+
+      def cartesian_product
+        @cartesian_product ||= left_rows_at_merge_key.product(right_rows_at_merge_key).map do |left_row, right_row|
+          merge_rows(left_row, right_row)
+        end
+      end
+
+      def end_cartesian_product
+        left_size = left_rows_at_merge_key.size
+        left_key_values.shift(left_size)
+        left.shift(left_size)
+
+        right_size = right_rows_at_merge_key.size
+        right_key_values.shift(right_size)
+        right.shift(right_size)
+        @cartesian_product = nil
+      end
+
+      def validate_on!(left_df, right_df)
+        @on.each do |on|
+          (left_df.has_vector?(on) && right_df.has_vector?(on)) or
+            raise ArgumentError, "Both dataframes expected to have #{on.inspect} field"
+        end
+      end
+
+      def safe_compare(left_array, right_array)
+        left_array.zip(right_array).map do |l, r|
+          next 0 if l.nil? && r.nil?
+          next 1 if r.nil?
+          next -1 if l.nil?
+
+          l <=> r
+        end.reject(&:zero?).first || 0
+      end
+    end
+
+    module Merge
+      class << self
+        def join(df1, df2, opts = {})
+          MergeFrame.new(df1, df2, opts).join
+        end
+      end
+    end
+  end
+end