daru 0.1.3.1 → 0.1.4

data/lib/daru/core/group_by.rb

@@ -11,20 +11,21 @@ module Daru
         end
       end
 
+      TUPLE_SORTER = lambda do |a, b|
+        if a && b
+          a.compact <=> b.compact
+        else
+          a ? 1 : -1
+        end
+      end
+
       def initialize context, names
         @groups = {}
         @non_group_vectors = context.vectors.to_a - names
         @context = context
         vectors = names.map { |vec| context[vec].to_a }
         tuples  = vectors[0].zip(*vectors[1..-1])
-        keys    =
-          tuples.uniq.sort do |a,b|
-            if a && b
-              a.compact <=> b.compact
-            else
-              a ? 1 : -1
-            end
-          end
+        keys    = tuples.uniq.sort(&TUPLE_SORTER)
 
         keys.each do |key|
           @groups[key] = all_indices_for(tuples, key)
@@ -189,17 +190,9 @@ module Daru
       #   #         5        bar        two          6         66
       def get_group group
         indexes   = @groups[group]
-        elements  = []
-
-        @context.each_vector do |vector|
-          elements << vector.to_a
-        end
-        rows = []
+        elements  = @context.each_vector.map(&:to_a)
         transpose = elements.transpose
-
-        indexes.each do |idx|
-          rows << transpose[idx]
-        end
+        rows      = indexes.each.map { |idx| transpose[idx] }
 
         new_index =
           begin
@@ -207,6 +200,7 @@ module Daru
           rescue IndexError
             indexes
           end
+
         Daru::DataFrame.rows(
           rows, index: new_index, order: @context.vectors
         )
@@ -224,7 +218,7 @@ module Daru
       #   })
       #   df.group_by([:a]).reduce('') { |result, row| result += row[:c]; result }
       #   # =>
-      #   # #<Daru::Vector:70343147159900 @name = nil @metadata = {} @size = 2 >
+      #   # #<Daru::Vector:70343147159900 @name = nil @size = 2 >
       #   #     nil
       #   #   a ACE
       #   #   b BDF
@@ -268,33 +262,30 @@ module Daru
       end
 
       def apply_method method_type, method
-        multi_index = multi_indexed_grouping?
-        rows, order = [], []
-
-        @groups.each do |_group, indexes|
-          single_row = []
-          @non_group_vectors.each do |ngvector|
-            vec = @context[ngvector]
-            if method_type == :numeric && vec.type == :numeric
-              slice = vec[*indexes]
-              single_row << (slice.is_a?(Daru::Vector) ? slice.send(method) : slice)
-            end
-          end
-
-          rows << single_row
+        order = @non_group_vectors.select do |ngvec|
+          method_type == :numeric && @context[ngvec].type == :numeric
         end
 
-        @non_group_vectors.each do |ngvec|
-          order << ngvec if
-            method_type == :numeric && @context[ngvec].type == :numeric
+        rows = @groups.map do |_group, indexes|
+          order.map do |ngvector|
+            slice = @context[ngvector][*indexes]
+            slice.is_a?(Daru::Vector) ? slice.send(method) : slice
+          end
         end
 
-        index = @groups.keys
-        index = multi_index ? Daru::MultiIndex.from_tuples(index) : Daru::Index.new(index.flatten)
+        index = apply_method_index
         order = Daru::Index.new(order)
         Daru::DataFrame.new(rows.transpose, index: index, order: order)
       end
 
+      def apply_method_index
+        if multi_indexed_grouping?
+          Daru::MultiIndex.from_tuples(@groups.keys)
+        else
+          Daru::Index.new(@groups.keys.flatten)
+        end
+      end
+
       def all_indices_for arry, element
         found, index, indexes = -1, -1, []
         while found

data/lib/daru/core/merge.rb

@@ -1,210 +1,251 @@
 module Daru
   module Core
-    module MergeHelper
-      class << self
-        def replace_keys_if_duplicates hash, matcher
-          matched = nil
-          hash.keys.each { |d|
-            if matcher.match(Regexp.new(d.to_s))
-              matched = d
-              break
-            end
-          }
-
-          return unless matched
-
-          hash[matcher] = hash[matched]
-          hash.delete matched
-        end
+    class MergeFrame
+      class NilSorter
+        include Comparable
 
-        def resolve_duplicates df_hash1, df_hash2, on
-          hk = df_hash1.keys + df_hash2.keys - on
-          recoded = hk.recode_repeated.map(&:to_sym)
-          diff = (recoded - hk).sort
+        def nil?
+          true
+        end
 
-          diff.each_slice(2) do |a|
-            replace_keys_if_duplicates df_hash1, a[0]
-            replace_keys_if_duplicates df_hash2, a[1]
-          end
+        def ==(_other)
+          false
         end
 
-        def hashify df
-          hsh = df.to_h
-          hsh.each { |k,v| hsh[k] = v.to_a }
-          hsh
+        def <=>(other)
+          other.nil? ? 0 : -1
         end
+      end
 
-        def arrayify df
-          arr = df.to_a
-          col_names = arr[0][0].keys
-          values = arr[0].map(&:values)
+      def initialize left_df, right_df, opts={}
+        @on = opts[:on]
+        @keep_left, @keep_right = extract_left_right(opts[:how])
 
-          [col_names, values]
-        end
+        validate_on!(left_df, right_df)
 
-        def arrayify_with_sort_keys(size, df_hash, on)
-          # Converting to a hash and then to an array is more complex
-          # than using df.to_a or df.map(:row).  However, it's
-          # substantially faster this way.
-
-          # idx_keys = on.map { |key| df_hash.keys.index(key) }
-
-          (0...size).reduce([]) do |r, idx|
-            key_values = on.map { |col| df_hash[col][idx] }
-            row_values = df_hash.map { |_col, val| val[idx] }
-            r << [key_values, row_values]
-          end
-
-          # Conceptually simpler and does the same thing, but slows down the
-          # total merge algorithm by 2x.  Would be nice to improve the performance
-          # of df.map(:row)
-          #
-          # df.map(:row) do |row|
-          #   key_values = on.map { |key| row[key] }
-          #   [key_values, row.to_a]
-          # end
-        end
+        key_sanitizer = ->(h) { sanitize_merge_keys(h.values_at(*on)) }
+
+        @left = df_to_a(left_df)
+        @left.sort_by!(&key_sanitizer)
+        @left_key_values = @left.map(&key_sanitizer)
+
+        @right = df_to_a(right_df)
+        @right.sort_by!(&key_sanitizer)
+        @right_key_values = @right.map(&key_sanitizer)
+
+        @left_keys, @right_keys = merge_keys(left_df, right_df, on)
+      end
 
-        def verify_dataframes df_hash1, df_hash2, on
-          raise ArgumentError,
-            'All fields in :on must be present in self' unless on.all? { |e| df_hash1[e] }
-          raise ArgumentError,
-            'All fields in :on must be present in other DF' unless on.all? { |e| df_hash2[e] }
+      def join
+        res = []
+
+        until left.empty? && right.empty?
+          lkey = first_left_key
+          rkey = first_right_key
+
+          row(lkey, rkey).tap { |r| res << r if r }
         end
+
+        Daru::DataFrame.new(res, order: left_keys.values + on + right_keys.values)
       end
-    end
 
-    class MergeFrame
-      def initialize(df1, df2, on: nil)
-        @df1 = df1
-        @df2 = df2
-        @on = on
+      private
+
+      attr_reader :on,
+        :left, :left_key_values, :keep_left, :left_keys,
+        :right, :right_key_values, :keep_right, :right_keys
+
+      attr_accessor :merge_key
+
+      LEFT_RIGHT_COMBINATIONS = {
+        #       left   right
+        inner: [false, false],
+        left:  [true, false],
+        right: [false, true],
+        outer: [true, true]
+      }.freeze
+
+      def extract_left_right(how)
+        LEFT_RIGHT_COMBINATIONS[how] or
+          raise ArgumentError, "Unrecognized join option: #{how}"
+      end
+
+      def sanitize_merge_keys(merge_keys)
+        merge_keys.map { |v| v || NilSorter.new }
       end
 
-      def inner _opts
-        merge_join(left: false, right: false)
+      def df_to_a df
+        # FIXME: much faster than "native" DataFrame#to_a. Should not be
+        h = df.to_h
+        keys = h.keys
+        h.values.map(&:to_a).transpose.map { |r| keys.zip(r).to_h }
       end
 
-      def left _opts
-        merge_join(left: true, right: false)
+      def merge_keys(df1, df2, on)
+        duplicates =
+          (df1.vectors.to_a + df2.vectors.to_a - on)
+          .group_by(&:itself)
+          .select { |_, g| g.count == 2 }.map(&:first)
+
+        [
+          guard_keys(df1.vectors.to_a - on, duplicates, 1),
+          guard_keys(df2.vectors.to_a - on, duplicates, 2)
+        ]
       end
 
-      def right _opts
-        merge_join(left: false, right: true)
+      def guard_keys keys, duplicates, num
+        keys.map { |v| [v, guard_duplicate(v, duplicates, num)] }.to_h
       end
 
-      def outer _opts
-        merge_join(left: true, right: true)
+      def guard_duplicate val, duplicates, num
+        duplicates.include?(val) ? :"#{val}_#{num}" : val
       end
 
-      def merge_join(left: true, right: true)
-        MergeHelper.verify_dataframes df1_hash, df2_hash, @on
-        MergeHelper.resolve_duplicates df1_hash, df2_hash, @on
+      def row(lkey, rkey)
+        case
+        when !lkey && !rkey
+          # :nocov:
+          # It's just an impossibility handler, can't be covered :)
+          raise 'Unexpected condition met during merge'
+          # :nocov:
+        when lkey == rkey
+          self.merge_key = lkey
+          merge_matching_rows
+        when !rkey || lt(lkey, rkey)
+          left_row_missing_right
+        else # !lkey || lt(rkey, lkey)
+          right_row_missing_left
+        end
+      end
 
-        # TODO: Use native dataframe sorting.
-        #  It would be ideal to reuse sorting functionality that is native
-        #  to dataframes.  Unfortunately, native dataframe sort introduces
-        #  an overhead that reduces join performance by a factor of 4!  Until
-        #  that aspect is improved, we resort to a simpler array sort.
-        df1_array.sort_by! { |row| [row[0].nil? ? 0 : 1, row[0]] }
-        df2_array.sort_by! { |row| [row[0].nil? ? 0 : 1, row[0]] }
+      def merge_matching_rows
+        if one_to_one_merge?
+          merge_rows(one_to_one_left_row, one_to_one_right_row)
+        elsif one_to_many_merge?
+          merge_rows(one_to_many_left_row, one_to_many_right_row)
+        else
+          result = cartesian_product.shift
+          end_cartesian_product if cartesian_product.empty?
+          result
+        end
+      end
 
-        idx1 = 0
-        idx2 = 0
+      def one_to_one_merge?
+        merge_key != next_left_key && merge_key != next_right_key
+      end
 
-        while idx1 < @df1.size || idx2 < @df2.size
+      def one_to_many_merge?
+        !(merge_key == next_left_key && merge_key == next_right_key)
+      end
 
-          key1 = df1_array[idx1][0] if idx1 < @df1.size
-          key2 = df2_array[idx2][0] if idx2 < @df2.size
+      def one_to_one_left_row
+        left_key_values.shift
+        left.shift
+      end
 
-          if key1 == key2 && idx1 < @df1.size && idx2 < @df2.size
-            idx2_start = idx2
+      def one_to_many_left_row
+        if next_right_key && first_right_key == next_right_key
+          left.first
+        else
+          left_key_values.shift
+          left.shift
+        end
+      end
 
-            while (idx2 < @df2.size) && (df1_array[idx1][0] == df2_array[idx2][0])
-              add_merge_row_to_hash([df1_array[idx1], df2_array[idx2]], joined_hash)
-              idx2 += 1
-            end
+      def one_to_one_right_row
+        right_key_values.shift
+        right.shift
+      end
 
-            idx2 = idx2_start if idx1+1 < @df1.size && df1_array[idx1][0] == df1_array[idx1+1][0]
-            idx1 += 1
-          elsif ((key2.nil? || [key1,key2].sort == [key1,key2]) && idx1 < @df1.size) || idx2 == @df2.size
-            add_merge_row_to_hash([df1_array[idx1], nil], joined_hash) if left
-            idx1 += 1
-          elsif idx2 < @df2.size || idx1 == @df1.size
-            add_merge_row_to_hash([nil, df2_array[idx2]], joined_hash) if right
-            idx2 += 1
-          else
-            raise 'Unexpected condition met during merge'
-          end
+      def one_to_many_right_row
+        if next_left_key && first_left_key == next_left_key
+          right.first
+        else
+          right_key_values.shift
+          right.shift
         end
+      end
 
-        Daru::DataFrame.new(joined_hash, order: joined_hash.keys)
+      def left_row_missing_right
+        val = one_to_one_left_row
+        expand_row(val, left_keys) if keep_left
       end
 
-      private
+      def right_row_missing_left
+        val = one_to_one_right_row
+        expand_row(val, right_keys) if keep_right
+      end
 
-      def joined_hash
-        return @joined_hash if @joined_hash
-        @joined_hash ||= {}
+      def lt(k1, k2)
+        (k1 <=> k2) == -1
+      end
 
-        ((df1_keys - @on) | @on | (df2_keys - @on)).each do |k|
-          @joined_hash[k] = []
-        end
+      def merge_rows lrow, rrow
+        left_keys
+          .map { |from, to| [to, lrow[from]] }.to_h
+          .merge(on.map { |col| [col, lrow[col]] }.to_h)
+          .merge(right_keys.map { |from, to| [to, rrow[from]] }.to_h)
+      end
 
-        @joined_hash
+      def expand_row row, renamings
+        renamings
+          .map { |from, to| [to, row[from]] }.to_h
+          .merge(on.map { |col| [col, row[col]] }.to_h)
       end
 
-      def df1_hash
-        @df1_hash ||= MergeHelper.hashify @df1
+      def first_right_key
+        right_key_values.empty? ? nil : right_key_values.first
       end
 
-      def df2_hash
-        @df2_hash ||= MergeHelper.hashify @df2
+      def next_right_key
+        right_key_values.size <= 1 ? nil : right_key_values[1]
       end
 
-      def df1_array
-        @df1_array ||= MergeHelper.arrayify_with_sort_keys @df1.size, df1_hash, @on
+      def first_left_key
+        left_key_values.empty? ? nil : left_key_values.first
       end
 
-      def df2_array
-        @df2_array ||= MergeHelper.arrayify_with_sort_keys @df2.size, df2_hash, @on
+      def next_left_key
+        left_key_values.size <= 1 ? nil : left_key_values[1]
       end
 
-      def df1_keys
-        df1_hash.keys
+      def left_rows_at_merge_key
+        left.take_while { |arr| sanitize_merge_keys(arr.values_at(*on)) == merge_key }
       end
 
-      def df2_keys
-        df2_hash.keys
+      def right_rows_at_merge_key
+        right.take_while { |arr| sanitize_merge_keys(arr.values_at(*on)) == merge_key }
       end
 
-      # Private: The merge row contains two elements, the first is the row from the
-      # first dataframe, the second is the row from the second dataframe.
-      def add_merge_row_to_hash row, hash
-        @df1_key_to_index ||= df1_keys.each_with_index.map { |k,idx| [k, idx] }.to_h
-        @df2_key_to_index ||= df2_keys.each_with_index.map { |k,idx| [k, idx] }.to_h
+      def cartesian_product
+        @cartesian_product ||= left_rows_at_merge_key.product(right_rows_at_merge_key).map do |left_row, right_row|
+          merge_rows(left_row, right_row)
+        end
+      end
 
-        hash.each do |k,v|
-          v ||= []
+      def end_cartesian_product
+        left_size = left_rows_at_merge_key.size
+        left_key_values.shift(left_size)
+        left.shift(left_size)
 
-          left  = df1_keys.include?(k) ? row[0] && row[0][1][@df1_key_to_index[k]] : nil
-          right = df2_keys.include?(k) ? row[1] && row[1][1][@df2_key_to_index[k]] : nil
+        right_size = right_rows_at_merge_key.size
+        right_key_values.shift(right_size)
+        right.shift(right_size)
+        @cartesian_product = nil
+      end
 
-          v << (left || right)
+      def validate_on!(left_df, right_df)
+        @on.each do |on|
+          left_df.has_vector?(on) && right_df.has_vector?(on) or
+            raise ArgumentError, "Both dataframes expected to have #{on.inspect} field"
         end
       end
     end
 
-    # Private module containing methods for join, merge, concat operations on
-    # dataframes and vectors.
-    # @private
     module Merge
       class << self
         def join df1, df2, opts={}
-          on = opts[:on]
-
-          mf = MergeFrame.new df1, df2, on: on
-          mf.send opts[:how], {}
+          MergeFrame.new(df1, df2, opts).join
         end
       end
     end