red_amber 0.1.7 → 0.2.1

data/lib/red_amber/data_frame_variable_operation.rb

@@ -9,12 +9,16 @@ module RedAmber
       if block
         raise DataFrameArgumentError, 'Must not specify both arguments and block.' unless args.empty?
 
-        picker = instance_eval(&block)
+        picker = [instance_eval(&block)]
       end
-      picker = [picker].flatten
+      picker.flatten!
       return DataFrame.new if picker.empty? || picker == [nil]
 
-      picker = keys_by_booleans(picker) if booleans?(picker)
+      key_vector = Vector.new(keys)
+      picker_vector = parse_to_vector(picker)
+
+      picker = key_vector.filter(*picker_vector).to_a if picker_vector.boolean?
+      picker = key_vector.take(*picker_vector).to_a if picker_vector.numeric?
 
       # DataFrame#[] creates a Vector with single key is specified.
       # DataFrame#pick creates a DataFrame with single key.
@@ -29,12 +33,22 @@ module RedAmber
       if block
         raise DataFrameArgumentError, 'Must not specify both arguments and block.' unless args.empty?
 
-        dropper = instance_eval(&block)
+        dropper = [instance_eval(&block)]
       end
-      dropper = [dropper].flatten
-      dropper = keys_by_booleans(dropper) if booleans?(dropper)
+      dropper.flatten!
+
+      key_vector = Vector.new(keys)
+      dropper_vector = parse_to_vector(dropper)
+
+      picker =
+        if dropper_vector.boolean?
+          key_vector.filter(*dropper_vector.primitive_invert).each.map(&:to_sym)
+        elsif dropper_vector.numeric?
+          keys - key_vector.take(*dropper_vector).each.map(&:to_sym)
+        else
+          keys - dropper
+        end
 
-      picker = keys - dropper
       return DataFrame.new if picker.empty?
 
       # DataFrame#[] creates a Vector with single key is specified.
@@ -44,64 +58,118 @@ module RedAmber
       raise DataFrameArgumentError, "Invalid argument #{args}"
     end
 
-    # rename variables to create new DataFrame
-    def rename(*args, &block)
-      renamer = args
+    # rename variables to create a new DataFrame
+    def rename(*renamer, &block)
       if block
-        raise DataFrameArgumentError, 'Must not specify both arguments and a block' unless args.empty?
+        raise DataFrameArgumentError, 'Must not specify both arguments and a block' unless renamer.empty?
 
-        renamer = instance_eval(&block)
+        renamer = [instance_eval(&block)]
+      end
+      case renamer
+      in [] | [nil] | [{}] | [[]]
+        return self
+      in [Hash => key_pairs]
+      # noop
+      in [ (Symbol | String) => from, (Symbol | String) => to]
+        key_pairs = { from => to }
+      in [Array => array_in_array]
+        key_pairs = try_convert_to_hash(array_in_array)
+      in [Array, *] => array_in_array1
+        key_pairs = try_convert_to_hash(array_in_array1)
+      else
+        raise DataFrameArgumentError, "Invalid argument #{renamer}"
       end
-      renamer = [renamer].flatten
-      return self if renamer.empty?
+      rename_by_hash(key_pairs)
+    end
 
-      return rename_by_hash([renamer].to_h) if renamer.size == 2 && sym_or_str?(renamer) # rename(from, to)
-      return rename_by_hash(renamer[0]) if renamer.one? && renamer[0].is_a?(Hash) # rename({from => to})
+    # assign variables to create a new DataFrame
+    def assign(*assigner, &block)
+      appender, fields, arrays = assign_update(*assigner, &block)
+      return self if appender.is_a?(DataFrame)
 
-      raise DataFrameArgumentError, "Invalid argument #{args}"
+      append_to_fields_and_arrays(appender, fields, arrays, append_to_left: false) unless appender.empty?
+
+      DataFrame.new(Arrow::Table.new(Arrow::Schema.new(fields), arrays))
     end
 
-    # assign variables to create new DataFrame
-    def assign(*args, &block)
-      assigner = args
-      if block
-        raise DataFrameArgumentError, 'Must not specify both arguments and a block' unless args.empty?
+    def assign_left(*assigner, &block)
+      appender, fields, arrays = assign_update(*assigner, &block)
+      return self if appender.is_a?(DataFrame)
+
+      append_to_fields_and_arrays(appender, fields, arrays, append_to_left: true) unless appender.empty?
 
-        assigner = instance_eval(&block)
+      DataFrame.new(Arrow::Table.new(Arrow::Schema.new(fields), arrays))
+    end
+
+    private
+
+    def assign_update(*assigner, &block)
+      if block
+        assigner_from_block = instance_eval(&block)
+        assigner =
+          if assigner.empty?
+            # block only
+            [assigner_from_block]
+          # If Ruby >= 3.0, one line pattern match can be used
+          # assigner_from_block in [Array, *]
+          elsif multiple_assigner?(assigner_from_block)
+            assigner.zip(assigner_from_block)
+          else
+            assigner.zip([assigner_from_block])
+          end
       end
-      assigner = [assigner].flatten
-      return self if assigner.empty? || assigner == [nil]
 
-      raise DataFrameArgumentError, "Invalid argument #{args}" unless assigner.one? && assigner[0].is_a?(Hash)
+      case assigner
+      in [] | [nil] | [{}] | [[]]
+        return self
+      in [Hash => key_array_pairs]
+      # noop
+      in [(Symbol | String) => key, (Vector | Array | Arrow::Array) => array]
+        key_array_pairs = { key => array }
+      in [Array => array_in_array]
+        key_array_pairs = try_convert_to_hash(array_in_array)
+      in [Array, *] => array_in_array1
+        key_array_pairs = try_convert_to_hash(array_in_array1)
+      else
+        raise DataFrameArgumentError, "Invalid argument #{assigner}"
+      end
 
       updater = {}
       appender = {}
-      assigner[0].each do |key, value|
+      key_array_pairs.each do |key, array|
+        raise DataFrameArgumentError, "Empty column data: #{key} => nil" if array.nil?
+
         if keys.include? key
-          updater[key] = value
+          updater[key] = array
         else
-          appender[key] = value
+          appender[key] = array
         end
       end
-      fields, arrays = update_fields_and_arrays(updater)
-      append_to_fields_and_arrays(appender, fields, arrays) unless appender.empty?
-
-      DataFrame.new(Arrow::Table.new(Arrow::Schema.new(fields), arrays))
+      [appender, *update_fields_and_arrays(updater)]
     end
 
-    private
+    def try_convert_to_hash(array)
+      array.to_h
+    rescue TypeError
+      [array].to_h
+    rescue TypeError # rubocop:disable Lint/DuplicateRescueException
+      raise DataFrameArgumentError, "Invalid argument in Array #{array}"
+    end
 
     def rename_by_hash(key_pairs)
-      fields = keys.map do |key|
-        new_key = key_pairs[key]
-        if new_key
-          Arrow::Field.new(new_key.to_sym, @table[key].data_type)
-        else
-          @table.schema[key]
+      not_existing_keys = key_pairs.keys - keys
+      raise DataFrameArgumentError, "Not existing: #{not_existing_keys}" unless not_existing_keys.empty?
+
+      fields =
+        keys.map do |key|
+          new_key = key_pairs[key]
+          if new_key
+            Arrow::Field.new(new_key.to_sym, @table[key].data_type)
+          else
+            @table.schema[key]
+          end
         end
-      end
-      schema = Arrow::Schema.new(fields)
-      DataFrame.new(Arrow::Table.new(schema, @table.columns))
+      DataFrame.new(Arrow::Table.new(Arrow::Schema.new(fields), @table.columns))
     end
 
     def update_fields_and_arrays(updater)
@@ -111,7 +179,7 @@ module RedAmber
         data = updater[key]
         next unless data
 
-        raise DataFrameArgumentError, "Data size mismatch (#{data.size} != #{size})" if data.size != size
+        raise DataFrameArgumentError, "Data size mismatch (#{data.size} != #{size})" if data.nil? || data.size != size
 
         a = Arrow::Array.new(data.is_a?(Vector) ? data.to_a : data)
         fields[i] = Arrow::Field.new(key, a.value_data_type)
@@ -120,18 +188,30 @@ module RedAmber
       [fields, arrays]
     end
 
-    def append_to_fields_and_arrays(appender, fields, arrays)
-      appender.each do |key, data|
+    def append_to_fields_and_arrays(appender, fields, arrays, append_to_left: false)
+      enum = append_to_left ? appender.reverse_each : appender.each
+      enum.each do |key, data|
         raise DataFrameArgumentError, "Data size mismatch (#{data.size} != #{size})" if data.size != size
 
         a = Arrow::Array.new(data.is_a?(Vector) ? data.to_a : data)
-        fields << Arrow::Field.new(key.to_sym, a.value_data_type)
-        arrays << Arrow::ChunkedArray.new([a])
+
+        if append_to_left
+          fields.unshift(Arrow::Field.new(key.to_sym, a.value_data_type))
+          arrays.unshift(Arrow::ChunkedArray.new([a]))
+        else
+          fields << Arrow::Field.new(key.to_sym, a.value_data_type)
+          arrays << Arrow::ChunkedArray.new([a])
+        end
       end
     end
 
-    def keys_by_booleans(booleans)
-      keys.select.with_index { |_, i| booleans[i] }
+    def multiple_assigner?(assigner)
+      case assigner
+      in [Vector, *] | [Array, *] | [Arrow::Array, *]
+        true
+      else
+        false
+      end
     end
   end
 end

data/lib/red_amber/group.rb

@@ -3,6 +3,10 @@
 module RedAmber
   # group class
   class Group
+    # Creates a new Group object.
+    #
+    # @param dataframe [DataFrame] dataframe to be grouped.
+    # @param group_keys [Array<>] keys for grouping.
     def initialize(dataframe, *group_keys)
       @dataframe = dataframe
       @table = @dataframe.table
@@ -16,36 +20,30 @@ module RedAmber
       @group = @table.group(*@group_keys)
     end
 
-    def count(*summary_keys)
-      by(:count, summary_keys)
+    functions = %i[count sum product mean min max stddev variance]
+    functions.each do |function|
+      define_method(function) do |*summary_keys|
+        by(function, summary_keys)
+      end
     end
 
-    def sum(*summary_keys)
-      by(:sum, summary_keys)
+    def inspect
+      tallys = @dataframe.pick(@group_keys).vectors.map.with_object({}) do |v, h|
+        h[v.key] = v.tally
+      end
+      "#<#{self.class}:#{format('0x%016x', object_id)}\n#{tallys}>"
     end
 
-    def product(*summary_keys)
-      by(:product, summary_keys)
-    end
-
-    def mean(*summary_keys)
-      by(:mean, summary_keys)
-    end
-
-    def min(*summary_keys)
-      by(:min, summary_keys)
-    end
-
-    def max(*summary_keys)
-      by(:max, summary_keys)
-    end
-
-    def stddev(*summary_keys)
-      by(:stddev, summary_keys)
-    end
-
-    def variance(*summary_keys)
-      by(:variance, summary_keys)
+    def summarize(&block)
+      agg = instance_eval(&block)
+      case agg
+      when DataFrame
+        agg
+      when Array
+        agg.reduce { |aggregated, df| aggregated.assign(df.to_h) }
+      else
+        raise GroupArgumentError, "Unknown argument: #{agg}"
+      end
     end
 
     private
@@ -55,7 +53,11 @@ module RedAmber
       d = summary_keys - @dataframe.keys
       raise GroupArgumentError, "#{d} is not a key of\n #{@dataframe}." unless summary_keys.empty? || d.empty?
 
-      RedAmber::DataFrame.new(@group.send(func, *summary_keys))
+      df = RedAmber::DataFrame.new(@group.send(func, *summary_keys))
+      df = df[@group_keys, df.keys - @group_keys]
+      # if counts are the same (no nil included), aggregate count columns.
+      df = df[df.keys[0..1]].rename(df.keys[1], :count) if func == :count && df.to_h.values[1..].uniq.size == 1
+      df
     end
   end
 end

data/lib/red_amber/vector.rb

@@ -122,7 +122,7 @@ module RedAmber
       return enum_for(:each) unless block_given?
 
       size.times do |i|
-        yield self[i]
+        yield data[i]
       end
     end
 

data/lib/red_amber/vector_functions.rb

@@ -34,18 +34,56 @@ module RedAmber
     end
     alias_method :std, :sd
 
-    # option(s) required
-    # - index
+    # Return quantile
+    #   0.5 quantile (median) is returned by default.
+    #   Or return quantile for specified probability (prob).
+    #   If quantile lies between two data points, interpolated value is
+    #   returned based on selected interpolation method.
+    #   Nils and NaNs are ignored.
+    #   Nil is returned if there are no valid data point.
+    #
+    # @param prob [Float] probability.
+    # @param interpolation [Symbol] specifies interpolation method to use,
+    #   when the quantile lies between the data i and j.
+    #   - Default value is :linear, which returns i + (j - i) * fraction.
+    #   - :lower returns i.
+    #   - :higher returns j.
+    #   - :nearest returns i or j, whichever is closer.
+    #   - :midpoint returns (i + j) / 2.
+    # @param skip_nils [Boolean] wheather to ignore nil.
+    # @param min_count [Integer] min count.
+    # @return [Float] quantile.
+    def quantile(prob = 0.5, interpolation: :linear, skip_nils: true, min_count: 0)
+      raise VectorArgumentError, "Invalid: probability #{prob} must be between 0 and 1" unless (0..1).cover? prob
+
+      datum = find(:quantile).execute([data],
+                                      q: prob,
+                                      interpolation: interpolation,
+                                      skip_nulls: skip_nils,
+                                      min_count: min_count)
+      datum.value.to_a.first
+    end
 
-    # Returns other than value
-    # - mode
-    # - quantile
-    # - tdigest
+    # Return quantiles in a DataFrame
+    #
+    def quantiles(probs = [1.0, 0.75, 0.5, 0.25, 0.0], interpolation: :linear, skip_nils: true, min_count: 0)
+      if probs.empty? || !probs.all? { |q| (0..1).cover?(q) }
+        raise VectorArgumentError, "Invarid probavilities #{probs}"
+      end
+
+      DataFrame.new(
+        probs: probs,
+        quantiles: probs.map do |q|
+          quantile(q, interpolation: interpolation, skip_nils: skip_nils, min_count: min_count)
+        end
+      )
+    end
 
     # [Unary element-wise]: vector.func => vector
     unary_element_wise =
-      %i[abs array_sort_indices atan bit_wise_not ceil cos fill_null_backward fill_null_forward floor is_finite
-         is_inf is_nan is_null is_valid round round_to_multiple sign sin tan trunc unique]
+      %i[abs acos asin array_sort_indices atan bit_wise_not ceil cos fill_null_backward \
+         fill_null_forward floor is_finite is_inf is_nan is_null is_valid ln log10 log1p log2 \
+         round round_to_multiple sign sin tan trunc unique]
     unary_element_wise.each do |function|
       define_method(function) do |**options|
         datum = exec_func_unary(function, options)
@@ -63,6 +101,7 @@ module RedAmber
 
     alias_method :sort_indexes, :array_sort_indices
     alias_method :sort_indices, :array_sort_indices
+    alias_method :sort_index, :array_sort_indices
 
     alias_method :uniq, :unique
 
@@ -84,16 +123,9 @@ module RedAmber
     end
     alias_method :not, :invert
 
-    # NaN support needed
-    # - acos asin ln log10 log1p log2
-
-    # Functions with numerical range check
-    # - abs_checked acos_checked asin_checked cos_checked ln_checked
-    #   log10_checked log1p_checked log2_checked sin_checked tan_checked
-
     # [Binary element-wise]: vector.func(other) => vector
     binary_element_wise =
-      %i[atan2 and_not and_not_kleene bit_wise_and bit_wise_or bit_wise_xor]
+      %i[atan2 and_not and_not_kleene bit_wise_and bit_wise_or bit_wise_xor logb]
     binary_element_wise.each do |function|
       define_method(function) do |other, **options|
         datum = exec_func_binary(function, other, options)
@@ -117,13 +149,6 @@ module RedAmber
       end
     end
 
-    # NaN support needed
-    # - logb
-
-    # Functions with numerical range check
-    # - add_checked divide_checked logb_checked multiply_checked power_checked subtract_checked
-    #   shift_left_checked shift_right_checked
-
     # [Binary element-wise with operator]: vector.func(other) => vector
     binary_element_wise_op = {
       add: '+',
@@ -171,6 +196,23 @@ module RedAmber
       [Vector.new(Array(other) * size), self]
     end
 
+    # < Not implimented yet > ---
+
+    # option(s) required
+    # - index
+
+    # Returns other than value
+    # - mode
+    # - tdigest
+
+    # Functions with numerical range check (unary)
+    # - abs_checked acos_checked asin_checked cos_checked ln_checked
+    #   log10_checked log1p_checked log2_checked sin_checked tan_checked
+
+    # Functions with numerical range check (binary)
+    # - add_checked divide_checked logb_checked multiply_checked power_checked subtract_checked
+    #   shift_left_checked shift_right_checked
+
     # (array functions)
     # dictionary_encode,
     # partition_nth_indices,

data/lib/red_amber/vector_selectable.rb

@@ -64,6 +64,8 @@ module RedAmber
         return filter_by_array(arg)
       when Arrow::Array
         array = arg
+      when Range
+        array = normalize_element(arg)
       else
         unless arg.is_a?(Numeric) || booleans?([arg])
           raise VectorArgumentError, "Argument must be numeric or boolean: #{args}"
@@ -80,16 +82,17 @@ module RedAmber
 
     #   @param values [Array, Arrow::Array, Vector]
     def is_in(*values)
-      values.flatten!
+      self_data = chunked? ? data.pack : data
+
       array =
-        case values[0]
-        when Vector
-          values[0].data
-        when Arrow::Array
-          values[0]
+        case values
+        in [Vector] | [Arrow::Array] | [Arrow::ChunkedArray]
+          values[0].to_a
+        else
+          Array(values).flatten
         end
-      array ||= data.class.new(values)
-      Vector.new(data.is_in(array))
+
+      Vector.new(self_data.is_in(array))
     end
 
     # Arrow's support required
@@ -109,7 +112,7 @@ module RedAmber
 
       index_array = Arrow::UInt64ArrayBuilder.build(normalized_indices.data) # round to integer array
 
-      datum = find(:array_take).execute([data, index_array])
+      datum = find(:take).execute([data, index_array]) # :array_take will fail with ChunkedArray
       Vector.new(datum.value)
     end
 

data/lib/red_amber/vector_updatable.rb

@@ -12,7 +12,15 @@ module RedAmber
     # @param replacer [Array, Vector, Arrow::Array] new data to replace for.
     # @return [Vector] Replaced new Vector
     def replace(args, replacer)
-      args = args.is_a?(Array) ? args : Array(args)
+      args =
+        case args
+        when Array
+          args
+        when Range
+          normalize_element(args)
+        else
+          Array(args)
+        end
       replacer = Array(replacer)
       return self if args.empty? || args[0].nil?
 
@@ -22,6 +30,7 @@ module RedAmber
         if vector.boolean?
           vector
         elsif vector.numeric?
+          replacer.sort_by! { |x| args[replacer.index(x)] } if replacer # rubocop:disable Style/SafeNavigation
           Vector.new(indices).is_in(vector)
         else
           raise VectorArgumentError, "Invalid data type #{args}"
@@ -50,6 +59,18 @@ module RedAmber
       is_nil.if_else(false, self).invert
     end
 
+    def shift(amount = 1, fill: nil)
+      raise VectorArgumentError, 'Shift amount is too large' if amount.abs > size
+
+      if amount.positive?
+        replace(amount..-1, self[0...-amount]).replace(0...amount, fill)
+      elsif amount.negative?
+        replace(0...amount, self[-amount..]).replace(amount..-1, fill)
+      else # amount == 0
+        self
+      end
+    end
+
     private
 
     # [Ternary]: replace_with(booleans, replacements) => vector

data/lib/red_amber/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RedAmber
-  VERSION = '0.1.7'
+  VERSION = '0.2.1'
 end

data/lib/red_amber.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
 require 'arrow'
-require 'rover-df'
 
 require_relative 'red_amber/helper'
 require_relative 'red_amber/data_frame_displayable'
 require_relative 'red_amber/data_frame_indexable'
+require_relative 'red_amber/data_frame_reshaping'
 require_relative 'red_amber/data_frame_selectable'
 require_relative 'red_amber/data_frame_variable_operation'
 require_relative 'red_amber/data_frame'

data/red_amber.gemspec

@@ -30,7 +30,7 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'red-arrow', '>= 8.0.0'
+  spec.add_dependency 'red-arrow', '>= 9.0.0'
 
   # Development dependency has gone to the Gemfile (rubygems/bundler#7237)
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red_amber
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.2.1
 platform: ruby
 authors:
 - Hirokazu SUZUKI (heronshoes)
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-07-15 00:00:00.000000000 Z
+date: 2022-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: red-arrow
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 8.0.0
+        version: 9.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 8.0.0
+        version: 9.0.0
 description: RedAmber is a simple dataframe library inspired by Rover-df and powered
   by Red Arrow.
 email:
@@ -34,6 +34,7 @@ extra_rdoc_files: []
 files:
 - ".rubocop.yml"
 - ".rubocop_todo.yml"
+- ".yardopts"
 - CHANGELOG.md
 - Gemfile
 - LICENSE
@@ -41,10 +42,10 @@ files:
 - Rakefile
 - benchmark/csv_load_penguins.yml
 - benchmark/drop_nil.yml
-- doc/47_examples_of_red_amber.ipynb
 - doc/CODE_OF_CONDUCT.md
 - doc/DataFrame.md
 - doc/Vector.md
+- doc/examples_of_red_amber.ipynb
 - doc/image/arrow_table_new.png
 - doc/image/dataframe/assign.png
 - doc/image/dataframe/drop.png
@@ -68,6 +69,7 @@ files:
 - lib/red_amber/data_frame.rb
 - lib/red_amber/data_frame_displayable.rb
 - lib/red_amber/data_frame_indexable.rb
+- lib/red_amber/data_frame_reshaping.rb
 - lib/red_amber/data_frame_selectable.rb
 - lib/red_amber/data_frame_variable_operation.rb
 - lib/red_amber/group.rb