red_amber 0.2.0 → 0.2.2

data/lib/red_amber/data_frame_selectable.rb

@@ -3,8 +3,8 @@
 module RedAmber
   # mix-in for the class DataFrame
   module DataFrameSelectable
-    # select variables: [symbol] or [string]
-    # select observations: [array of index], [range]
+    # select columns: [symbol] or [string]
+    # select rows: [array of index], [range]
     def [](*args)
       args.flatten!
       raise DataFrameArgumentError, 'Empty dataframe' if empty?
@@ -22,17 +22,17 @@ module RedAmber
       raise DataFrameArgumentError, "Invalid argument: #{args}"
     end
 
-    # slice and select some observations to create sub DataFrame
+    # slice and select rows to create sub DataFrame
     def slice(*args, &block)
       slicer = args
       if block
         raise DataFrameArgumentError, 'Must not specify both arguments and block.' unless args.empty?
 
-        slicer = instance_eval(&block)
+        slicer = [instance_eval(&block)]
       end
-      slicer = [slicer].flatten
+      slicer.flatten!
 
-      raise DataFrameArgumentError, 'Empty dataframe' if empty?
+      raise DataFrameArgumentError, 'Self is an empty dataframe' if empty?
       return remove_all_values if slicer.empty? || slicer[0].nil?
 
       vector = parse_to_vector(slicer)
@@ -46,15 +46,59 @@ module RedAmber
       raise DataFrameArgumentError, "Invalid argument #{slicer}"
     end
 
-    # remove selected observations to create sub DataFrame
+    def slice_by(key, keep_key: false, &block)
+      raise DataFrameArgumentError, 'Self is an empty dataframe' if empty?
+      raise DataFrameArgumentError, 'No block given' unless block
+      raise DataFrameArgumentError, "#{key} is no a key of self" unless key?(key)
+      return self if key.nil?
+
+      slicer = instance_eval(&block)
+      return DataFrame.new unless slicer
+
+      if slicer.is_a?(Range)
+        from = slicer.begin
+        from =
+          if from.is_a?(String)
+            self[key].index(from)
+          elsif from.nil?
+            0
+          elsif from < 0
+            size + from
+          else
+            from
+          end
+        to = slicer.end
+        to =
+          if to.is_a?(String)
+            self[key].index(to)
+          elsif to.nil?
+            size - 1
+          elsif to < 0
+            size + to
+          else
+            to
+          end
+        slicer = (from..to).to_a
+      else
+        slicer = slicer.map { |x| x.is_a?(String) ? self[key].index(x) : x }
+      end
+
+      if keep_key
+        take(slicer)
+      else
+        take(slicer).drop(key)
+      end
+    end
+
+    # remove selected rows to create remainer DataFrame
     def remove(*args, &block)
       remover = args
       if block
         raise DataFrameArgumentError, 'Must not specify both arguments and block.' unless args.empty?
 
-        remover = instance_eval(&block)
+        remover = [instance_eval(&block)]
       end
-      remover = [remover].flatten
+      remover.flatten!
 
       raise DataFrameArgumentError, 'Empty dataframe' if empty?
       return self if remover.empty? || remover[0].nil?

data/lib/red_amber/data_frame_variable_operation.rb

@@ -9,18 +9,28 @@ module RedAmber
       if block
         raise DataFrameArgumentError, 'Must not specify both arguments and block.' unless args.empty?
 
-        picker = instance_eval(&block)
+        picker = [instance_eval(&block)]
       end
-      picker = [picker].flatten
+      picker.flatten!
       return DataFrame.new if picker.empty? || picker == [nil]
 
-      picker = keys_by_booleans(picker) if booleans?(picker)
+      key_vector = Vector.new(keys)
+      vec = parse_to_vector(picker, vsize: n_keys)
+
+      ary =
+        if vec.boolean?
+          key_vector.filter(*vec).to_a
+        elsif vec.numeric?
+          key_vector.take(*vec).to_a
+        elsif vec.string? || vec.dictionary?
+          picker
+        else
+          raise DataFrameArgumentError, "Invalid argument #{args}"
+        end
 
       # DataFrame#[] creates a Vector with single key is specified.
       # DataFrame#pick creates a DataFrame with single key.
-      return DataFrame.new(@table[picker]) if sym_or_str?(picker)
-
-      raise DataFrameArgumentError, "Invalid argument #{args}"
+      DataFrame.new(@table[ary])
     end
 
     # drop some variables to create remainer sub DataFrame
@@ -29,19 +39,29 @@ module RedAmber
       if block
         raise DataFrameArgumentError, 'Must not specify both arguments and block.' unless args.empty?
 
-        dropper = instance_eval(&block)
+        dropper = [instance_eval(&block)]
       end
-      dropper = [dropper].flatten
-      dropper = keys_by_booleans(dropper) if booleans?(dropper)
+      dropper.flatten!
+
+      key_vector = Vector.new(keys)
+      vec = parse_to_vector(dropper, vsize: n_keys)
+
+      ary =
+        if vec.boolean?
+          key_vector.filter(*vec.primitive_invert).each.map(&:to_sym) # Array
+        elsif vec.numeric?
+          keys - key_vector.take(*vec).each.map(&:to_sym) # Array
+        elsif vec.string? || vec.dictionary?
+          keys - dropper
+        else
+          raise DataFrameArgumentError, "Invalid argument #{args}"
+        end
 
-      picker = keys - dropper
-      return DataFrame.new if picker.empty?
+      return DataFrame.new if ary.empty?
 
       # DataFrame#[] creates a Vector with single key is specified.
       # DataFrame#drop creates a DataFrame with single key.
-      return DataFrame.new(@table[picker]) if sym_or_str?(picker)
-
-      raise DataFrameArgumentError, "Invalid argument #{args}"
+      DataFrame.new(@table[ary])
     end
 
     # rename variables to create a new DataFrame
@@ -91,10 +111,20 @@ module RedAmber
 
     def assign_update(*assigner, &block)
       if block
-        raise DataFrameArgumentError, 'Must not specify both arguments and a block' unless assigner.empty?
-
-        assigner = [instance_eval(&block)]
+        assigner_from_block = instance_eval(&block)
+        assigner =
+          if assigner.empty?
+            # block only
+            [assigner_from_block]
+          # If Ruby >= 3.0, one line pattern match can be used
+          # assigner_from_block in [Array, *]
+          elsif multiple_assigner?(assigner_from_block)
+            assigner.zip(assigner_from_block)
+          else
+            assigner.zip([assigner_from_block])
+          end
       end
+
       case assigner
       in [] | [nil] | [{}] | [[]]
         return self
@@ -113,6 +143,8 @@ module RedAmber
       updater = {}
       appender = {}
       key_array_pairs.each do |key, array|
+        raise DataFrameArgumentError, "Empty column data: #{key} => nil" if array.nil?
+
         if keys.include? key
           updater[key] = array
         else
@@ -153,7 +185,7 @@ module RedAmber
         data = updater[key]
         next unless data
 
-        raise DataFrameArgumentError, "Data size mismatch (#{data.size} != #{size})" if data.size != size
+        raise DataFrameArgumentError, "Data size mismatch (#{data.size} != #{size})" if data.nil? || data.size != size
 
         a = Arrow::Array.new(data.is_a?(Vector) ? data.to_a : data)
         fields[i] = Arrow::Field.new(key, a.value_data_type)
@@ -179,8 +211,13 @@ module RedAmber
       end
     end
 
-    def keys_by_booleans(booleans)
-      keys.select.with_index { |_, i| booleans[i] }
+    def multiple_assigner?(assigner)
+      case assigner
+      in [Vector, *] | [Array, *] | [Arrow::Array, *]
+        true
+      else
+        false
+      end
     end
   end
 end

data/lib/red_amber/group.rb

@@ -54,9 +54,11 @@ module RedAmber
       raise GroupArgumentError, "#{d} is not a key of\n #{@dataframe}." unless summary_keys.empty? || d.empty?
 
       df = RedAmber::DataFrame.new(@group.send(func, *summary_keys))
-      df = df[@group_keys, df.keys - @group_keys]
-      # if counts are the same (no nil included), aggregate count columns.
-      df = df[df.keys[0..1]].rename(df.keys[1], :count) if func == :count && df.to_h.values[1..].uniq.size == 1
+      df = df.pick(@group_keys, df.keys - @group_keys)
+      # if counts are the same (and do not include NaN or nil), aggregate count columns.
+      if func == :count && df.pick(@group_keys.size..).to_h.values.uniq.size == 1
+        df = df.pick(0..@group_keys.size).rename { [keys[-1], :count] }
+      end
       df
     end
   end

data/lib/red_amber/helper.rb

@@ -17,10 +17,6 @@ module RedAmber
       enum.all?(Integer)
     end
 
-    def sym_or_str?(enum)
-      enum.all? { |e| e.is_a?(Symbol) || e.is_a?(String) }
-    end
-
     def booleans?(enum)
       enum.all? { |e| e.is_a?(TrueClass) || e.is_a?(FalseClass) || e.is_a?(NilClass) }
     end
@@ -29,32 +25,38 @@ module RedAmber
       DataFrame.new(key => vector.data)
     end
 
-    def parse_to_vector(args)
+    def parse_to_vector(args, vsize: size)
       a = args.reduce([]) do |accum, elem|
-        accum.concat(normalize_element(elem))
+        accum.concat(normalize_element(elem, vsize: vsize))
       end
       Vector.new(a)
     end
 
-    def normalize_element(elem)
+    def normalize_element(elem, vsize: size)
       case elem
-      when Numeric, String, Symbol, TrueClass, FalseClass, NilClass
-        [elem]
+      when NilClass
+        [nil]
       when Range
-        both_end = [elem.begin, elem.end]
-        both_end[1] -= 1 if elem.exclude_end? && elem.end.is_a?(Integer)
-
-        if both_end.any?(Integer) || both_end.all?(&:nil?)
-          if both_end.any? { |e| e&.>=(size) || e&.<(-size) }
-            raise DataFrameArgumentError, "Index out of range: #{elem} for 0..#{size - 1}"
+        bg = elem.begin
+        en = elem.end
+        if [bg, en].any?(Integer)
+          bg += vsize if bg&.negative?
+          en += vsize if en&.negative?
+          en -= 1 if en.is_a?(Integer) && elem.exclude_end?
+          if bg&.negative? || (en && en >= vsize)
+            raise DataFrameArgumentError, "Index out of range: #{elem} for 0..#{vsize - 1}"
           end
 
-          (0...size).to_a[elem]
+          Array(0...vsize)[elem]
+        elsif bg.nil? && en.nil?
+          Array(0...vsize)
         else
-          elem.to_a
+          Array[elem]
         end
+      when Enumerator
+        elem.to_a
       else
-        Array(elem)
+        Array[elem]
       end
     end
   end

data/lib/red_amber/vector.rb

@@ -11,27 +11,28 @@ module RedAmber
     include Helper
 
     def initialize(*array)
-      @key = nil # default is 'headless'
-      if array.empty? || array[0].nil?
+      @key = nil # default is 'headless' Vector
+      if array.empty? || array.first.nil?
         Vector.new([])
       else
         array.flatten!
-        case array[0]
-        when Vector
-          @data = array[0].data
-          return
-        when Arrow::Array, Arrow::ChunkedArray
-          @data = array[0]
-          return
-        when Range
-          @data = Arrow::Array.new(Array(array[0]))
-          return
-        end
-        begin
-          @data = Arrow::Array.new(Array(array))
-        rescue Error
-          raise VectorArgumentError, "Invalid argument: #{array}"
-        end
+        @data =
+          case array
+          in [Vector => v]
+            v.data
+          in [Arrow::Array => a]
+            a
+          in [Arrow::ChunkedArray => ca]
+            ca
+          in [Range => r]
+            Arrow::Array.new(Array(r))
+          else
+            begin
+              Arrow::Array.new(Array(array))
+            rescue Error
+              raise VectorArgumentError, "Invalid argument: #{array}"
+            end
+          end
       end
     end
 
@@ -43,19 +44,24 @@ module RedAmber
     end
 
     def inspect(limit: 80)
-      sio = StringIO.new << '['
-      to_a.each_with_object(sio).with_index do |(e, s), i|
-        next_str = "#{s.size > 1 ? ', ' : ''}#{e.inspect}"
-        if (s.size + next_str.size) < limit
-          s << next_str
-        else
-          s << ', ... ' if i < size
-          break
+      if ENV.fetch('RED_AMBER_OUTPUT_MODE', 'Table').casecmp('MINIMUM').zero?
+        # Better performance than `.upcase == 'MINIMUM'``
+        "#{self.class}(:#{type}, size=#{size})"
+      else
+        sio = StringIO.new << '['
+        to_a.each_with_object(sio).with_index do |(e, s), i|
+          next_str = "#{s.size > 1 ? ', ' : ''}#{e.inspect}"
+          if (s.size + next_str.size) < limit
+            s << next_str
+          else
+            s << ', ... ' if i < size
+            break
+          end
         end
-      end
-      sio << ']'
+        sio << ']'
 
-      format "#<#{self.class}(:#{type}, size=#{size}):0x%016x>\n%s\n", object_id, sio.string
+        format "#<#{self.class}(:#{type}, size=#{size}):0x%016x>\n%s\n", object_id, sio.string
+      end
     end
 
     def values
@@ -71,7 +77,7 @@ module RedAmber
     alias_method :indeces, :indices
 
     def to_ary
-      to_a
+      values
     end
 
     def size
@@ -110,6 +116,10 @@ module RedAmber
       type_class == Arrow::StringDataType
     end
 
+    def dictionary?
+      type_class == Arrow::DictionaryDataType
+    end
+
     def temporal?
       type_class < Arrow::TemporalDataType
     end
@@ -122,14 +132,23 @@ module RedAmber
       return enum_for(:each) unless block_given?
 
       size.times do |i|
-        yield self[i]
+        yield data[i]
       end
     end
 
+    def map(&block)
+      return enum_for(:map) unless block
+
+      Vector.new(to_a.map(&block))
+    end
+    alias_method :collect, :map
+
+    # undocumented
     def chunked?
       @data.is_a? Arrow::ChunkedArray
     end
 
+    # undocumented
     def n_chunks
       chunked? ? @data.n_chunks : 0
     end

data/lib/red_amber/vector_functions.rb

@@ -34,13 +34,6 @@ module RedAmber
     end
     alias_method :std, :sd
 
-    # option(s) required
-    # - index
-
-    # Returns other than value
-    # - mode
-    # - tdigest
-
     # Return quantile
     #   0.5 quantile (median) is returned by default.
     #   Or return quantile for specified probability (prob).
@@ -88,8 +81,9 @@ module RedAmber
 
     # [Unary element-wise]: vector.func => vector
     unary_element_wise =
-      %i[abs array_sort_indices atan bit_wise_not ceil cos fill_null_backward fill_null_forward floor is_finite
-         is_inf is_nan is_null is_valid round round_to_multiple sign sin tan trunc unique]
+      %i[abs acos asin array_sort_indices atan bit_wise_not ceil cos fill_null_backward \
+         fill_null_forward floor is_finite is_inf is_nan is_null is_valid ln log10 log1p log2 \
+         round round_to_multiple sign sin tan trunc unique]
     unary_element_wise.each do |function|
       define_method(function) do |**options|
         datum = exec_func_unary(function, options)
@@ -129,16 +123,9 @@ module RedAmber
     end
     alias_method :not, :invert
 
-    # NaN support needed
-    # - acos asin ln log10 log1p log2
-
-    # Functions with numerical range check
-    # - abs_checked acos_checked asin_checked cos_checked ln_checked
-    #   log10_checked log1p_checked log2_checked sin_checked tan_checked
-
     # [Binary element-wise]: vector.func(other) => vector
     binary_element_wise =
-      %i[atan2 and_not and_not_kleene bit_wise_and bit_wise_or bit_wise_xor]
+      %i[atan2 and_not and_not_kleene bit_wise_and bit_wise_or bit_wise_xor logb]
     binary_element_wise.each do |function|
       define_method(function) do |other, **options|
         datum = exec_func_binary(function, other, options)
@@ -162,13 +149,6 @@ module RedAmber
       end
     end
 
-    # NaN support needed
-    # - logb
-
-    # Functions with numerical range check
-    # - add_checked divide_checked logb_checked multiply_checked power_checked subtract_checked
-    #   shift_left_checked shift_right_checked
-
     # [Binary element-wise with operator]: vector.func(other) => vector
     binary_element_wise_op = {
       add: '+',
@@ -216,6 +196,23 @@ module RedAmber
       [Vector.new(Array(other) * size), self]
     end
 
+    # < Not implimented yet > ---
+
+    # option(s) required
+    # - index
+
+    # Returns other than value
+    # - mode
+    # - tdigest
+
+    # Functions with numerical range check (unary)
+    # - abs_checked acos_checked asin_checked cos_checked ln_checked
+    #   log10_checked log1p_checked log2_checked sin_checked tan_checked
+
+    # Functions with numerical range check (binary)
+    # - add_checked divide_checked logb_checked multiply_checked power_checked subtract_checked
+    #   shift_left_checked shift_right_checked
+
     # (array functions)
     # dictionary_encode,
     # partition_nth_indices,

data/lib/red_amber/vector_selectable.rb

@@ -25,7 +25,13 @@ module RedAmber
     end
 
     # TODO: support for option {null_selection_behavior: :drop}
-    def filter(*booleans)
+    def filter(*booleans, &block)
+      if block
+        raise VectorArgumentError, 'Must not specify both arguments and block.' unless booleans.empty?
+
+        booleans = [yield]
+      end
+
       booleans.flatten!
       return Vector.new([]) if booleans.empty?
 
@@ -46,6 +52,8 @@ module RedAmber
 
       filter_by_array(boolean_array) # returns sub Vector
     end
+    alias_method :select, :filter
+    alias_method :find_all, :filter
 
     #   @param indices
     #   @param booleans
@@ -82,16 +90,17 @@ module RedAmber
 
     #   @param values [Array, Arrow::Array, Vector]
     def is_in(*values)
-      values.flatten!
+      self_data = chunked? ? data.pack : data
+
       array =
-        case values[0]
-        when Vector
-          values[0].data
-        when Arrow::Array
-          values[0]
+        case values
+        in [Vector] | [Arrow::Array] | [Arrow::ChunkedArray]
+          values[0].to_a
+        else
+          Array(values).flatten
         end
-      array ||= data.class.new(values)
-      Vector.new(data.is_in(array))
+
+      Vector.new(self_data.is_in(array))
     end
 
     # Arrow's support required

data/lib/red_amber/vector_updatable.rb

@@ -8,9 +8,10 @@ module RedAmber
   # Functions to make up some data (especially missing) for new data.
   module VectorUpdatable
     # Replace data
-    # @param arg [Array, Vector, Arrow::Array] index specifier
+    # @param arg [Array, Vector, Arrow::Array] index specifier or boolean
     # @param replacer [Array, Vector, Arrow::Array] new data to replace for.
-    # @return [Vector] Replaced new Vector
+    # @return [Vector] Replaced new Vector.
+    #   If arg has no true, return self.
     def replace(args, replacer)
       args =
         case args
@@ -24,10 +25,12 @@ module RedAmber
       replacer = Array(replacer)
       return self if args.empty? || args[0].nil?
 
-      replacer = nil if replacer.empty?
       vector = parse_to_vector(args)
+      replacer = nil if replacer.empty?
       booleans =
         if vector.boolean?
+          return self unless vector.any
+
           vector
         elsif vector.numeric?
           replacer.sort_by! { |x| args[replacer.index(x)] } if replacer # rubocop:disable Style/SafeNavigation

data/lib/red_amber/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RedAmber
-  VERSION = '0.2.0'
+  VERSION = '0.2.2'
 end

data/lib/red_amber.rb

@@ -5,6 +5,7 @@ require 'arrow'
 require_relative 'red_amber/helper'
 require_relative 'red_amber/data_frame_displayable'
 require_relative 'red_amber/data_frame_indexable'
+require_relative 'red_amber/data_frame_loadsave'
 require_relative 'red_amber/data_frame_reshaping'
 require_relative 'red_amber/data_frame_selectable'
 require_relative 'red_amber/data_frame_variable_operation'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red_amber
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.2
 platform: ruby
 authors:
 - Hirokazu SUZUKI (heronshoes)
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-08-15 00:00:00.000000000 Z
+date: 2022-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: red-arrow
@@ -45,16 +45,25 @@ files:
 - doc/CODE_OF_CONDUCT.md
 - doc/DataFrame.md
 - doc/Vector.md
-- doc/examples_of_red_amber.ipynb
 - doc/image/arrow_table_new.png
+- doc/image/basic_verbs.png
 - doc/image/dataframe/assign.png
+- doc/image/dataframe/assign_operation.png
 - doc/image/dataframe/drop.png
 - doc/image/dataframe/pick.png
+- doc/image/dataframe/pick_operation.png
 - doc/image/dataframe/remove.png
 - doc/image/dataframe/rename.png
+- doc/image/dataframe/rename_operation.png
+- doc/image/dataframe/reshaping_DataFrames.png
 - doc/image/dataframe/slice.png
+- doc/image/dataframe/slice_operation.png
 - doc/image/dataframe_model.png
 - doc/image/example_in_red_arrow.png
+- doc/image/group_operation.png
+- doc/image/replace-if_then.png
+- doc/image/reshaping_dataframe.png
+- doc/image/screenshot.png
 - doc/image/tdr.png
 - doc/image/tdr_and_table.png
 - doc/image/tidy_data_in_TDR.png
@@ -69,6 +78,7 @@ files:
 - lib/red_amber/data_frame.rb
 - lib/red_amber/data_frame_displayable.rb
 - lib/red_amber/data_frame_indexable.rb
+- lib/red_amber/data_frame_loadsave.rb
 - lib/red_amber/data_frame_reshaping.rb
 - lib/red_amber/data_frame_selectable.rb
 - lib/red_amber/data_frame_variable_operation.rb