red_amber 0.1.5 → 0.1.8

data/lib/red_amber/vector.rb

@@ -1,25 +1,37 @@
 # frozen_string_literal: true
 
 module RedAmber
-  # Columnar data object
+  # Values in variable (columnar) data object
   #   @data : holds Arrow::ChunkedArray
   class Vector
     # mix-in
-    include VectorCompensable
     include VectorFunctions
+    include VectorUpdatable
+    include VectorSelectable
+    include Helper
 
-    # chunked_array may come from column.data
-    def initialize(array)
+    def initialize(*array)
       @key = nil # default is 'headless'
-      case array
-      when Vector
-        @data = array.data
-      when Arrow::Array, Arrow::ChunkedArray
-        @data = array
-      when Array
-        @data = Arrow::Array.new(array)
+      if array.empty? || array[0].nil?
+        Vector.new([])
       else
-        raise VectorArgumentError, 'Unknown array in argument'
+        array.flatten!
+        case array[0]
+        when Vector
+          @data = array[0].data
+          return
+        when Arrow::Array, Arrow::ChunkedArray
+          @data = array[0]
+          return
+        when Range
+          @data = Arrow::Array.new(Array(array[0]))
+          return
+        end
+        begin
+          @data = Arrow::Array.new(Array(array))
+        rescue Error
+          raise VectorArgumentError, "Invalid argument: #{array}"
+        end
       end
     end
 
@@ -52,6 +64,16 @@ module RedAmber
     alias_method :to_a, :values
     alias_method :entries, :values
 
+    def indices
+      (0...size).to_a
+    end
+    alias_method :indexes, :indices
+    alias_method :indeces, :indices
+
+    def to_ary
+      to_a
+    end
+
     def size
       # only defined :length in Arrow?
       @data.length
@@ -60,20 +82,32 @@ module RedAmber
     alias_method :n_rows, :size
     alias_method :nrow, :size
 
+    def empty?
+      size.zero?
+    end
+
     def type
       @data.value_type.nick.to_sym
     end
 
     def boolean?
-      type == :boolean
+      type_class == Arrow::BooleanDataType
     end
 
     def numeric?
       type_class < Arrow::NumericDataType
     end
 
+    def float?
+      type_class < Arrow::FloatingPointDataType
+    end
+
+    def integer?
+      type_class < Arrow::IntegerDataType
+    end
+
     def string?
-      type == :string
+      type_class == Arrow::StringDataType
     end
 
     def temporal?
@@ -84,7 +118,13 @@ module RedAmber
       @data.value_data_type.class
     end
 
-    # def each() end
+    def each
+      return enum_for(:each) unless block_given?
+
+      size.times do |i|
+        yield self[i]
+      end
+    end
 
     def chunked?
       @data.is_a? Arrow::ChunkedArray
@@ -124,5 +164,9 @@ module RedAmber
     def n_nans
       numeric? ? is_nan.to_a.count(true) : 0
     end
+
+    def has_nil?
+      is_nil.any
+    end
   end
 end

data/lib/red_amber/vector_functions.rb

@@ -14,21 +14,23 @@ module RedAmber
     unary_aggregations =
       %i[all any approximate_median count count_distinct max mean min min_max product stddev sum variance]
     unary_aggregations.each do |function|
-      define_method(function) do |opts: nil|
-        datum = exec_func_unary(function, options: opts)
-        take_out_scalar(datum)
+      define_method(function) do |**options|
+        datum = exec_func_unary(function, options)
+        get_scalar(datum)
       end
     end
     alias_method :median, :approximate_median
     alias_method :count_uniq, :count_distinct
+    alias_method :all?, :all
+    alias_method :any?, :any
 
     def unbiased_variance
-      variance(opts: { ddof: 1 })
+      variance(ddof: 1)
     end
     alias_method :var, :unbiased_variance
 
     def sd
-      stddev(opts: { ddof: 1 })
+      stddev(ddof: 1)
     end
     alias_method :std, :sd
 
@@ -45,9 +47,9 @@ module RedAmber
       %i[abs array_sort_indices atan bit_wise_not ceil cos fill_null_backward fill_null_forward floor is_finite
          is_inf is_nan is_null is_valid round round_to_multiple sign sin tan trunc unique]
     unary_element_wise.each do |function|
-      define_method(function) do |opts: nil|
-        datum = exec_func_unary(function, options: opts)
-        take_out_element_wise(datum)
+      define_method(function) do |**options|
+        datum = exec_func_unary(function, options)
+        Vector.new(datum.value)
       end
     end
     alias_method :is_nil, :is_null
@@ -70,14 +72,14 @@ module RedAmber
       negate: '-@',
     }
     unary_element_wise_op.each do |function, operator|
-      define_method(function) do |opts: nil|
-        datum = exec_func_unary(function, options: opts)
-        take_out_element_wise(datum)
+      define_method(function) do |**options|
+        datum = exec_func_unary(function, options)
+        Vector.new(datum.value)
       end
 
-      define_method(operator) do |opts: nil|
-        datum = exec_func_unary(function, options: opts)
-        take_out_element_wise(datum)
+      define_method(operator) do |**options|
+        datum = exec_func_unary(function, options)
+        Vector.new(datum.value)
       end
     end
     alias_method :not, :invert
@@ -93,9 +95,9 @@ module RedAmber
     binary_element_wise =
       %i[atan2 and_not and_not_kleene bit_wise_and bit_wise_or bit_wise_xor]
     binary_element_wise.each do |function|
-      define_method(function) do |other, opts: nil|
-        datum = exec_func_binary(function, other, options: opts)
-        take_out_element_wise(datum)
+      define_method(function) do |other, **options|
+        datum = exec_func_binary(function, other, options)
+        Vector.new(datum.value)
       end
     end
 
@@ -109,9 +111,9 @@ module RedAmber
       or_org: :or,
     }
     logical_binary_element_wise.each do |method, function|
-      define_method(method) do |other, opts: nil|
-        datum = exec_func_binary(function, other, options: opts)
-        take_out_element_wise(datum)
+      define_method(method) do |other, **options|
+        datum = exec_func_binary(function, other, options)
+        Vector.new(datum.value)
       end
     end
 
@@ -142,14 +144,14 @@ module RedAmber
       not_equal: '!=',
     }
     binary_element_wise_op.each do |function, operator|
-      define_method(function) do |other, opts: nil|
-        datum = exec_func_binary(function, other, options: opts)
-        take_out_element_wise(datum)
+      define_method(function) do |other, **options|
+        datum = exec_func_binary(function, other, options)
+        Vector.new(datum.value)
       end
 
-      define_method(operator) do |other, opts: nil|
-        datum = exec_func_binary(function, other, options: opts)
-        take_out_element_wise(datum)
+      define_method(operator) do |other, **options|
+        datum = exec_func_binary(function, other, options)
+        Vector.new(datum.value)
       end
     end
     alias_method :eq, :equal
@@ -159,8 +161,17 @@ module RedAmber
     alias_method :lt, :less
     alias_method :ne, :not_equal
 
+    def coerce(other)
+      case other
+      when Vector, Array, Arrow::Array
+        raise VectorArgumentError, "Size unmatch: #{size} != #{other.length}" unless size == other.length
+
+        [Vector.new(Array(other)), self]
+      end
+      [Vector.new(Array(other) * size), self]
+    end
+
     # (array functions)
-    # array_filter, array_take
     # dictionary_encode,
     # partition_nth_indices,
     # quarter, quarters_between,
@@ -192,25 +203,27 @@ module RedAmber
     # strptime, subsecond, us_week, week, weeks_between, year, year_month_day, years_between
 
     # (onditional)
-    # case_when, cast, if_else
+    # case_when, cast,
 
     # (indices)
     # choose, index_in, index_in_meta_binary, indices_nonzero
 
     # (others)
-    # coalesce, drop_null,
-    # filter, is_in, is_in_meta_binary,
+    # coalesce,
+    # is_in_meta_binary,
     # list_element, list_flatten, list_parent_indices, list_value_length, make_struct,
     # max_element_wise, min_element_wise, random, select_k_unstable,
-    # sort_indices, struct_field, take
+    # struct_field,
 
     private # =======
 
-    def exec_func_unary(function, options: nil)
+    def exec_func_unary(function, options)
+      options = nil if options.empty?
       find(function).execute([data], options)
     end
 
-    def exec_func_binary(function, other, options: nil)
+    def exec_func_binary(function, other, options)
+      options = nil if options.empty?
       case other
       when Vector
         find(function).execute([data, other.data], options)
@@ -221,7 +234,7 @@ module RedAmber
       end
     end
 
-    def take_out_scalar(datum)
+    def get_scalar(datum)
       output = datum.value
       case output
       when Arrow::StringScalar then output.to_s
@@ -232,10 +245,6 @@ module RedAmber
       end
     end
 
-    def take_out_element_wise(datum)
-      Vector.new(datum.value)
-    end
-
     module_function # ======
 
     def find(function_name)

data/lib/red_amber/vector_selectable.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+# Available functions in Arrow are shown by `Arrow::Function.all.map(&:name)`
+# reference: https://arrow.apache.org/docs/cpp/compute.html
+
+module RedAmber
+  # mix-ins for class Vector
+  # Functions to select some data.
+  module VectorSelectable
+    def drop_nil
+      datum = find(:drop_null).execute([data])
+      Vector.new(datum.value)
+    end
+
+    # vector calculation version of selection by indices
+    # TODO: support for option {boundscheck: true}
+    def take(*indices)
+      indices.flatten!
+      return Vector.new([]) if indices.empty?
+
+      indices = indices[0] if indices.one? && !indices[0].is_a?(Numeric)
+      indices = Vector.new(indices) unless indices.is_a?(Vector)
+
+      take_by_vector(indices) # returns sub Vector
+    end
+
+    # TODO: support for option {null_selection_behavior: :drop}
+    def filter(*booleans)
+      booleans.flatten!
+      return Vector.new([]) if booleans.empty?
+
+      b = booleans[0]
+      boolean_array =
+        case b
+        when Vector
+          raise VectorTypeError, 'Argument is not a boolean.' unless b.boolean?
+
+          b.data
+        when Arrow::BooleanArray
+          b
+        else
+          raise VectorTypeError, 'Argument is not a boolean.' unless booleans?(booleans)
+
+          Arrow::BooleanArray.new(booleans)
+        end
+
+      filter_by_array(boolean_array) # returns sub Vector
+    end
+
+    #   @param indices
+    #   @param booleans
+    def [](*args)
+      args.flatten!
+      return Vector.new([]) if args.empty?
+
+      arg = args[0]
+      case arg
+      when Vector
+        return take_by_vector(arg) if arg.numeric?
+        return filter_by_array(arg.data) if arg.boolean?
+
+        raise VectorTypeError, "Argument must be numeric or boolean: #{arg}"
+      when Arrow::BooleanArray
+        return filter_by_array(arg)
+      when Arrow::Array
+        array = arg
+      when Range
+        array = normalize_element(arg)
+      else
+        unless arg.is_a?(Numeric) || booleans?([arg])
+          raise VectorArgumentError, "Argument must be numeric or boolean: #{args}"
+        end
+      end
+      array ||= Arrow::Array.new(args)
+      return filter_by_array(array) if array.is_a?(Arrow::BooleanArray)
+
+      vector = Vector.new(array)
+      return take_by_vector(vector) if vector.numeric?
+
+      raise VectorArgumentError, "Invalid argument: #{args}"
+    end
+
+    #   @param values [Array, Arrow::Array, Vector]
+    def is_in(*values)
+      values.flatten!
+      array =
+        case values[0]
+        when Vector
+          values[0].data
+        when Arrow::Array
+          values[0]
+        end
+      array ||= data.class.new(values)
+      Vector.new(data.is_in(array))
+    end
+
+    # Arrow's support required
+    def index(element)
+      to_a.index(element)
+    end
+
+    private
+
+    # Accepts indices by numeric Vector
+    def take_by_vector(indices)
+      raise VectorTypeError, "Indices must be numeric Vector: #{indices}" unless indices.numeric?
+      raise VectorArgumentError, "Index out of range: #{indices.min}" if indices.min <= -size - 1
+
+      normalized_indices = (indices < 0).if_else(indices + size, indices) # normalize index from tail
+      raise VectorArgumentError, "Index out of range: #{normalized_indices.max}" if normalized_indices.max >= size
+
+      index_array = Arrow::UInt64ArrayBuilder.build(normalized_indices.data) # round to integer array
+
+      datum = find(:array_take).execute([data, index_array])
+      Vector.new(datum.value)
+    end
+
+    # Accepts booleans by Arrow::BooleanArray
+    def filter_by_array(boolean_array)
+      raise VectorArgumentError, 'Booleans must be same size as self.' unless boolean_array.length == size
+
+      datum = find(:array_filter).execute([data, boolean_array])
+      Vector.new(datum.value)
+    end
+  end
+end

data/lib/red_amber/vector_updatable.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+# Available functions in Arrow are shown by `Arrow::Function.all.map(&:name)`
+# reference: https://arrow.apache.org/docs/cpp/compute.html
+
+module RedAmber
+  # mix-ins for class Vector
+  # Functions to make up some data (especially missing) for new data.
+  module VectorUpdatable
+    # Replace data
+    # @param arg [Array, Vector, Arrow::Array] index specifier
+    # @param replacer [Array, Vector, Arrow::Array] new data to replace for.
+    # @return [Vector] Replaced new Vector
+    def replace(args, replacer)
+      args =
+        case args
+        when Array
+          args
+        when Range
+          normalize_element(args)
+        else
+          Array(args)
+        end
+      replacer = Array(replacer)
+      return self if args.empty? || args[0].nil?
+
+      replacer = nil if replacer.empty?
+      vector = parse_to_vector(args)
+      booleans =
+        if vector.boolean?
+          vector
+        elsif vector.numeric?
+          replacer.sort_by! { |x| args[replacer.index(x)] } if replacer # rubocop:disable Style/SafeNavigation
+          Vector.new(indices).is_in(vector)
+        else
+          raise VectorArgumentError, "Invalid data type #{args}"
+        end
+      replace_with(booleans, replacer)
+    end
+
+    # (related functions)
+    # fill_null_backward, fill_null_forward
+
+    # [Ternary element-wise]: boolean_vector.func(if_true, else) => vector
+    def if_else(true_choice, false_choice)
+      true_choice = true_choice.data if true_choice.is_a? Vector
+      false_choice = false_choice.data if false_choice.is_a? Vector
+      raise VectorTypeError, 'Reciever must be a boolean' unless boolean?
+
+      datum = find(:if_else).execute([data, true_choice, false_choice])
+      Vector.new(datum.value)
+    end
+
+    # same behavior as Ruby's invert
+    # ![true, false, nil] #=> [false, true, true]
+    def primitive_invert
+      raise VectorTypeError, "Not a boolean Vector: #{self}" unless boolean?
+
+      is_nil.if_else(false, self).invert
+    end
+
+    def shift(amount = 1, fill: nil)
+      raise VectorArgumentError, 'Shift amount is too large' if amount.abs > size
+
+      if amount.positive?
+        replace(amount..-1, self[0...-amount]).replace(0...amount, fill)
+      elsif amount.negative?
+        replace(0...amount, self[-amount..]).replace(amount..-1, fill)
+      else # amount == 0
+        self
+      end
+    end
+
+    private
+
+    # [Ternary]: replace_with(booleans, replacements) => vector
+    # Replace items selected with a boolean mask
+    #
+    # (from Arrow C++ inline doc.)
+    # Given an array and a boolean mask (either scalar or of equal length),
+    # along with replacement values (either scalar or array),
+    # each element of the array for which the corresponding mask element is
+    # true will be replaced by the next value from the replacements,
+    # or with null if the mask is null.
+    # Hence, for replacement arrays, len(replacements) == sum(mask == true).
+
+    def replace_with(booleans, replacer = nil)
+      specifier =
+        if booleans.is_a?(Arrow::BooleanArray)
+          booleans
+        elsif booleans.is_a?(Vector) && booleans.boolean?
+          booleans.data
+        elsif booleans.is_a?(Array) && booleans?(booleans)
+          Arrow::BooleanArray.new(booleans)
+        else
+          raise VectorTypeError, 'Not a valid type'
+        end
+      raise VectorArgumentError, 'Booleans size unmatch' if specifier.length != size
+      raise VectorArgumentError, 'Booleans not have any `true`' unless specifier.any?
+
+      r = Array(replacer) # scalar to [scalar]
+      r = [nil] if r.empty?
+
+      replacer =
+        if r.size == 1
+          case replacer
+          when Arrow::Array then replacer
+          when Vector then replacer.data
+          else
+            Arrow::Array.new(r * specifier.to_a.count(true)) # broadcast
+          end
+        else
+          Arrow::Array.new(r)
+        end
+      replacer = data.class.new(replacer) if replacer.uniq == [nil]
+
+      raise VectorArgumentError, 'Replacements size unmatch' if Array(specifier).count(true) != replacer.length
+
+      values = replacer.class.new(data)
+
+      datum = find('replace_with_mask').execute([values, specifier, replacer])
+      Vector.new(datum.value)
+    end
+  end
+end

data/lib/red_amber/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RedAmber
-  VERSION = '0.1.5'
+  VERSION = '0.1.8'
 end

data/lib/red_amber.rb

@@ -3,15 +3,16 @@
 require 'arrow'
 require 'rover-df'
 
+require_relative 'red_amber/helper'
 require_relative 'red_amber/data_frame_displayable'
-require_relative 'red_amber/data_frame_helper'
 require_relative 'red_amber/data_frame_indexable'
 require_relative 'red_amber/data_frame_selectable'
-require_relative 'red_amber/data_frame_observation_operation'
 require_relative 'red_amber/data_frame_variable_operation'
 require_relative 'red_amber/data_frame'
-require_relative 'red_amber/vector_compensable'
+require_relative 'red_amber/group'
 require_relative 'red_amber/vector_functions'
+require_relative 'red_amber/vector_updatable'
+require_relative 'red_amber/vector_selectable'
 require_relative 'red_amber/vector'
 require_relative 'red_amber/version'
 
@@ -23,4 +24,6 @@ module RedAmber
 
   class VectorArgumentError < ArgumentError; end
   class VectorTypeError < TypeError; end
+
+  class GroupArgumentError < ArgumentError; end
 end

data/red_amber.gemspec

@@ -31,8 +31,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'red-arrow', '>= 8.0.0'
-  spec.add_dependency 'red-parquet', '>= 8.0.0'
-  spec.add_dependency 'rover-df', '~> 0.3.0'
 
   # Development dependency has gone to the Gemfile (rubygems/bundler#7237)
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red_amber
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.8
 platform: ruby
 authors:
 - Hirokazu SUZUKI (heronshoes)
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-06-12 00:00:00.000000000 Z
+date: 2022-08-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: red-arrow
@@ -24,34 +24,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 8.0.0
-- !ruby/object:Gem::Dependency
-  name: red-parquet
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 8.0.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 8.0.0
-- !ruby/object:Gem::Dependency
-  name: rover-df
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.3.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.3.0
 description: RedAmber is a simple dataframe library inspired by Rover-df and powered
   by Red Arrow.
 email:
@@ -62,6 +34,7 @@ extra_rdoc_files: []
 files:
 - ".rubocop.yml"
 - ".rubocop_todo.yml"
+- ".yardopts"
 - CHANGELOG.md
 - Gemfile
 - LICENSE
@@ -72,6 +45,7 @@ files:
 - doc/CODE_OF_CONDUCT.md
 - doc/DataFrame.md
 - doc/Vector.md
+- doc/examples_of_red_amber.ipynb
 - doc/image/arrow_table_new.png
 - doc/image/dataframe/assign.png
 - doc/image/dataframe/drop.png
@@ -90,17 +64,19 @@ files:
 - doc/image/vector/unary_element_wise.png
 - doc/tdr.md
 - doc/tdr_ja.md
+- lib/red-amber.rb
 - lib/red_amber.rb
 - lib/red_amber/data_frame.rb
 - lib/red_amber/data_frame_displayable.rb
-- lib/red_amber/data_frame_helper.rb
 - lib/red_amber/data_frame_indexable.rb
-- lib/red_amber/data_frame_observation_operation.rb
 - lib/red_amber/data_frame_selectable.rb
 - lib/red_amber/data_frame_variable_operation.rb
+- lib/red_amber/group.rb
+- lib/red_amber/helper.rb
 - lib/red_amber/vector.rb
-- lib/red_amber/vector_compensable.rb
 - lib/red_amber/vector_functions.rb
+- lib/red_amber/vector_selectable.rb
+- lib/red_amber/vector_updatable.rb
 - lib/red_amber/version.rb
 - red_amber.gemspec
 - sig/red_amber.rbs