red_amber 0.1.2 → 0.1.5

data/lib/red_amber/vector_compensable.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# Available functions in Arrow are shown by `Arrow::Function.all.map(&:name)`
+# reference: https://arrow.apache.org/docs/cpp/compute.html
+
+module RedAmber
+  # mix-ins for class Vector
+  # Functions to make up some data (especially missing) for new data.
+  module VectorCompensable
+    # [Ternary]: replace_with(booleans, replacements) => vector
+    # Replace items selected with a boolean mask
+    #
+    # (from Arrow C++ inline doc.)
+    # Given an array and a boolean mask (either scalar or of equal length),
+    # along with replacement values (either scalar or array),
+    # each element of the array for which the corresponding mask element is
+    # true will be replaced by the next value from the replacements,
+    # or with null if the mask is null.
+    # Hence, for replacement arrays, len(replacements) == sum(mask == true).
+
+    def replace_with(booleans, replacements = nil)
+      specifier =
+        if booleans.is_a?(Arrow::BooleanArray)
+          booleans
+        elsif booleans.is_a?(Vector) && booleans.boolean?
+          booleans.data
+        elsif booleans.is_a?(Array) && booleans?(booleans)
+          Arrow::BooleanArray.new(booleans)
+        else
+          raise VectorTypeError, 'Not a valid type'
+        end
+      raise VectorArgumentError, 'Booleans size unmatch' if specifier.length != size
+      raise VectorArgumentError, 'Booleans not have any `true`' unless specifier.any?
+
+      r = Array(replacements) # scalar to [scalar]
+      r = [nil] if r.empty?
+
+      replacer =
+        if r.size == 1
+          case replacements
+          when Arrow::Array then replacements
+          when Vector then replacements.data
+          else
+            Arrow::Array.new(r * specifier.to_a.count(true)) # broadcast
+          end
+        else
+          Arrow::Array.new(r)
+        end
+      replacer = data.class.new(replacer) if replacer.uniq == [nil]
+
+      raise VectorArgumentError, 'Replacements size unmatch' if Array(specifier).count(true) != replacer.length
+
+      values = replacer.class.new(data)
+
+      datum = find('replace_with_mask').execute([values, specifier, replacer])
+      take_out_element_wise(datum)
+    end
+
+    # (related functions)
+    # fill_null_backward, fill_null_forward
+
+    private
+
+    def booleans?(enum)
+      enum.all? { |e| e.is_a?(TrueClass) || e.is_a?(FalseClass) || e.is_a?(NilClass) }
+    end
+  end
+end

data/lib/red_amber/vector_functions.rb

@@ -1,69 +1,128 @@
 # frozen_string_literal: true
 
+# Available functions in Arrow are shown by `Arrow::Function.all.map(&:name)`
+# reference: https://arrow.apache.org/docs/cpp/compute.html
+
+# Not implemented in Red Arrow 8.0.0
+# divmod,  # '%',
+# true_unless_null
+
 module RedAmber
   # mix-ins for class Vector
   module VectorFunctions
-    # Available functions in Arrow are shown by `Arrow::Function.all.map(&:name)`
-    # reference: https://arrow.apache.org/docs/cpp/compute.html
-
-    # [Unary aggregations]: vector.func => Scalar
+    # [Unary aggregations]: vector.func => scalar
     unary_aggregations =
-      %i[all any approximate_median count count_distinct max mean min \
-         product stddev sum variance]
+      %i[all any approximate_median count count_distinct max mean min min_max product stddev sum variance]
     unary_aggregations.each do |function|
-      define_method(function) { exec_func(function, other: nil, options: { aggregate: true }) }
+      define_method(function) do |opts: nil|
+        datum = exec_func_unary(function, options: opts)
+        take_out_scalar(datum)
+      end
     end
+    alias_method :median, :approximate_median
     alias_method :count_uniq, :count_distinct
 
+    def unbiased_variance
+      variance(opts: { ddof: 1 })
+    end
+    alias_method :var, :unbiased_variance
+
+    def sd
+      stddev(opts: { ddof: 1 })
+    end
+    alias_method :std, :sd
+
     # option(s) required
-    # index
+    # - index
 
     # Returns other than value
-    # min_max
-    # mode
-    # quantile
-    # tdigest
-
-    # [Unary element-wise]: vector.func => Vector
-    unary_element_wise = %i[abs atan ceil cos floor sign sin tan trunc]
+    # - mode
+    # - quantile
+    # - tdigest
+
+    # [Unary element-wise]: vector.func => vector
+    unary_element_wise =
+      %i[abs array_sort_indices atan bit_wise_not ceil cos fill_null_backward fill_null_forward floor is_finite
+         is_inf is_nan is_null is_valid round round_to_multiple sign sin tan trunc unique]
     unary_element_wise.each do |function|
-      define_method(function) { exec_func(function, other: nil, options: {}) }
+      define_method(function) do |opts: nil|
+        datum = exec_func_unary(function, options: opts)
+        take_out_element_wise(datum)
+      end
     end
+    alias_method :is_nil, :is_null
 
-    # [Unary element-wise with operator]: vector.func => Vector
+    def is_na
+      numeric? ? (is_nil | is_nan) : is_nil
+    end
+
+    alias_method :fill_nil_backward, :fill_null_backward
+    alias_method :fill_nil_forward, :fill_null_forward
+
+    alias_method :sort_indexes, :array_sort_indices
+    alias_method :sort_indices, :array_sort_indices
+
+    alias_method :uniq, :unique
+
+    # [Unary element-wise with operator]: vector.func => vector, op vector
     unary_element_wise_op = {
+      invert: '!',
       negate: '-@',
     }
     unary_element_wise_op.each do |function, operator|
-      define_method(function) { exec_func(function, other: nil, options: {}) }
-      define_method(operator) { exec_func(function, other: nil, options: {}) }
-    end
+      define_method(function) do |opts: nil|
+        datum = exec_func_unary(function, options: opts)
+        take_out_element_wise(datum)
+      end
 
-    # bit_wise_not => '!', invert, round, round_to_multiple
+      define_method(operator) do |opts: nil|
+        datum = exec_func_unary(function, options: opts)
+        take_out_element_wise(datum)
+      end
+    end
+    alias_method :not, :invert
 
     # NaN support needed
-    # %i[acos asin ln log10 log1p log2]
+    # - acos asin ln log10 log1p log2
 
-    # With numerical range check
-    # %i[abs_checked acos_checked asin_checked cos_checked ln_checked \
-    #    log10_checked log1p_checked log2_checked sin_checked tan_checked]
+    # Functions with numerical range check
+    # - abs_checked acos_checked asin_checked cos_checked ln_checked
+    #   log10_checked log1p_checked log2_checked sin_checked tan_checked
 
-    # [Binary element-wise]: vector.func(other) => Vector
-    binary_element_wise = %i[atan2 and and_kleene and_not and_not_kleene or or_kleene xor]
+    # [Binary element-wise]: vector.func(other) => vector
+    binary_element_wise =
+      %i[atan2 and_not and_not_kleene bit_wise_and bit_wise_or bit_wise_xor]
     binary_element_wise.each do |function|
-      define_method(function) do |other|
-        exec_func(function, other: other, options: {})
+      define_method(function) do |other, opts: nil|
+        datum = exec_func_binary(function, other, options: opts)
+        take_out_element_wise(datum)
+      end
+    end
+
+    # [Logical binary element-wise]: vector.func(other) => vector
+    logical_binary_element_wise = {
+      '&': :and_kleene,
+      and_kleene: :and_kleene,
+      and_org: :and,
+      '|': :or_kleene,
+      or_kleene: :or_kleene,
+      or_org: :or,
+    }
+    logical_binary_element_wise.each do |method, function|
+      define_method(method) do |other, opts: nil|
+        datum = exec_func_binary(function, other, options: opts)
+        take_out_element_wise(datum)
       end
     end
 
     # NaN support needed
-    # logb
+    # - logb
 
-    # With numerical range check
-    # %i[add_checked divide_checked logb_checked multiply_checked power_checked subtract_checked \
-    #    shift_left_checked shift_right_checked]
+    # Functions with numerical range check
+    # - add_checked divide_checked logb_checked multiply_checked power_checked subtract_checked
+    #   shift_left_checked shift_right_checked
 
-    # [Binary element-wise with operator]: vector.func(other) => Vector
+    # [Binary element-wise with operator]: vector.func(other) => vector
     binary_element_wise_op = {
       add: '+',
       divide: '/',
@@ -71,9 +130,7 @@ module RedAmber
       power: '**',
       subtract: '-',
 
-      bit_wise_and: '&',
-      bit_wise_or: '|',
-      bit_wise_xor: '^',
+      xor: '^',
       shift_left: '<<',
       shift_right: '>>',
 
@@ -85,11 +142,14 @@ module RedAmber
       not_equal: '!=',
     }
     binary_element_wise_op.each do |function, operator|
-      define_method(function) do |other|
-        exec_func(function, other: other, options: {})
+      define_method(function) do |other, opts: nil|
+        datum = exec_func_binary(function, other, options: opts)
+        take_out_element_wise(datum)
       end
-      define_method(operator) do |other|
-        exec_func(function, other: other, options: {})
+
+      define_method(operator) do |other, opts: nil|
+        datum = exec_func_binary(function, other, options: opts)
+        take_out_element_wise(datum)
       end
     end
     alias_method :eq, :equal
@@ -99,16 +159,11 @@ module RedAmber
     alias_method :lt, :less
     alias_method :ne, :not_equal
 
-    # mod: '%',
-
     # (array functions)
-    # array_filter, array_sort_indices, array_take
-    # dictionary_encode, hash_all, hash_any, hash_approximate_median,
-    # hash_count, hash_count_distinct, hash_distinct, hash_max, hash_mean, hash_min,
-    # hash_min_max, hash_product, hash_stddev, hash_sum, hash_tdigest, hash_variance,
+    # array_filter, array_take
+    # dictionary_encode,
     # partition_nth_indices,
-    # quarter, quarters_between, unique,
-    # value_counts
+    # quarter, quarters_between,
 
     # (strings)
     # ascii_capitalize, ascii_center, ascii_is_alnum, ascii_is_alpha, ascii_is_decimal,
@@ -143,30 +198,54 @@ module RedAmber
     # choose, index_in, index_in_meta_binary, indices_nonzero
 
     # (others)
-    # coalesce, drop_null, fill_null_backward, fill_null_forward,
-    # filter, is_finite, is_in, is_in_meta_binary, is_inf, is_nan, is_null, is_valid,
+    # coalesce, drop_null,
+    # filter, is_in, is_in_meta_binary,
     # list_element, list_flatten, list_parent_indices, list_value_length, make_struct,
-    # max_element_wise, min_element_wise, random, replace_with_mask, select_k_unstable,
+    # max_element_wise, min_element_wise, random, select_k_unstable,
     # sort_indices, struct_field, take
 
     private # =======
 
-    def exec_func(function, other: nil, options: {})
-      func = Arrow::Function.find(function)
-      output =
-        case other
-        when nil
-          func.execute([data])
-        when Arrow::Array, Arrow::ChunkedArray, Arrow::Scalar, Numeric
-          func.execute([data, other])
-        when Vector
-          func.execute([data, other.data])
-        when Rover::Vector
-          func.execute([data, other.to_a])
-        else
-          raise ArgumentError, "Operand is not supported: #{other.class}"
-        end
-      options[:aggregate] ? output.value : Vector.new(output.value)
+    def exec_func_unary(function, options: nil)
+      find(function).execute([data], options)
+    end
+
+    def exec_func_binary(function, other, options: nil)
+      case other
+      when Vector
+        find(function).execute([data, other.data], options)
+      when Arrow::Array, Arrow::ChunkedArray, Arrow::Scalar, Array, Numeric, String, TrueClass, FalseClass
+        find(function).execute([data, other], options)
+      else
+        raise VectorArgumentError, "Operand is not supported: #{other.class}"
+      end
+    end
+
+    def take_out_scalar(datum)
+      output = datum.value
+      case output
+      when Arrow::StringScalar then output.to_s
+      when Arrow::StructScalar
+        output.value.map { |s| s.is_a?(Arrow::StringScalar) ? s.to_s : s.value }
+      else
+        output.value
+      end
+    end
+
+    def take_out_element_wise(datum)
+      Vector.new(datum.value)
+    end
+
+    module_function # ======
+
+    def find(function_name)
+      Arrow::Function.find(function_name)
+    end
+
+    # temporary API until RedAmber document prepared.
+    def arrow_doc(function_name)
+      f = find(function_name)
+      "#{f}\n#{'-' * function_name.size}\n#{f.doc.description}"
     end
   end
 end

data/lib/red_amber/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RedAmber
-  VERSION = '0.1.2'
+  VERSION = '0.1.5'
 end

data/lib/red_amber.rb

@@ -3,9 +3,14 @@
 require 'arrow'
 require 'rover-df'
 
-require_relative 'red_amber/data_frame_output'
+require_relative 'red_amber/data_frame_displayable'
+require_relative 'red_amber/data_frame_helper'
+require_relative 'red_amber/data_frame_indexable'
 require_relative 'red_amber/data_frame_selectable'
+require_relative 'red_amber/data_frame_observation_operation'
+require_relative 'red_amber/data_frame_variable_operation'
 require_relative 'red_amber/data_frame'
+require_relative 'red_amber/vector_compensable'
 require_relative 'red_amber/vector_functions'
 require_relative 'red_amber/vector'
 require_relative 'red_amber/version'
@@ -15,4 +20,7 @@ module RedAmber
 
   class DataFrameArgumentError < ArgumentError; end
   class DataFrameTypeError < TypeError; end
+
+  class VectorArgumentError < ArgumentError; end
+  class VectorTypeError < TypeError; end
 end

data/red_amber.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.email = ['heronshoes877@gmail.com']
 
   spec.summary = 'Simple dataframe library for Ruby'
-  spec.description = 'RedAmber is a simple dataframe library powered by Red Arrow with simple API similar to Rover-df.'
+  spec.description = 'RedAmber is a simple dataframe library inspired by Rover-df and powered by Red Arrow.'
   spec.homepage = 'https://github.com/heronshoes/red_amber'
   spec.license = 'MIT'
   spec.required_ruby_version = '>= 2.7'
@@ -30,14 +30,11 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'red-arrow', '>= 7.0.0'
-  spec.add_dependency 'red-parquet', '>= 7.0.0'
+  spec.add_dependency 'red-arrow', '>= 8.0.0'
+  spec.add_dependency 'red-parquet', '>= 8.0.0'
   spec.add_dependency 'rover-df', '~> 0.3.0'
 
   # Development dependency has gone to the Gemfile (rubygems/bundler#7237)
 
   spec.metadata['rubygems_mfa_required'] = 'true'
-
-  # For more information and examples about making a new gem, check out our
-  # guide at: https://bundler.io/guides/creating_gem.html
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red_amber
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.5
 platform: ruby
 authors:
 - Hirokazu SUZUKI (heronshoes)
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-05-08 00:00:00.000000000 Z
+date: 2022-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: red-arrow
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.0.0
+        version: 8.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.0.0
+        version: 8.0.0
 - !ruby/object:Gem::Dependency
   name: red-parquet
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.0.0
+        version: 8.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.0.0
+        version: 8.0.0
 - !ruby/object:Gem::Dependency
   name: rover-df
   requirement: !ruby/object:Gem::Requirement
@@ -52,8 +52,8 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.0
-description: RedAmber is a simple dataframe library powered by Red Arrow with simple
-  API similar to Rover-df.
+description: RedAmber is a simple dataframe library inspired by Rover-df and powered
+  by Red Arrow.
 email:
 - heronshoes877@gmail.com
 executables: []
@@ -67,12 +67,39 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- benchmark/csv_load_penguins.yml
+- benchmark/drop_nil.yml
 - doc/CODE_OF_CONDUCT.md
+- doc/DataFrame.md
+- doc/Vector.md
+- doc/image/arrow_table_new.png
+- doc/image/dataframe/assign.png
+- doc/image/dataframe/drop.png
+- doc/image/dataframe/pick.png
+- doc/image/dataframe/remove.png
+- doc/image/dataframe/rename.png
+- doc/image/dataframe/slice.png
+- doc/image/dataframe_model.png
+- doc/image/example_in_red_arrow.png
+- doc/image/tdr.png
+- doc/image/tdr_and_table.png
+- doc/image/tidy_data_in_TDR.png
+- doc/image/vector/binary_element_wise.png
+- doc/image/vector/unary_aggregation.png
+- doc/image/vector/unary_aggregation_w_option.png
+- doc/image/vector/unary_element_wise.png
+- doc/tdr.md
+- doc/tdr_ja.md
 - lib/red_amber.rb
 - lib/red_amber/data_frame.rb
-- lib/red_amber/data_frame_output.rb
+- lib/red_amber/data_frame_displayable.rb
+- lib/red_amber/data_frame_helper.rb
+- lib/red_amber/data_frame_indexable.rb
+- lib/red_amber/data_frame_observation_operation.rb
 - lib/red_amber/data_frame_selectable.rb
+- lib/red_amber/data_frame_variable_operation.rb
 - lib/red_amber/vector.rb
+- lib/red_amber/vector_compensable.rb
 - lib/red_amber/vector_functions.rb
 - lib/red_amber/version.rb
 - red_amber.gemspec

data/lib/red_amber/data_frame_output.rb

@@ -1,116 +0,0 @@
-# frozen_string_literal: true
-
-require 'stringio'
-
-module RedAmber
-  # mix-ins for the class DataFrame
-  module DataFrameOutput
-    def to_s
-      @table.to_s
-    end
-
-    # def describe() end
-
-    # def summary() end
-
-    def inspect_raw
-      format "#<#{self.class}:0x%016x>\n#{self}", object_id
-    end
-
-    # - tally_level: max level to use tally mode
-    # - max_element: max element to show values in each row
-    # TODO: Is it better to change name other than `inspect` ?
-    # TODO: Add na count capability
-    # TODO: Fall back to inspect_raw when treating large dataset
-    # TODO: Refactor code to smaller methods
-    def inspect(tally_level: 5, max_element: 5)
-      return '#<RedAmber::DataFrame (empty)>' if empty?
-
-      stringio = StringIO.new # output string buffer
-
-      # 1st row: show shape of the dataframe
-      r = pl(nrow)
-      c = pl(ncol)
-      stringio.puts \
-        "#{self.class} : #{nrow} observation#{r}(row#{r}) of #{ncol} variable#{c}(column#{c})"
-
-      # 2nd row: show var counts by type
-      type_groups = data_types.map { |t| type_group(t) }
-
-      stringio.puts "Variable#{pl(ncol)} : #{var_type_count(type_groups).join(', ')}"
-
-      # 3rd row: print header of rows
-      levels = vectors.map { |v| v.to_a.uniq.size }
-      row_headers = { idx: '#', key: 'key', type: 'type', levels: 'level', data: 'data_preview' }
-      # find longest word to adjust column width
-      w_idx = ncol.to_s.size
-      w_key = (keys.map { |key| key.size + 1 } << row_headers[:key].size).max
-      w_type = (types.map(&:size) << row_headers[:type].size).max
-      w_row = (levels.map { |l| l.to_s.size } << row_headers[:levels].size).max
-      stringio.printf("%-#{w_idx}s %-#{w_key}s %-#{w_type}s %-#{w_row}s %s\n", *row_headers.values)
-
-      # (4) show details for each column (vector)
-      vectors.each.with_index(1) do |vector, i|
-        key = keys[i - 1]
-        type = types[i - 1]
-        type_group = type_groups[i - 1]
-        data_tally = vector.tally
-
-        str = format("%#{w_row}d ", data_tally.size)
-        str <<
-          case type_group
-          when :numeric, :string, :boolean
-            if data_tally.size <= tally_level && data_tally.size != nrow
-              data_tally.to_s
-            else
-              reduced_vector_presentation(vector, nrow, max_element)
-            end
-            #  c = vector.is_na.tally[1]   # release when `#is_na` impremented
-            #  str << " #{c} NaN#{pl(c)}" if c&.>(0)  # safely call c>0
-          else
-            reduced_vector_presentation(vector, nrow, max_element)
-          end
-
-        stringio.printf("%#{w_idx}d %-#{w_key}s %-#{w_type}s %s\n", i, ":#{key}", type, str)
-      end
-
-      stringio.string
-    end
-
-    private # =====
-
-    def pl(num)
-      num > 1 ? 's' : ''
-    end
-
-    def type_group(type)
-      if Arrow::NumericDataType >= type
-        :numeric
-      elsif Arrow::StringDataType >= type
-        :string
-      elsif Arrow::BooleanDataType >= type
-        :boolean
-      elsif Arrow::TemporalDataType >= type
-        :temporal
-      else
-        :other
-      end
-    end
-
-    def var_type_count(type_groups)
-      tg = type_groups.tally
-      a = []
-      a << "#{tg[:numeric]} numeric" if tg[:numeric]
-      a << "#{tg[:string]} string#{pl(tg[:string])}" if tg[:string]
-      a << "#{tg[:boolean]} boolean" if tg[:boolean]
-      a << "#{tg[:temporal]} temporal" if tg[:temporal]
-      a
-    end
-
-    def reduced_vector_presentation(vector, nrow, max_element)
-      a = vector.to_a.take(max_element)
-      a << '...' if nrow > max_element
-      "[#{a.join(', ')}]"
-    end
-  end
-end