red_amber 0.1.3 → 0.1.4

data/lib/red_amber/data_frame_observation_operation.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module RedAmber
+  # mix-ins for the class DataFrame
+  module DataFrameObservationOperation
+    # slice and select some observations to create sub DataFrame
+    def slice(*args, &block)
+      slicer = args
+      if block
+        raise DataFrameArgumentError, 'Must not specify both arguments and block.' unless args.empty?
+
+        slicer = instance_eval(&block)
+      end
+      slicer = [slicer].flatten
+      return remove_all_values if slicer.empty? || slicer[0].nil?
+
+      # filter with same length
+      booleans = nil
+      if slicer[0].is_a?(Vector) || slicer[0].is_a?(Arrow::BooleanArray)
+        booleans = slicer[0].to_a
+      elsif slicer.size == size && booleans?(slicer)
+        booleans = slicer
+      end
+      return select_obs_by_boolean(booleans) if booleans
+
+      # filter with indexes
+      slicer = expand_range(slicer)
+      return select_obs_by_indeces(slicer) if integers?(slicer)
+
+      raise DataFrameArgumentError, "Invalid argument #{args}"
+    end
+
+    # remove selected observations to create sub DataFrame
+    def remove(*args, &block)
+      remover = args
+      if block
+        raise DataFrameArgumentError, 'Must not specify both arguments and block.' unless args.empty?
+
+        remover = instance_eval(&block)
+      end
+      remover = [remover].flatten
+
+      return self if remover.empty?
+
+      # filter with same length
+      booleans = nil
+      if remover[0].is_a?(Vector) || remover[0].is_a?(Arrow::BooleanArray)
+        booleans = remover[0].to_a
+      elsif remover.size == size && booleans?(remover)
+        booleans = remover
+      end
+      if booleans
+        inverted = booleans.map(&:!)
+        return select_obs_by_boolean(inverted)
+      end
+
+      # filter with indexes
+      slicer = indexes.to_a - expand_range(remover)
+      return remove_all_values if slicer.empty?
+      return select_obs_by_indeces(slicer) if integers?(slicer)
+
+      raise DataFrameArgumentError, "Invalid argument #{args}"
+    end
+
+    private
+
+    # return a DataFrame with same keys as self without values
+    def remove_all_values
+      DataFrame.new(keys.each_with_object({}) { |key, h| h[key] = [] })
+    end
+  end
+end

data/lib/red_amber/data_frame_selectable.rb

@@ -9,14 +9,26 @@ module RedAmber
       raise DataFrameArgumentError, 'Empty dataframe' if empty?
       raise DataFrameArgumentError, 'Empty argument' if args.empty?
 
-      # expand Range like [1..3, 4] to [1, 2, 3, 4]
-      expanded =
-        args.each_with_object([]) do |e, a|
-          e.is_a?(Range) ? a.concat(normalized_array(e)) : a.append(e)
+      if args.one?
+        case args[0]
+        when Vector
+          return select_obs_by_boolean(Arrow::BooleanArray.new(args[0].data))
+        when Arrow::BooleanArray
+          return select_obs_by_boolean(args[0])
+        when Array
+          return select_obs_by_boolean(Arrow::BooleanArray.new(args[0]))
+
+          # when Hash
+          # specify conditions to select by a Hash
         end
+      end
 
-      return select_rows(expanded) if integers?(expanded)
-      return select_columns(expanded.map(&:to_sym)) if sym_or_str?(expanded)
+      return select_obs_by_boolean(args) if booleans?(args)
+
+      # expand Range like [1..3, 4] to [1, 2, 3, 4]
+      expanded = expand_range(args)
+      return select_obs_by_indeces(expanded) if integers?(expanded)
+      return select_vars_by_keys(expanded.map(&:to_sym)) if sym_or_str?(expanded)
 
       raise DataFrameArgumentError, "Invalid argument #{args}"
     end
@@ -41,51 +53,17 @@ module RedAmber
       tail(n_rows)
     end
 
-    private # =====
+    private
 
-    def select_columns(keys)
+    def select_vars_by_keys(keys)
       if keys.one?
         t = @table[*keys]
-        raise DataFrameArgumentError, "Key is not exists #{keys}" unless t
+        raise DataFrameArgumentError, "Key does not exist #{keys}" unless t
 
         Vector.new(t.data)
       else
         DataFrame.new(@table[keys])
       end
     end
-
-    def select_rows(indeces)
-      out_of_range?(indeces) && raise(DataFrameArgumentError, "Invalid index: #{indeces} for 0..#{size - 1}")
-
-      a = indeces.map { |i| @table.slice(i).to_a }
-      DataFrame.new(@table.schema, a)
-    end
-
-    def normalized_array(range)
-      both_end = [range.begin, range.end]
-      both_end[1] -= 1 if range.exclude_end? && range.end.is_a?(Integer)
-
-      if both_end.any?(Integer) || both_end.all?(&:nil?)
-        if both_end.any? { |e| e&.>=(size) || e&.<(-size) }
-          raise DataFrameArgumentError, "Index out of range: #{range} for 0..#{size - 1}"
-        end
-
-        (0...size).to_a[range]
-      else
-        range.to_a
-      end
-    end
-
-    def out_of_range?(indeces)
-      indeces.max >= size || indeces.min < -size
-    end
-
-    def integers?(enum)
-      enum.all?(Integer)
-    end
-
-    def sym_or_str?(enum)
-      enum.all? { |e| e.is_a?(Symbol) || e.is_a?(String) }
-    end
   end
 end

data/lib/red_amber/data_frame_variable_operation.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+module RedAmber
+  # mix-ins for the class DataFrame
+  module DataFrameVariableOperation
+    # pick up some variables to create sub DataFrame
+    def pick(*args, &block)
+      picker = args
+      if block
+        raise DataFrameArgumentError, 'Must not specify both arguments and block.' unless args.empty?
+
+        picker = instance_eval(&block)
+      end
+      picker = [picker].flatten
+      return DataFrame.new if picker.empty? || picker == [nil]
+
+      picker = keys_by_booleans(picker) if booleans?(picker)
+
+      # DataFrame#[] creates a Vector with single key is specified.
+      # DataFrame#pick creates a DataFrame with single key.
+      return DataFrame.new(@table[picker]) if sym_or_str?(picker)
+
+      raise DataFrameArgumentError, "Invalid argument #{args}"
+    end
+
+    # drop some variables to create remainer sub DataFrame
+    def drop(*args, &block)
+      dropper = args
+      if block
+        raise DataFrameArgumentError, 'Must not specify both arguments and block.' unless args.empty?
+
+        dropper = instance_eval(&block)
+      end
+      dropper = [dropper].flatten
+      dropper = keys_by_booleans(dropper) if booleans?(dropper)
+
+      picker = keys - dropper
+      return DataFrame.new if picker.empty?
+
+      # DataFrame#[] creates a Vector with single key is specified.
+      # DataFrame#drop creates a DataFrame with single key.
+      return DataFrame.new(@table[picker]) if sym_or_str?(picker)
+
+      raise DataFrameArgumentError, "Invalid argument #{args}"
+    end
+
+    # rename variables to create new DataFrame
+    def rename(*args, &block)
+      renamer = args
+      if block
+        raise DataFrameArgumentError, 'Must not specify both arguments and a block' unless args.empty?
+
+        renamer = instance_eval(&block)
+      end
+      renamer = [renamer].flatten
+      return self if renamer.empty?
+
+      return rename_by_hash([renamer].to_h) if renamer.size == 2 && sym_or_str?(renamer) # rename(from, to)
+      return rename_by_hash(renamer[0]) if renamer.one? && renamer[0].is_a?(Hash) # rename({from => to})
+
+      raise DataFrameArgumentError, "Invalid argument #{args}"
+    end
+
+    # assign variables to create new DataFrame
+    def assign(*args, &block)
+      assigner = args
+      if block
+        raise DataFrameArgumentError, 'Must not specify both arguments and a block' unless args.empty?
+
+        assigner = instance_eval(&block)
+      end
+      assigner = [assigner].flatten
+      return self if assigner.empty? || assigner == [nil]
+
+      raise DataFrameArgumentError, "Invalid argument #{args}" unless assigner.one? && assigner[0].is_a?(Hash)
+
+      updater = {}
+      appender = {}
+      assigner[0].each do |key, value|
+        if keys.include? key
+          updater[key] = value
+        else
+          appender[key] = value
+        end
+      end
+      fields, arrays = update_fields_and_arrays(updater)
+      append_to_fields_and_arrays(appender, fields, arrays) unless appender.empty?
+
+      DataFrame.new(Arrow::Table.new(Arrow::Schema.new(fields), arrays))
+    end
+
+    private
+
+    def rename_by_hash(key_pairs)
+      fields = keys.map do |key|
+        new_key = key_pairs[key]
+        if new_key
+          Arrow::Field.new(new_key.to_sym, @table[key].data_type)
+        else
+          @table.schema[key]
+        end
+      end
+      schema = Arrow::Schema.new(fields)
+      DataFrame.new(Arrow::Table.new(schema, @table.columns))
+    end
+
+    def update_fields_and_arrays(updater)
+      fields = @table.columns.map(&:field)
+      arrays = @table.columns.map(&:data) # chunked_arrays
+      keys.each_with_index do |key, i|
+        data = updater[key]
+        next unless data
+
+        raise DataFrameArgumentError, "Data size mismatch (#{data.size} != #{size})" if data.size != size
+
+        a = Arrow::Array.new(data.is_a?(Vector) ? data.to_a : data)
+        fields[i] = Arrow::Field.new(key, a.value_data_type)
+        arrays[i] = Arrow::ChunkedArray.new([a])
+      end
+      [fields, arrays]
+    end
+
+    def append_to_fields_and_arrays(appender, fields, arrays)
+      appender.each do |key, data|
+        raise DataFrameArgumentError, "Data size mismatch (#{data.size} != #{size})" if data.size != size
+
+        a = Arrow::Array.new(data.is_a?(Vector) ? data.to_a : data)
+        fields << Arrow::Field.new(key.to_sym, a.value_data_type)
+        arrays << Arrow::ChunkedArray.new([a])
+      end
+    end
+  end
+end

data/lib/red_amber/vector_functions.rb

@@ -12,21 +12,30 @@ module RedAmber
   module VectorFunctions
     # [Unary aggregations]: vector.func => scalar
     unary_aggregations =
-      %i[all any approximate_median count count_distinct max mean min product stddev sum variance]
+      %i[all any approximate_median count count_distinct max mean min min_max product stddev sum variance]
     unary_aggregations.each do |function|
       define_method(function) do |opts: nil|
-        output = exec_func_unary(function, options: opts)
-        take_out_scalar(output)
+        datum = exec_func_unary(function, options: opts)
+        take_out_scalar(datum)
       end
     end
     alias_method :median, :approximate_median
     alias_method :count_uniq, :count_distinct
 
+    def unbiased_variance
+      variance(opts: { ddof: 1 })
+    end
+    alias_method :var, :unbiased_variance
+
+    def sd
+      stddev(opts: { ddof: 1 })
+    end
+    alias_method :std, :sd
+
     # option(s) required
     # - index
 
     # Returns other than value
-    # - min_max
     # - mode
     # - quantile
     # - tdigest
@@ -36,8 +45,8 @@ module RedAmber
       %i[abs atan bit_wise_not ceil cos floor is_finite is_inf is_nan is_null is_valid sign sin tan trunc]
     unary_element_wise.each do |function|
       define_method(function) do |opts: nil|
-        output = exec_func_unary(function, options: opts)
-        take_out_element_wise(output)
+        datum = exec_func_unary(function, options: opts)
+        take_out_element_wise(datum)
       end
     end
     alias_method :is_nil, :is_null
@@ -53,13 +62,13 @@ module RedAmber
     }
     unary_element_wise_op.each do |function, operator|
       define_method(function) do |opts: nil|
-        output = exec_func_unary(function, options: opts)
-        take_out_element_wise(output)
+        datum = exec_func_unary(function, options: opts)
+        take_out_element_wise(datum)
       end
 
       define_method(operator) do |opts: nil|
-        output = exec_func_unary(function, options: opts)
-        take_out_element_wise(output)
+        datum = exec_func_unary(function, options: opts)
+        take_out_element_wise(datum)
       end
     end
     alias_method :not, :invert
@@ -79,8 +88,8 @@ module RedAmber
       %i[atan2 and_not and_not_kleene bit_wise_and bit_wise_or bit_wise_xor]
     binary_element_wise.each do |function|
       define_method(function) do |other, opts: nil|
-        output = exec_func_binary(function, other, options: opts)
-        take_out_element_wise(output)
+        datum = exec_func_binary(function, other, options: opts)
+        take_out_element_wise(datum)
       end
     end
 
@@ -95,8 +104,8 @@ module RedAmber
     }
     logical_binary_element_wise.each do |method, function|
       define_method(method) do |other, opts: nil|
-        output = exec_func_binary(function, other, options: opts)
-        take_out_element_wise(output)
+        datum = exec_func_binary(function, other, options: opts)
+        take_out_element_wise(datum)
       end
     end
 
@@ -128,13 +137,13 @@ module RedAmber
     }
     binary_element_wise_op.each do |function, operator|
       define_method(function) do |other, opts: nil|
-        output = exec_func_binary(function, other, options: opts)
-        take_out_element_wise(output)
+        datum = exec_func_binary(function, other, options: opts)
+        take_out_element_wise(datum)
       end
 
       define_method(operator) do |other, opts: nil|
-        output = exec_func_binary(function, other, options: opts)
-        take_out_element_wise(output)
+        datum = exec_func_binary(function, other, options: opts)
+        take_out_element_wise(datum)
       end
     end
     alias_method :eq, :equal
@@ -195,29 +204,45 @@ module RedAmber
     private # =======
 
     def exec_func_unary(function, options: nil)
-      func = Arrow::Function.find(function)
-      func.execute([data], options)
+      find(function).execute([data], options)
     end
 
     def exec_func_binary(function, other, options: nil)
-      func = Arrow::Function.find(function)
       case other
       when Vector
-        func.execute([data, other.data], options)
-      when Arrow::Array, Arrow::ChunkedArray, Arrow::Scalar, Array, Numeric
-        func.execute([data, other], options)
+        find(function).execute([data, other.data], options)
+      when Arrow::Array, Arrow::ChunkedArray, Arrow::Scalar, Array, Numeric, String, TrueClass, FalseClass
+        find(function).execute([data, other], options)
       else
         raise ArgumentError, "Operand is not supported: #{other.class}"
       end
     end
 
-    def take_out_scalar(output)
-      output = output.value
-      output.is_a?(Arrow::StringScalar) ? output.to_s : output.value
+    def take_out_scalar(datum)
+      output = datum.value
+      case output
+      when Arrow::StringScalar then output.to_s
+      when Arrow::StructScalar
+        output.value.map { |s| s.is_a?(Arrow::StringScalar) ? s.to_s : s.value }
+      else
+        output.value
+      end
+    end
+
+    def take_out_element_wise(datum)
+      Vector.new(datum.value)
+    end
+
+    module_function # ======
+
+    def find(function_name)
+      Arrow::Function.find(function_name)
     end
 
-    def take_out_element_wise(output)
-      Vector.new(output.value)
+    # temporary API until RedAmber document prepared.
+    def arrow_doc(function_name)
+      f = find(function_name)
+      "#{f}\n#{'-' * function_name.size}\n#{f.doc.description}"
     end
   end
 end

data/lib/red_amber/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RedAmber
-  VERSION = '0.1.3'
+  VERSION = '0.1.4'
 end

data/lib/red_amber.rb

@@ -3,8 +3,11 @@
 require 'arrow'
 require 'rover-df'
 
-require_relative 'red_amber/data_frame_output'
+require_relative 'red_amber/data_frame_displayable'
+require_relative 'red_amber/data_frame_helper'
 require_relative 'red_amber/data_frame_selectable'
+require_relative 'red_amber/data_frame_observation_operation'
+require_relative 'red_amber/data_frame_variable_operation'
 require_relative 'red_amber/data_frame'
 require_relative 'red_amber/vector_functions'
 require_relative 'red_amber/vector'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red_amber
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Hirokazu SUZUKI (heronshoes)
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-05-14 00:00:00.000000000 Z
+date: 2022-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: red-arrow
@@ -68,10 +68,34 @@ files:
 - README.md
 - Rakefile
 - doc/CODE_OF_CONDUCT.md
+- doc/DataFrame.md
+- doc/Vector.md
+- doc/image/TDR_operations.pdf
+- doc/image/arrow_table_new.png
+- doc/image/dataframe/assign.png
+- doc/image/dataframe/drop.png
+- doc/image/dataframe/pick.png
+- doc/image/dataframe/remove.png
+- doc/image/dataframe/rename.png
+- doc/image/dataframe/slice.png
+- doc/image/dataframe_model.png
+- doc/image/example_in_red_arrow.png
+- doc/image/tdr.png
+- doc/image/tdr_and_table.png
+- doc/image/tidy_data_in_TDR.png
+- doc/image/vector/binary_element_wise.png
+- doc/image/vector/unary_aggregation.png
+- doc/image/vector/unary_aggregation_w_option.png
+- doc/image/vector/unary_element_wise.png
+- doc/tdr.md
+- doc/tdr_ja.md
 - lib/red_amber.rb
 - lib/red_amber/data_frame.rb
-- lib/red_amber/data_frame_output.rb
+- lib/red_amber/data_frame_displayable.rb
+- lib/red_amber/data_frame_helper.rb
+- lib/red_amber/data_frame_observation_operation.rb
 - lib/red_amber/data_frame_selectable.rb
+- lib/red_amber/data_frame_variable_operation.rb
 - lib/red_amber/vector.rb
 - lib/red_amber/vector_functions.rb
 - lib/red_amber/version.rb