red_amber 0.4.2 → 0.5.1

data/lib/red_amber/helper.rb

@@ -78,6 +78,32 @@ module RedAmber
         Array(range)
       end
     end
+
+    # Create sink node and execute plan
+    #
+    # @param plan [Arrow::ExecutePlan]
+    #   Execute plan of Acero.
+    # @param node [Arrow::ExecuteNode]
+    #   Execute node of Acero.
+    # @param output_schema [Arrow::Schema, nil]
+    #   Schema of table to output. If it is nil, output_schema of
+    #   sink node is used.
+    # @return [Arrow::Table]
+    #   Result of plan.
+    # @since 0.5.0
+    #
+    def sink_and_start_plan(plan, node, output_schema: nil)
+      sink_node_options = Arrow::SinkNodeOptions.new
+      plan.build_sink_node(node, sink_node_options)
+      plan.validate
+      plan.start
+      plan.wait
+      output_schema = node.output_schema if output_schema.nil?
+      reader = sink_node_options.get_reader(output_schema)
+      table = reader.read_all
+      plan.stop
+      table
+    end
   end
 
   # rubocop:disable Layout/LineLength

data/lib/red_amber/subframes.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module RedAmber
-  # class SubFrames treats a set of subsets of a DataFrame
+  # class SubFrames treats subsets of a DataFrame
   # [Experimental feature] Class SubFrames may be removed or be changed in the future.
   class SubFrames
     include Enumerable # may change to use Forwardable.
@@ -20,6 +20,7 @@ module RedAmber
         @sizes = []
       end
 
+      # Generic iterator method
       def each
         @selectors.each
       end
@@ -27,14 +28,20 @@ module RedAmber
 
     # Boolean selectors of sub-dataframes
     class Filters < Selectors
+      # Return sizes of filter
+      # @return [Array<Integer>]
+      #   sizes of each sub dataframes.
+      #   Counts true for each filter.
       def sizes
-        # count true
         @sizes = @selectors.map { |s| s.to_a.count { _1 } } # rubocop:disable Performance/Size
       end
     end
 
     # Index selectors of sub-dataframes
     class Indices < Selectors
+      # Return sizes of selector indices.
+      # @return [Array<Integer>]
+      #   sizes of each sub dataframes.
       def sizes
         @sizes = @selectors.map(&:size)
       end
@@ -93,7 +100,7 @@ module RedAmber
       # @since 0.4.0
       #
       def by_group(group)
-        SubFrames.new(group.dataframe, group.filters)
+        SubFrames.by_filters(group.dataframe, group.filters)
       end
 
       # Create a new SubFrames object from a DataFrame and an array of indices.
@@ -291,15 +298,15 @@ module RedAmber
         selectors = yield(dataframe)
       end
 
-      if dataframe.empty? || selectors.nil? || selectors.empty?
+      if dataframe.empty? || selectors.nil? || selectors.size.zero? # rubocop:disable Style/ZeroLengthPredicate
         @baseframe = DataFrame.new
         @selectors = Selectors.new([])
       else
         @baseframe = dataframe
         @selectors =
-          if selectors[0].boolean?
+          if selectors.first.boolean?
             Filters.new(selectors)
-          elsif selectors[0].numeric?
+          elsif selectors.first.numeric?
             Indices.new(selectors)
           else
             raise SubFramesArgumentError, "illegal type: #{selectors}"
@@ -427,7 +434,7 @@ module RedAmber
     #   @return [DataFrame]
     #     created DataFrame.
     #   @example Aggregate by key labels in arguments and values from block.
-    #     subframes.aggregate(:y, :sum_x) { [y.first, x.sum] }
+    #     subframes.aggregate(:y, :sum_x) { [y.one, x.sum] }
     #
     #     # =>
     #     #<RedAmber::DataFrame : 3 x 2 Vectors, 0x0000000000003b24>
@@ -438,7 +445,7 @@ module RedAmber
     #     2 C              6
     #
     #   @example Aggregate by key labels in an Array and values from block.
-    #     subframes.aggregate([:y, :sum_x]) { [y.first, x.sum] }
+    #     subframes.aggregate([:y, :sum_x]) { [y.one, x.sum] }
     #
     #     # =>
     #     #<RedAmber::DataFrame : 3 x 2 Vectors, 0x0000000000003b24>
@@ -450,7 +457,7 @@ module RedAmber
     #
     # @overload aggregate
     #
-    #   Aggregate SubFrames creating DataFrame with pairs of key and aggregated value
+    #   Aggregate SubFrames creating DataFrame with pairs of key and aggregated values
     #   in Hash from the block.
     #
     #   @yieldparam dataframe [DataFrame]
@@ -463,7 +470,7 @@ module RedAmber
     #     created DataFrame.
     #   @example Aggregate by key and value pairs from block.
     #     subframes.aggregate do
-    #       { y: y.first, sum_x: x.sum }
+    #       { y: y.one, sum_x: x.sum }
     #     end
     #
     #     # =>
@@ -705,7 +712,7 @@ module RedAmber
     #   @example
     #     subframes.assign(:sum_x, :frac_x) do
     #       group_sum = x.sum
-    #       [[group_sum] * size, x / s.to_f]
+    #       [[group_sum] * size, x / group_sum.to_f]
     #     end
     #
     #     # =>

data/lib/red_amber/vector.rb

@@ -10,21 +10,54 @@ module RedAmber
     include ArrowFunction
     include VectorUpdatable
     include VectorSelectable
+    include VectorStringFunction
 
     using RefineArrayLike
 
-    # Quicker constructor of Vector.
+    # Entity of Vector.
     #
-    # @param arrow_array [Arrow::Array]
-    #   Arrow::Array object to have in the Vector.
-    # @return [Vector]
-    #   created Vector.
-    # @note This method doesn't check argment type.
+    # @return [Arrow::Array]
+    #
+    attr_reader :data
+    alias_method :to_arrow_array, :data
+
+    # Associated key name when self is in a DataFrame.
+    #
+    # Default Vector is 'head-less' (key-less).
+    # @return [Symbol]
     #
-    def self.create(arrow_array)
-      instance = allocate
-      instance.instance_variable_set(:@data, arrow_array)
-      instance
+    attr_accessor :key
+
+    class << self
+      # Create a Vector (calling `.new`).
+      #
+      # @param (see #initialize)
+      # @return (see #initialize)
+      # @example Create an empty Vector.
+      #   Vector[]
+      #   # =>
+      #   #<RedAmber::Vector(:string, size=0):0x000000000000e2cc>
+      #   []
+      #
+      # @since 0.5.0
+      #
+      def [](...)
+        new(...)
+      end
+
+      # Quicker constructor of Vector.
+      #
+      # @param arrow_array [Arrow::Array]
+      #   Arrow::Array object to have in the Vector.
+      # @return [Vector]
+      #   created Vector.
+      # @note This method doesn't check argment type.
+      #
+      def create(arrow_array)
+        instance = allocate
+        instance.instance_variable_set(:@data, arrow_array)
+        instance
+      end
     end
 
     # Create a Vector.
@@ -51,20 +84,6 @@ module RedAmber
         end
     end
 
-    # Entity of Vector.
-    #
-    # @return [Arrow::Array]
-    #
-    attr_reader :data
-    alias_method :to_arrow_array, :data
-
-    # Associated key name when self is in a DataFrame.
-    #
-    # Default Vector is 'head-less' (key-less).
-    # @return [Symbol]
-    #
-    attr_accessor :key
-
     # Return other as a Vector which is same data type as self.
     #
     # @param other [Vector, Array, Arrow::Array, Arrow::ChunkedArray]
@@ -161,7 +180,8 @@ module RedAmber
         end
         sio << ']'
 
-        format "#<#{self.class}(:#{type}, size=#{size}):0x%016x>\n%s\n",
+        chunked = chunked? ? ', chunked' : ''
+        format "#<#{self.class}(:#{type}, size=#{size}#{chunked}):0x%016x>\n%s\n",
                object_id, sio.string
       end
     end

data/lib/red_amber/vector_aggregation.rb

@@ -161,6 +161,22 @@ module RedAmber
     #
     define_unary_aggregation :min_max
 
+    # Compute the 1 most common values and their respective
+    #   occurence counts.
+    #
+    # @note Self must be a numeric or a boolean Vector.
+    # @note ModeOptions are not supported in 0.5.0 .
+    #   Only one mode value is returned.
+    # @api private
+    # @return [Hash{'mode'=>mode, 'count'=>count}]
+    #    mode and count of self in an array.
+    # @since 0.5.0
+    #
+    def mode
+      datum = find(:mode).execute([data])
+      datum.value.to_a.first
+    end
+
     # Compute product value of self.
     #
     # @note Self must be a numeric Vector.
@@ -241,6 +257,16 @@ module RedAmber
     #     - nearest: returns i or j, whichever is closer.
     #     - midpoint: returns (i + j) / 2.
 
+    # Get a non-nil element in self.
+    #
+    # @return [Object, nil]
+    #   first non-nil value detected. If all elements are nil, return nil.
+    # @since 0.5.0
+    #
+    def one
+      each.find { !_1.nil? }
+    end
+
     # Returns a quantile value.
     # - 0.5 quantile (median) is returned by default.
     # - Or return quantile for specified probability (prob).

data/lib/red_amber/vector_selectable.rb

@@ -153,10 +153,23 @@ module RedAmber
     # @param element
     #   an element of self.
     # @return [integer, nil]
-    #   founded position of element. If it is not found, returns nil.
+    #   position of element. If it is not found, returns nil.
     #
     def index(element)
-      (0...size).find { |i| self[i] == element }
+      if element.nil?
+        datum = find(:is_null).execute([data])
+        value = Arrow::Scalar.resolve(true, :boolean)
+      else
+        datum = data
+        value = Arrow::Scalar.resolve(element, type)
+      end
+      datum = find(:index).execute([datum], value: value)
+      index = get_scalar(datum)
+      if index.negative?
+        nil
+      else
+        index
+      end
     end
 
     # Returns first element of self.
@@ -229,55 +242,118 @@ module RedAmber
       take(sort_indices(order: order))
     end
 
-    # Returns numerical rank of self.
+    # Returns 1-based numerical rank of self.
     # - Nil values are considered greater than any value.
     # - NaN values are considered greater than any value but smaller than nil values.
-    # - Tiebreakers are ranked in order of appearance.
-    # - `RankOptions` in C++ function is not implemented in C GLib yet.
-    #   This method is currently fixed to the default behavior.
-    #
+    # - Order of each element is considered as ascending by default. It is
+    #   changable by the parameter `order = :descending`.
+    # - Tiebreakers are ranked in order of appearance by default or
+    #   with `tie: :first` option.
+    # - Null values (nil and NaN) are placed at end by default.
+    #   This behavior can be changed by the option `null_placement: :at_start`.
+    #
+    # @param order [:ascending, '+', :descending, '-']
+    #   the order of the elements should be ranked in.
+    #   - :ascending or '+' : rank is computed in ascending order.
+    #   - :descending or '-' : rank is computed in descending order.
+    # @param tie [:first, :min, :max, :dense]
+    #   configure how ties between equal values are handled.
+    #   - first: Ranks are assigned in order of when ties appear in the input.
+    #   - min: Ties get the smallest possible rank in the sorted order.
+    #   - max: Ties get the largest possible rank in the sorted order.
+    #   - dense: The ranks span a dense [1, M] interval where M is the number
+    #     of distinct values in the input.
+    # @param null_placement [:at_end, :at_start]
+    #   configure the position of nulls to be located.
+    #   Nulls are considered as `NaN < nil`.
     # @return [Vector]
-    #   0-based rank of self (0...size in range).
+    #   1-based rank in uint64 of self (1..size in range) at maximum.
     # @example Rank of float Vector
-    #   fv = Vector.new(0.1, nil, Float::NAN, 0.2, 0.1); fv
+    #   float = Vector[1, 0, nil, Float::NAN, Float::INFINITY, -Float::INFINITY, 3, 2]
+    #   float
     #
     #   # =>
-    #   #<RedAmber::Vector(:double, size=5):0x000000000000c65c>
-    #   [0.1, nil, NaN, 0.2, 0.1]
+    #   #<RedAmber::Vector(:double, size=8):0x0000000000036858>
+    #   [1.0, 0.0, nil, NaN, Infinity, -Infinity, 3.0, 2.0]
     #
-    #   fv.rank
+    #   float.rank
+    #   # or float.rank(:ascending, tie: :first, null_placement: :at_end)
     #
     #   # =>
-    #   #<RedAmber::Vector(:uint64, size=5):0x0000000000003868>
-    #   [0, 4, 3, 2, 1]
+    #   #<RedAmber::Vector(:uint64, size=8):0x000000000003af84>
+    #   [3, 2, 8, 7, 6, 1, 5, 4]
     #
     # @example Rank of string Vector
-    #   sv = Vector.new("A", "B", nil, "A", "C"); sv
+    #   string = Vector["A", "A", nil, nil, "C", "B"]
+    #   string
+    #
+    #   # =>
+    #   #<RedAmber::Vector(:string, size=6):0x000000000003d568>
+    #   ["A", "A", nil, nil, "C", "B"]
+    #
+    #   string.rank
+    #
+    #   # =>
+    #   #<RedAmber::Vector(:uint64, size=6):0x0000000000049bc4>
+    #   [1, 2, 5, 6, 4, 3]
+    #
+    # @example Rank with order = :descending
+    #   float.rank(:descending) # or float.rank('-')
+    #
+    #   # =>
+    #   #<RedAmber::Vector(:uint64, size=8):0x000000000006ef00>
+    #   [4, 5, 8, 7, 1, 6, 2, 3]
+    #
+    # @example Rank with tie: :min
+    #   string.rank(tie: :min)
+    #
+    #   # =>
+    #   #<RedAmber::Vector(:uint64, size=6):0x000000000007a1d4>
+    #   [1, 1, 5, 5, 4, 3]
+    #
+    # @example Rank with tie: :max
+    #   string.rank(tie: :max)
+    #
+    #   # =>
+    #   #<RedAmber::Vector(:uint64, size=6):0x000000000007cba0>
+    #   [2, 2, 6, 6, 4, 3]
+    #
+    # @example Rank with tie: :dense
+    #   string.rank(tie: :dense)
     #
     #   # =>
-    #   #<RedAmber::Vector(:string, size=5):0x0000000000003854>
-    #   ["A", "B", nil, "A", "C"]
+    #   #<RedAmber::Vector(:uint64, size=6):0x0000000000080930>
+    #   [1, 1, 4, 4, 3, 2]
     #
-    #   sv.rank
+    # @example Rank with null_placement: :at_start
+    #   float.rank(null_placement: :at_start)
     #
     #   # =>
-    #   #<RedAmber::Vector(:uint64, size=5):0x0000000000003868>
-    #   [0, 2, 4, 1, 3]
+    #   #<RedAmber::Vector(:uint64, size=8):0x0000000000082104>
+    #   [5, 4, 1, 2, 8, 3, 7, 6]
     #
     # @since 0.4.0
     #
-    def rank
-      datum =
-        case data
-        when Arrow::ChunkedArray
-          Arrow::Function.find(:rank).execute([data.pack])
+    def rank(order = :ascending, tie: :first, null_placement: :at_end)
+      func = find(:rank)
+      options = func.default_options
+      order =
+        case order.to_sym
+        when :+, :ascending, :increasing
+          :ascending
+        when :-, :descending, :decreasing
+          :descending
         else
-          Arrow::Function.find(:rank).execute([data])
+          raise VectorArgumentError, "illegal order option: #{order}"
         end
-      Vector.create(datum.value) - 1
+      options.sort_keys = [Arrow::SortKey.resolve('', order)]
+      options.tiebreaker = tie
+      options.null_placement = null_placement
+      Vector.create(func.execute([data], options).value)
     end
 
     # Pick up elements at random.
+    # @note This method requires 'arrow-numo-narray' gem.
     #
     # @overload sample()
     #   Return a randomly selected element.
@@ -298,12 +374,12 @@ module RedAmber
     #     "C"
     #
     # @overload sample(n)
-    #   Pick up n elements at random.
+    #   Select n elements at random.
     #
     #   @param n [Integer]
-    #     positive number of elements to pick.
-    #     If n is smaller or equal to size, elements are picked by non-repeating.
-    #     If n is greater than `size`, elements are picked repeatedly.
+    #     positive number of elements to select.
+    #     If n is smaller or equal to size, elements are selected by non-repeating.
+    #     If n is greater than `size`, elements are selected repeatedly.
     #   @return [Vector]
     #     sampled elements.
     #     If n == 1 (in case of `sample(1)`), it returns a Vector of size == 1
@@ -315,7 +391,7 @@ module RedAmber
     #     #<RedAmber::Vector(:string, size=1):0x000000000001a3b0>
     #     ["H"]
     #
-    #   @example Sample same size of self: every element is picked in random order
+    #   @example Sample same size of self: every element is selected in random order
     #     v.sample(8)
     #
     #     # =>
@@ -330,18 +406,18 @@ module RedAmber
     #     ["E", "E", "A", "D", "H", "C", "A", "F", "H"]
     #
     # @overload sample(prop)
-    #   Pick up elements by proportion `prop` at random.
+    #   Select elements by proportion `prop` at random.
     #
     #   @param prop [Float]
-    #     positive proportion of elements to pick.
-    #     Absolute number of elements to pick:`prop*size` is rounded (by `half: :up``).
-    #     If prop is smaller or equal to 1.0, elements are picked by non-repeating.
-    #     If prop is greater than 1.0, some elements are picked repeatedly.
+    #     positive proportion of elements to select.
+    #     Absolute number of elements to select:`prop*size` is rounded (by `half: :up`).
+    #     If prop is smaller or equal to 1.0, elements are selected by non-repeating.
+    #     If prop is greater than 1.0, some elements are selected repeatedly.
     #   @return [Vector]
     #     sampled elements.
-    #     If picked element is only one, it returns a Vector of size == 1
+    #     If selected element is only one, it returns a Vector of size == 1
     #     not a scalar.
-    #   @example Sample same size of self: every element is picked in random order
+    #   @example Sample same size of self: every element is selected in random order
     #     v.sample(1.0)
     #
     #     # =>
@@ -355,6 +431,14 @@ module RedAmber
     #     #<RedAmber::Vector(:string, size=16):0x00000000000233e8>
     #     ["H", "B", "C", "B", "C", "A", "F", "A", "E", "C", "H", "F", "F", "A", ... ]
     #
+    #   @example prop less than 1.0
+    #     v.sample(0.7)
+    #
+    #     # =>
+    #     # Take (8 * 0.7).truncate => 5 samples
+    #     #<RedAmber::Vector(:string, size=5):0x000000000001afe0>
+    #     ["C", "A", "E", "H", "D"]
+    #
     # @since 0.4.0
     #
     def sample(n_or_prop = nil)
@@ -367,7 +451,7 @@ module RedAmber
         in Integer
           n_or_prop
         in Float
-          (n_or_prop * size).round
+          (n_or_prop * size).truncate
         in nil
           return to_a.sample
         else