red_amber 0.4.2 → 0.5.1

data/doc/qmd/red-amber.qmd

@@ -0,0 +1,90 @@
+---
+title: RedAmber Examples
+date: 2023-08-06
+author: heronshoes
+jupyter: ruby
+format:
+  pdf:
+    toc: true
+---
+
+This notebook walks through the [README of RedAmber](https://github.com/heronshoes/red_amber#readme).
+
+## `RedAmber::DataFrame`
+
+```{ruby}
+#| tags: []
+require 'red_amber'
+include RedAmber
+require 'datasets-arrow'
+
+{RedAmber: VERSION, Datasets: Datasets::VERSION}
+```
+
+## Example: diamonds dataset
+
+For the first loading of Datasets::Diamonds, it will take some time to download.
+
+```{ruby}
+#| tags: []
+dataset = Datasets::Diamonds.new
+diamonds = DataFrame.new(dataset)
+```
+
+```{ruby}
+#| tags: []
+df = diamonds
+  .slice { carat > 1 } # or use #filter instead of #slice
+  .group(:cut)
+  .mean(:price) # `pick` prior to `group` is not required if `:price` is specified here.
+  .sort('-mean(price)')
+```
+
+```{ruby}
+#| tags: []
+usdjpy = 110.0 # when the yen was stronger
+
+df.rename('mean(price)': :mean_price_USD)
+  .assign(:mean_price_JPY) { mean_price_USD * usdjpy }
+```
+
+## Example: starwars dataset
+
+```{ruby}
+#| tags: []
+uri = URI('https://vincentarelbundock.github.io/Rdatasets/csv/dplyr/starwars.csv')
+
+starwars = DataFrame.load(uri)
+```
+
+```{ruby}
+#| tags: []
+starwars
+  .drop(0) # delete unnecessary index column
+  .remove { species == "NA" } # delete unnecessary rows
+  .group(:species) { [count(:species), mean(:height, :mass)] }
+  .slice { count > 1 } # or use #filter instead of slice
+```
+
+## `RedAmber::Vector`
+
+```{ruby}
+#| tags: []
+penguins = DataFrame.new(Datasets::Penguins.new)
+```
+
+```{ruby}
+#| tags: []
+penguins[:bill_length_mm]
+```
+
+```{ruby}
+#| tags: []
+penguins[:bill_length_mm] < 40
+```
+
+```{ruby}
+#| tags: []
+penguins[:bill_length_mm].mean
+```
+

data/docker/Dockerfile

@@ -1,5 +1,5 @@
-# x86-64/Ubuntu-22.04/python-3.10.9/lab-3.6.1/notebook-6.5.3/2023-03-13
-ARG BASE_IMAGE_TAG=295612d3ade4
+# x86-64/Ubuntu-22.04/python-3.10.11/lab-3.6.3/notebook-6.5.4/2023-05-15
+ARG BASE_IMAGE_TAG=513d0cb8a67c
 
 FROM jupyter/minimal-notebook:$BASE_IMAGE_TAG
 

data/docker/Gemfile

@@ -5,11 +5,11 @@ source 'https://rubygems.org'
 gem 'irb'
 
 gem 'numo-narray'
-gem 'red-arrow', '~> 11.0.0'
+gem 'red-arrow', '~> 12.0.0'
 gem 'red-arrow-numo-narray'
-gem 'red-parquet', '~> 11.0.0'
+gem 'red-parquet', '~> 12.0.0'
 
-gem 'red_amber', '>=0.4.0'
+gem 'red_amber'
 gem 'red-amber-view'
 gem 'rover-df'
 
@@ -18,4 +18,9 @@ gem 'red-datasets-arrow'
 
 gem 'benchmark_driver'
 gem 'benchmark-ips'
+
+gem 'charty'
 gem 'faker'
+gem 'matplotlib'
+gem 'pycall'
+gem 'unicode_plot'

data/docker/docker-compose.yml

@@ -17,5 +17,5 @@ services:
     ports:
       - '8888:8888'
     volumes:
-      - ./notebook:/home/$NB_USER/work
+      - ../doc/notebook:/home/$NB_USER/work
     command: start-notebook.sh --NotebookApp.token=$TOKEN

data/docker/readme.md

@@ -6,12 +6,12 @@ This is a docker image containing RedAmber created from
 ## Contents
 
 - From jupyter/minimal-notebook:
-  - Based on 2023-03-13 (295612d3ade4)
+  - Based on 2023-05-15 (513d0cb8a67c)
   - x86-64
   - Ubuntu-22.04
-  - python-3.10.9
-  - lab-3.6.1
-  - notebook-6.5.3
+  - python-3.10.11
+  - lab-3.6.3
+  - notebook-6.5.4
 - System ruby-dev:
   - Ruby 3.0.2
 - Arrow 11.0.0 for Ubuntu:
@@ -22,7 +22,7 @@ This is a docker image containing RedAmber created from
 - Locally installed iruby:
   - Using Ruby 3.0.2
 - Locally installed bundler and Gemfile:
-  - RedAmber 0.4.1
+  - RedAmber 0.5.0
   - Others (see Gemfile)
 
 ## Install

data/lib/red_amber/data_frame.rb

@@ -422,12 +422,12 @@ module RedAmber
     # Create SubFrames by value grouping.
     #
     # [Experimental feature] this method may be removed or be changed in the future.
-    # @param keys [Symbol, String, Array<Symbol, String>]
+    # @param keys [List<Symbol, String>, Array<Symbol, String>]
     #   grouping keys.
     # @return [SubFrames]
     #   a created SubFrames grouped by column values on `keys`.
     # @example
-    #   df.sub_by_value(keys: :y)
+    #   df.sub_by_value(:y)
     #
     #   # =>
     #   #<RedAmber::SubFrames : 0x000000000000fc08>
@@ -454,10 +454,11 @@ module RedAmber
     #
     # @since 0.4.0
     #
-    def sub_by_value(keys: nil)
-      SubFrames.new(self, group(keys).filters)
+    def sub_by_value(*keys)
+      SubFrames.new(self, group(keys.flatten).filters)
     end
     alias_method :subframes_by_value, :sub_by_value
+    alias_method :sub_group, :sub_by_value
 
     # Create SubFrames by Windowing with `from`, `size` and `step`.
     #
@@ -697,6 +698,79 @@ module RedAmber
       end
     end
 
+    # Returns a Vector such that all elements have value `scalar`
+    #   and have same size as self.
+    #
+    # @overload propagate(scalar)
+    #   Specifies scalar as an agrument.
+    #
+    #   @param scalar [scalar]
+    #     a value to propagate in Vector.
+    #   @return [Vector]
+    #     created Vector.
+    #   @example propagate a value
+    #     df
+    #     # =>
+    #     #<RedAmber::DataFrame : 6 x 3 Vectors, 0x00000000000849a4>
+    #             x y        z
+    #       <uint8> <string> <boolean>
+    #     0       1 A        false
+    #     1       2 A        true
+    #     2       3 B        false
+    #     3       4 B        (nil)
+    #     4       5 B        true
+    #     5       6 C        false
+    #
+    #     df.assign(:sum_x) { propagate(x.sum) }
+    #     # =>
+    #     #<RedAmber::DataFrame : 6 x 4 Vectors, 0x000000000007bd04>
+    #             x y        z           sum_x
+    #       <uint8> <string> <boolean> <uint8>
+    #     0       1 A        false          21
+    #     1       2 A        true           21
+    #     2       3 B        false          21
+    #     3       4 B        (nil)          21
+    #     4       5 B        true           21
+    #     5       6 C        false          21
+    #
+    #     # Using `Vector#propagate` like below has same result as above.
+    #     df.assign(:sum_x) { x.propagate(:sum) }
+    #
+    #     # Also it is same as creating column from an Array.
+    #     df.assign(:sum_x) { [x.sum] * size }
+    #
+    # @overload propagate
+    #
+    #   @yieldparam self [DataFrame]
+    #     gives self to the block.
+    #   @yieldreturn [scalar]
+    #     a value to propagate in Vector
+    #   @return [Vector]
+    #     created Vector.
+    #   @example propagate the value from the block
+    #     df.assign(:range) { propagate { x.max - x.min } }
+    #     # =>
+    #     #<RedAmber::DataFrame : 6 x 4 Vectors, 0x00000000000e603c>
+    #             x y        z           range
+    #       <uint8> <string> <boolean> <uint8>
+    #     0       1 A        false           5
+    #     1       2 A        true            5
+    #     2       3 B        false           5
+    #     3       4 B        (nil)           5
+    #     4       5 B        true            5
+    #     5       6 C        false           5
+    #
+    # @since 0.5.0
+    #
+    def propagate(scalar = nil, &block)
+      if block
+        raise VectorArgumentError, "can't specify both function and block" if scalar
+
+        scalar = instance_eval(&block)
+      end
+      Vector.new([scalar] * size)
+    end
+
     # Catch variable (column) key as method name.
     def method_missing(name, *args, &block)
       return variables[name] if args.empty? && key?(name)

data/lib/red_amber/data_frame_combinable.rb

@@ -221,6 +221,11 @@ module RedAmber
     # - Same as `#join` with `type: :inner`
     # - A kind of mutating join.
     #
+    # @note the order of joined results will be preserved by default.
+    #   This is enabled by appending index column to sort after joining but
+    #   it will cause some performance degradation. If you don't matter
+    #   the order of the result, set `force_order` option to `false`.
+    #
     # @overload inner_join(other, suffix: '.1', force_order: true)
     #   If `join_key` is not specified, common keys in self and other are used
     #   (natural keys). Returns joined dataframe.
@@ -280,6 +285,11 @@ module RedAmber
     # - Same as `#join` with `type: :full_outer`
     # - A kind of mutating join.
     #
+    # @note the order of joined results will be preserved by default.
+    #   This is enabled by appending index column to sort after joining but
+    #   it will cause some performance degradation. If you don't matter
+    #   the order of the result, set `force_order` option to `false`.
+    #
     # @overload full_join(other, suffix: '.1', force_order: true)
     #   If `join_key` is not specified, common keys in self and other are used
     #   (natural keys). Returns joined dataframe.
@@ -348,6 +358,11 @@ module RedAmber
     # - Same as `#join` with `type: :left_outer`
     # - A kind of mutating join.
     #
+    # @note the order of joined results will be preserved by default.
+    #   This is enabled by appending index column to sort after joining but
+    #   it will cause some performance degradation. If you don't matter
+    #   the order of the result, set `force_order` option to `false`.
+    #
     # @overload left_join(other, suffix: '.1', force_order: true)
     #   If `join_key` is not specified, common keys in self and other are used
     #   (natural keys). Returns joined dataframe.
@@ -410,6 +425,11 @@ module RedAmber
     # - Same as `#join` with `type: :right_outer`
     # - A kind of mutating join.
     #
+    # @note the order of joined results will be preserved by default.
+    #   This is enabled by appending index column to sort after joining but
+    #   it will cause some performance degradation. If you don't matter
+    #   the order of the result, set `force_order` option to `false`.
+    #
     # @overload right_join(other, suffix: '.1', force_order: true)
     #   If `join_key` is not specified, common keys in self and other are used
     #   (natural keys). Returns joined dataframe.
@@ -422,11 +442,11 @@ module RedAmber
     #     df.right_join(other)
     #
     #     # =>
-    #       KEY           X1 X2
-    #       <string> <uint8> <boolean>
-    #     0 A              1 true
-    #     1 B              2 false
-    #     2 D          (nil) (nil)
+    #            X1 KEY      X2
+    #       <uint8> <string> <boolean>
+    #     0       1 A        true
+    #     1       2 B        false
+    #     2   (nil) D        (nil)
     #
     # @overload right_join(other, join_keys, suffix: '.1', force_order: true)
     #
@@ -439,11 +459,11 @@ module RedAmber
     #     df.right_join(other, :KEY)
     #
     #     # =>
-    #       KEY           X1 X2
-    #       <string> <uint8> <boolean>
-    #     0 A              1 true
-    #     1 B              2 false
-    #     2 D          (nil) (nil)
+    #            X1 KEY      X2
+    #       <uint8> <string> <boolean>
+    #     0       1 A        true
+    #     1       2 B        false
+    #     2   (nil) D        (nil)
     #
     # @overload right_join(other, join_key_pairs, suffix: '.1', force_order: true)
     #
@@ -456,11 +476,11 @@ module RedAmber
     #     df2.right_join(other2, { left: :KEY1, right: :KEY2 })
     #
     #     # =>
-    #       KEY1          X1 X2
-    #       <string> <uint8> <boolean>
-    #     0 A              1 true
-    #     1 B              2 false
-    #     2 D          (nil) (nil)
+    #             X1 KEY2     X2
+    #       <uint8> >string> <boolean>
+    #     0        1 A        true
+    #     1        2 B        false
+    #     2    (nil) D        (nil)
     #
     # @since 0.2.3
     #
@@ -480,6 +500,11 @@ module RedAmber
     # - Same as `#join` with `type: :left_semi`
     # - A kind of filtering join.
     #
+    # @note the order of joined results will be preserved by default.
+    #   This is enabled by appending index column to sort after joining but
+    #   it will cause some performance degradation. If you don't matter
+    #   the order of the result, set `force_order` option to `false`.
+    #
     # @overload semi_join(other, suffix: '.1', force_order: true)
     #   If `join_key` is not specified, common keys in self and other are used
     #   (natural keys). Returns joined dataframe.
@@ -539,6 +564,11 @@ module RedAmber
     # - Same as `#join` with `type: :left_anti`
     # - A kind of filtering join.
     #
+    # @note the order of joined results will be preserved by default.
+    #   This is enabled by appending index column to sort after joining but
+    #   it will cause some performance degradation. If you don't matter
+    #   the order of the result, set `force_order` option to `false`.
+    #
     # @overload anti_join(other, suffix: '.1', force_order: true)
     #   If `join_key` is not specified, common keys in self and other are used
     #   (natural keys). Returns joined dataframe.
@@ -661,7 +691,7 @@ module RedAmber
         raise DataFrameArgumentError, 'keys are not same with self and other'
       end
 
-      join(other, keys, type: :full_outer)
+      join(other, keys, type: :full_outer, force_order: true)
     end
 
     # Select records appearing in self but not in other.
@@ -733,12 +763,12 @@ module RedAmber
     #     1 B        E
     #     2 C        F
 
-    # @note the order of joined results will be preserved by default.
-    #   This is enabled by appending index column to sort after joining but
-    #   it will cause some performance degradation. If you don't matter
-    #   the order of the result, set `force_order` option to `false`.
+    # @note the order of joined results may not be preserved by default.
+    #   if you prefer to preserve the order of the result, set `force_order` option
+    #   to `true`. This is enabled by appending index column to sort after joining
+    #   so it will cause some performance degradation.
     #
-    # @overload join(other, type: :inner, suffix: '.1', force_order: true)
+    # @overload join(other, type: :inner, suffix: '.1', force_order: false)
     #
     #   If `join_key` is not specified, common keys in self and other are used
     #   (natural keys). Returns joined dataframe.
@@ -767,7 +797,7 @@ module RedAmber
     #     2 C              3 (nil)
     #     3 D          (nil) (nil)
     #
-    # @overload join(other, join_keys, type: :inner, suffix: '.1', force_order: true)
+    # @overload join(other, join_keys, type: :inner, suffix: '.1', force_order: false)
     #
     #   @macro join_before
     #   @macro join_key_in_array
@@ -792,7 +822,8 @@ module RedAmber
     #     0 A              1       1
     #     1 B              2       4
     #
-    # @overload join(other, join_key_pairs, type: :inner, suffix: '.1', force_order: true)
+    # @overload join(
+    #   other, join_key_pairs, type: :inner, suffix: '.1', force_order: false)
     #
     #   @macro join_before
     #   @macro join_key_in_hash
@@ -828,7 +859,8 @@ module RedAmber
     #
     # @since 0.2.3
     #
-    def join(other, join_keys = nil, type: :inner, suffix: '.1', force_order: true)
+    def join(other, join_keys = nil, type: :inner, suffix: '.1', force_order: false)
+      left_table = table
       right_table =
         case other
         when DataFrame
@@ -839,24 +871,26 @@ module RedAmber
           raise DataFrameArgumentError, 'other must be a DataFrame or an Arrow::Table'
         end
 
-      type = type.to_sym
-      left_index = :__LEFT_INDEX__
-      right_index = :__RIGHT_INDEX__
       if force_order
+        left_index = :__LEFT_INDEX__
+        right_index = :__RIGHT_INDEX__
         left_table = assign(left_index) { indices }.table
         other = DataFrame.create(other) if other.is_a?(Arrow::Table)
         right_table = other.assign(right_index) { indices }.table
-      else
-        left_table = table
       end
 
-      table_keys = left_table.keys
-      other_keys = right_table.keys
-
+      left_table_keys = ensure_keys(left_table.keys)
+      right_table_keys = ensure_keys(right_table.keys)
       # natural keys (implicit common keys)
-      join_keys ||= table_keys.intersection(other_keys)
+      join_keys ||= left_table_keys.intersection(right_table_keys)
+
+      type = Arrow::JoinType.try_convert(type) || type
+      type_nick = type.nick
+
+      plan = Arrow::ExecutePlan.new
+      left_node = plan.build_source_node(left_table)
+      right_node = plan.build_source_node(right_table)
 
-      # This is not necessary if additional procedure is contributed to Red Arrow.
       if join_keys.is_a?(Hash)
         left_keys = ensure_keys(join_keys[:left])
         right_keys = ensure_keys(join_keys[:right])
@@ -865,116 +899,110 @@ module RedAmber
         right_keys = left_keys
       end
 
-      case type
-      when :full_outer, :left_semi, :left_anti, :right_semi, :right_anti
-        left_outputs = nil
-        right_outputs = nil
-      when :inner, :left_outer
-        left_outputs = table_keys
-        right_outputs = other_keys - right_keys
-      when :right_outer
-        left_outputs = table_keys - left_keys
-        right_outputs = other_keys
+      context =
+        [type_nick, left_table_keys, right_table_keys, left_keys, right_keys, suffix]
+
+      hash_join_node_options = Arrow::HashJoinNodeOptions.new(type, left_keys, right_keys)
+      case type_nick
+      when 'inner', 'left-outer'
+        hash_join_node_options.left_outputs = left_table_keys
+        hash_join_node_options.right_outputs = right_table_keys - right_keys
+      when 'right-outer'
+        hash_join_node_options.left_outputs = left_table_keys - left_keys
+        hash_join_node_options.right_outputs = right_table_keys
       end
 
-      # Should we rescue errors in Arrow::Table#join for usability ?
-      joined_table =
-        left_table.join(
-          right_table,
-          join_keys,
-          type: type,
-          left_outputs: left_outputs,
-          right_outputs: right_outputs
-        )
-
-      case type
-      when :inner, :left_outer, :left_semi, :left_anti, :right_semi, :right_anti
-        dataframe =
-          if joined_table.keys.uniq!
-            DataFrame.create(rename_table(joined_table, n_keys, suffix))
-          else
-            DataFrame.create(joined_table)
-          end
+      hash_join_node =
+        plan.build_hash_join_node(left_node, right_node, hash_join_node_options)
+      merge_node = merge_keys(plan, hash_join_node, context)
+      rename_node = rename_keys(plan, merge_node, context)
+      joined_table = sink_and_start_plan(plan, rename_node)
+
+      df = DataFrame.create(joined_table)
+      if force_order
         sorter =
-          case type
-          when :inner, :left_outer
-            [left_index, right_index]
-          when :left_semi, :left_anti
-            [left_index]
-          when :right_semi, :right_anti
+          case type_nick
+          when 'right-semi', 'right-anti'
             [right_index]
-          end
-      when :full_outer
-        key_index_lr =
-          left_keys.map { left_table.keys.index(_1) }
-            .zip(right_keys.map { left_table.keys.size + right_table.keys.index(_1) })
-        renamed_table = rename_table(joined_table, n_keys, suffix)
-        dropper = []
-        dataframe =
-          DataFrame.create(renamed_table).assign do |df|
-            key_index_lr.map do |l, r|
-              dropper << df.keys[r]
-              [df.keys[l], merge_array(df.vectors[l].data, df.vectors[r].data)]
-            end
-          end
-        dataframe = dataframe.drop(dropper)
-        sorter = [left_index, right_index]
-      when :right_outer
-        dataframe =
-          if joined_table.keys.uniq!
-            DataFrame.create(rename_table(joined_table, left_outputs.size, suffix))
+          when 'left-semi', 'left-anti'
+            [left_index]
           else
-            DataFrame.create(joined_table)
+            [left_index, right_index]
           end
-        dataframe = dataframe.pick(right_keys, dataframe.keys - right_keys)
-        sorter = [left_index, right_index]
-      end
-
-      if force_order
-        dataframe
-          .sort(sorter)
+        df.sort(sorter)
           .drop(sorter)
       else
-        dataframe
+        df
       end
     end
 
     private
 
-    # To ensure Array of Symbols
+    # To ensure Array of Strings
     def ensure_keys(keys)
-      Array(keys).map(&:to_sym)
+      Array(keys).map(&:to_s)
+    end
+
+    # Merge key columns and preserve as left and remove right.
+    def merge_keys(plan, input_node, context)
+      type_nick, left_table_keys, right_table_keys, left_keys, right_keys, * = context
+      return input_node unless type_nick == 'full-outer'
+
+      left_indices = left_keys.map { left_table_keys.index(_1) }
+      right_offset = left_table_keys.size
+      right_indices = right_keys.map { right_table_keys.index(_1) + right_offset }
+      expressions = []
+      names = []
+      left_table_keys.each_with_index do |key, index|
+        names << key
+        expressions <<
+          if (i = left_indices.index(index))
+            left_field = Arrow::FieldExpression.new("[#{left_indices[i]}]")
+            right_field = Arrow::FieldExpression.new("[#{right_indices[i]}]")
+            is_left_null = Arrow::CallExpression.new('is_null', [left_field])
+            Arrow::CallExpression.new('if_else', [is_left_null, right_field, left_field])
+          else
+            Arrow::FieldExpression.new("[#{index}]")
+          end
+      end
+      right_table_keys.each.with_index(right_offset) do |key, index|
+        unless right_indices.include?(index)
+          names << key
+          expressions << Arrow::FieldExpression.new("[#{index}]")
+        end
+      end
+      project_node_options = Arrow::ProjectNodeOptions.new(expressions, names)
+      plan.build_project_node(input_node, project_node_options)
     end
 
-    # Rename duplicate keys by suffix
-    def rename_table(joined_table, n_keys, suffix)
-      joined_keys = joined_table.keys
-      other_keys = joined_keys[n_keys..]
+    def rename_keys(plan, input_node, context)
+      type_nick, left_table_keys, right_table_keys, *, suffix = context
+      names = input_node.output_schema.fields.map(&:name)
+      return input_node unless names.dup.uniq!
 
-      dup_keys = joined_keys.tally.select { |_, v| v > 1 }.keys
+      pos_rights =
+        if type_nick.start_with?('right')
+          names.size - right_table_keys.size
+        else
+          left_table_keys.size
+        end
+      rights = names[pos_rights..]
+      dup_keys = names.tally.select { |_, v| v > 1 }.keys
       renamed_right_keys =
-        other_keys.map do |key|
+        rights.map do |key|
           if dup_keys.include?(key)
-            suffixed = "#{key}#{suffix}".to_sym
+            suffixed = "#{key}#{suffix}".to_s
             # Find a key from suffixed.succ
-            (suffixed..).find { !joined_keys.include?(_1) }
+            (suffixed..).find { !names.include?(_1) }
           else
             key
           end
         end
-      joined_keys[n_keys..] = renamed_right_keys
-
-      fields =
-        joined_keys.map.with_index do |k, i|
-          Arrow::Field.new(k, joined_table[i].data_type)
-        end
-      Arrow::Table.new(Arrow::Schema.new(fields), joined_table.columns)
-    end
+      names[pos_rights..] = renamed_right_keys
 
-    # Merge two Arrow::Arrays
-    def merge_array(array1, array2)
-      t = Arrow::Function.find(:is_null).execute([array1])
-      Arrow::Function.find(:if_else).execute([t, array2, array1]).value
+      expressions = names.map.with_index { |_, i| Arrow::FieldExpression.new("[#{i}]") }
+      project_node_options = Arrow::ProjectNodeOptions.new(expressions, names)
+      plan.build_project_node(input_node, project_node_options)
     end
   end
 end