red_amber 0.5.0 → 0.5.1

data/doc/qmd/red-amber.qmd

@@ -0,0 +1,90 @@
+---
+title: RedAmber Examples
+date: 2023-08-06
+author: heronshoes
+jupyter: ruby
+format:
+  pdf:
+    toc: true
+---
+
+This notebook walks through the [README of RedAmber](https://github.com/heronshoes/red_amber#readme).
+
+## `RedAmber::DataFrame`
+
+```{ruby}
+#| tags: []
+require 'red_amber'
+include RedAmber
+require 'datasets-arrow'
+
+{RedAmber: VERSION, Datasets: Datasets::VERSION}
+```
+
+## Example: diamonds dataset
+
+For the first loading of Datasets::Diamonds, it will take some time to download.
+
+```{ruby}
+#| tags: []
+dataset = Datasets::Diamonds.new
+diamonds = DataFrame.new(dataset)
+```
+
+```{ruby}
+#| tags: []
+df = diamonds
+  .slice { carat > 1 } # or use #filter instead of #slice
+  .group(:cut)
+  .mean(:price) # `pick` prior to `group` is not required if `:price` is specified here.
+  .sort('-mean(price)')
+```
+
+```{ruby}
+#| tags: []
+usdjpy = 110.0 # when the yen was stronger
+
+df.rename('mean(price)': :mean_price_USD)
+  .assign(:mean_price_JPY) { mean_price_USD * usdjpy }
+```
+
+## Example: starwars dataset
+
+```{ruby}
+#| tags: []
+uri = URI('https://vincentarelbundock.github.io/Rdatasets/csv/dplyr/starwars.csv')
+
+starwars = DataFrame.load(uri)
+```
+
+```{ruby}
+#| tags: []
+starwars
+  .drop(0) # delete unnecessary index column
+  .remove { species == "NA" } # delete unnecessary rows
+  .group(:species) { [count(:species), mean(:height, :mass)] }
+  .slice { count > 1 } # or use #filter instead of slice
+```
+
+## `RedAmber::Vector`
+
+```{ruby}
+#| tags: []
+penguins = DataFrame.new(Datasets::Penguins.new)
+```
+
+```{ruby}
+#| tags: []
+penguins[:bill_length_mm]
+```
+
+```{ruby}
+#| tags: []
+penguins[:bill_length_mm] < 40
+```
+
+```{ruby}
+#| tags: []
+penguins[:bill_length_mm].mean
+```
+

data/docker/Dockerfile

@@ -1,5 +1,5 @@
-# x86-64/Ubuntu-22.04/python-3.10.9/lab-3.6.1/notebook-6.5.3/2023-03-13
-ARG BASE_IMAGE_TAG=295612d3ade4
+# x86-64/Ubuntu-22.04/python-3.10.11/lab-3.6.3/notebook-6.5.4/2023-05-15
+ARG BASE_IMAGE_TAG=513d0cb8a67c
 
 FROM jupyter/minimal-notebook:$BASE_IMAGE_TAG
 

data/docker/Gemfile

@@ -9,7 +9,7 @@ gem 'red-arrow', '~> 12.0.0'
 gem 'red-arrow-numo-narray'
 gem 'red-parquet', '~> 12.0.0'
 
-gem 'red_amber', path: '../'
+gem 'red_amber'
 gem 'red-amber-view'
 gem 'rover-df'
 

data/docker/docker-compose.yml

@@ -17,5 +17,5 @@ services:
     ports:
       - '8888:8888'
     volumes:
-      - ./notebook:/home/$NB_USER/work
+      - ../doc/notebook:/home/$NB_USER/work
     command: start-notebook.sh --NotebookApp.token=$TOKEN

data/docker/readme.md

@@ -6,12 +6,12 @@ This is a docker image containing RedAmber created from
 ## Contents
 
 - From jupyter/minimal-notebook:
-  - Based on 2023-03-13 (295612d3ade4)
+  - Based on 2023-05-15 (513d0cb8a67c)
   - x86-64
   - Ubuntu-22.04
-  - python-3.10.9
-  - lab-3.6.1
-  - notebook-6.5.3
+  - python-3.10.11
+  - lab-3.6.3
+  - notebook-6.5.4
 - System ruby-dev:
   - Ruby 3.0.2
 - Arrow 11.0.0 for Ubuntu:
@@ -22,7 +22,7 @@ This is a docker image containing RedAmber created from
 - Locally installed iruby:
   - Using Ruby 3.0.2
 - Locally installed bundler and Gemfile:
-  - RedAmber 0.4.1
+  - RedAmber 0.5.0
   - Others (see Gemfile)
 
 ## Install

data/lib/red_amber/data_frame_displayable.rb

@@ -540,7 +540,7 @@ module RedAmber
             in ''
               '""'
             in String
-              element.sub(/^(\s+)$/, '"\1"') # blank spaces
+              element.sub(/\A(\s+)$/, '"\1"') # blank spaces
             in Float
               format('%g', element)
             in Integer

data/lib/red_amber/data_frame_loadsave.rb

@@ -44,7 +44,7 @@ module RedAmber
       #   BUFFER
       #
       # @example Load from a Buffer skipping comment line
-      #   DataFrame.load(Arrow::Buffer.new(<<~BUFFER), format: :csv, skip_lines: /^#/)
+      #   DataFrame.load(Arrow::Buffer.new(<<~BUFFER), format: :csv, skip_lines: /\A#/)
       #     # comment
       #     name,age
       #     Yasuko,68

data/lib/red_amber/data_frame_selectable.rb

@@ -39,7 +39,7 @@ module RedAmber
     #     penguins[:bill_length_mm]
     #
     #     # =>
-    #     #<RedAmber::Vector(:double, size=344):0x00000000000104dc>
+    #     #<RedAmber::Vector(:double, size=344, chunked):0x0000000000008f0c>
     #     [39.1, 39.5, 40.3, nil, 36.7, 39.3, 38.9, 39.2, 34.1, 42.0, 37.8, 37.8, 41.1, ... ]
     #
     # @overload [](keys)
@@ -173,7 +173,7 @@ module RedAmber
     #   penguins.v(:bill_length_mm)
     #
     #   # =>
-    #   #<RedAmber::Vector(:double, size=344):0x000000000000f140>
+    #   #<RedAmber::Vector(:double, size=344, chunked):0x0000000000008f0c>
     #   [39.1, 39.5, 40.3, nil, 36.7, 39.3, 38.9, 39.2, 34.1, 42.0, 37.8, 37.8, 41.1, ... ]
     #
     def v(key)

data/lib/red_amber/data_frame_variable_operation.rb

@@ -44,7 +44,7 @@ module RedAmber
     #     languages[:Language]
     #
     #     # =>
-    #     #<RedAmber::Vector(:string, size=4):0x000000000010359c>
+    #     #<RedAmber::Vector(:string, size=4, chunked):0x000000000010359c>
     #     ["Ruby", "Python", "R", "Rust"]
     #
     # @overload pick(booleans)
@@ -512,8 +512,8 @@ module RedAmber
     #     1 Rui           49          78 (nil)
     #     2 Hinata        28          57 Momotaro
     #
-    def assign(*assigner, &block)
-      assign_update(*assigner, append_to_left: false, &block)
+    def assign(...)
+      assign_update(false, ...)
     end
 
     # Assign new or updated variables (columns) and create an updated DataFrame.
@@ -583,13 +583,13 @@ module RedAmber
     #   @return [DataFrame]
     #     assigned DataFrame.
     #
-    def assign_left(*assigner, &block)
-      assign_update(*assigner, append_to_left: true, &block)
+    def assign_left(...)
+      assign_update(true, ...)
     end
 
     private
 
-    def assign_update(*assigner, append_to_left: false, &block)
+    def assign_update(append_to_left, *assigner, &block)
       if block
         assigner_from_block = instance_eval(&block)
         assigner =

data/lib/red_amber/group.rb

@@ -26,12 +26,7 @@ module RedAmber
       private
 
       # @!macro [attach] define_group_aggregation
-      #   @!method $1(*summary_keys)
-      #     Group aggregation function `$1`.
-      #     @param summary_keys [Array<Symbol, String>]
-      #       summary keys.
-      #     @return [DataFrame]
-      #       aggregated DataFrame
+      #   Returns aggregated DataFrame.
       #
       def define_group_aggregation(function)
         define_method(function) do |*summary_keys|
@@ -55,7 +50,7 @@ module RedAmber
     # @param group_keys [Array<Symbol, String>]
     #   keys for grouping.
     # @return [Group]
-    #   Group object.
+    #   Group object. It inspects grouped columns and its count.
     # @example
     #   Group.new(penguins, :species)
     #
@@ -79,13 +74,93 @@ module RedAmber
       @group = @dataframe.table.group(*@group_keys)
     end
 
-    define_group_aggregation(:count)
+    # @!macro group_aggregation
+    #   @param group_keys [Array<Symbol, String>]
+    #     keys for grouping.
+    #   @return [DataFrame]
+    #     aggregated DataFrame
+
+    # Whether all elements in each group evaluate to true.
+    #
+    # @!method all(*group_keys)
+    #   @macro group_aggregation
+    #   @example For boolean columns by default.
+    #     dataframe
+    #
+    #     # =>
+    #     #<RedAmber::DataFrame : 6 x 3 Vectors, 0x00000000000230dc>
+    #             x y        z
+    #       <uint8> <string> <boolean>
+    #     0       1 A        false
+    #     1       2 A        true
+    #     2       3 B        false
+    #     3       4 B        (nil)
+    #     4       5 B        true
+    #     5       6 C        false
+    #
+    #     dataframe.group(:y).all
+    #
+    #     # =>
+    #     #<RedAmber::DataFrame : 3 x 2 Vectors, 0x000000000000fc08>
+    #       y        all(z)
+    #       <string> <boolean>
+    #     0 A        false
+    #     1 B        false
+    #     2 C        false
+    #
+    define_group_aggregation :all
+
+    # Whether any elements in each group evaluate to true.
+    #
+    # @!method any(*group_keys)
+    #   @macro group_aggregation
+    #   @example For boolean columns by default.
+    #     dataframe.group(:y).any
+    #
+    #     # =>
+    #     #<RedAmber::DataFrame : 3 x 2 Vectors, 0x00000000000117ec>
+    #       y        any(z)
+    #       <string> <boolean>
+    #     0 A        true
+    #     1 B        true
+    #     2 C        false
+    #
+    define_group_aggregation :any
+
+    # Count the number of non-nil values in each group.
+    #   If counts are the same (and do not include NaN or nil),
+    #   columns for counts are unified.
+    #
+    # @!method max(*group_keys)
+    # @macro group_aggregation
+    # @example Show counts for each group.
+    #   dataframe.group(:y).count
+    #
+    #   # =>
+    #   #<RedAmber::DataFrame : 3 x 3 Vectors, 0x000000000011ea04>
+    #     y        count(x) count(z)
+    #     <string>  <int64>  <int64>
+    #   0 A               2        2
+    #   1 B               3        2
+    #   2 C               1        1
+    #
+    #   dataframe.group(:z).count
+    #   # same as dataframe.group(:z).count(:x, :y)
+    #
+    #   =>
+    #   #<RedAmber::DataFrame : 3 x 2 Vectors, 0x0000000000122834>
+    #     z           count
+    #     <boolean> <int64>
+    #   0 false           3
+    #   1 true            2
+    #   2 (nil)           1
+    #
+    define_group_aggregation :count
     alias_method :__count, :count
     private :__count
 
-    def count(*summary_keys)
-      df = __count(summary_keys)
-      # if counts are the same (and do not include NaN or nil), aggregate count columns.
+    def count(*group_keys)
+      df = __count(group_keys)
       if df.pick(@group_keys.size..).to_h.values.uniq.size == 1
         df.pick(0..@group_keys.size).rename { [keys[-1], :count] }
       else
@@ -93,19 +168,213 @@ module RedAmber
       end
     end
 
-    define_group_aggregation(:sum)
+    # Returns each record group size as a DataFrame.
+    #
+    # @return [DataFrame]
+    #   DataFrame consists of:
+    #   - Group key columns.
+    #   - Result columns by group aggregation.
+    # @example
+    #   penguins.group(:species).group_count
+    #
+    #   # =>
+    #   #<RedAmber::DataFrame : 3 x 2 Vectors, 0x0000000000003a70>
+    #     species   group_count
+    #     <string>      <uint8>
+    #   0 Adelie            152
+    #   1 Chinstrap          68
+    #   2 Gentoo            124
+    #
+    def group_count
+      DataFrame.create(group_table)
+    end
+    alias_method :count_all, :group_count
+
+    # Count the unique values in each group.
+    #
+    # @!method count_uniq(*group_keys)
+    # @macro group_aggregation
+    # @example Show counts for each group.
+    #   dataframe.group(:y).count_uniq
+    #
+    #   # =>
+    #   #<RedAmber::DataFrame : 3 x 3 Vectors, 0x000000000011ea04>
+    #     y        count_uniq(x)
+    #     <string>       <int64>
+    #   0 A                    2
+    #   1 B                    3
+    #   2 C                    1
+    #
+    define_group_aggregation :count_distinct
+    def count_uniq(*group_keys)
+      df = count_distinct(*group_keys)
+      df.rename do
+        keys_org = keys.select { _1.start_with?('count_distinct') }
+        keys_renamed = keys_org.map { _1.to_s.gsub('distinct', 'uniq') }
+        keys_org.zip keys_renamed
+      end
+    end
+
+    # Compute maximum of values in each group for numeric columns.
+    #
+    # @!method max(*group_keys)
+    #   @macro group_aggregation
+    #   @example
+    #     dataframe.group(:y).max
+    #
+    #     # =>
+    #     #<RedAmber::DataFrame : 3 x 2 Vectors, 0x000000000014ae74>
+    #       y         max(x)
+    #       <string> <uint8>
+    #     0 A              2
+    #     1 B              5
+    #     2 C              6
+    #
+    define_group_aggregation :max
+
+    # Compute mean of values in each group for numeric columns.
+    #
+    # @!method mean(*group_keys)
+    #   @macro group_aggregation
+    #   @example
+    #     dataframe.group(:y).mean
+    #
+    #     # =>
+    #     #<RedAmber::DataFrame : 3 x 2 Vectors, 0x0000000000138a8>
+    #       y         mean(x)
+    #       <string> <double>
+    #     0 A             1.5
+    #     1 B             4.0
+    #     2 C             6.0
+    #
+    define_group_aggregation :mean
+
+    # Compute median of values in each group for numeric columns.
+    #
+    # @!method median(*group_keys)
+    #   @macro group_aggregation
+    #   @example
+    #     dataframe.group(:y).median
+    #
+    #     # =>
+    #     #<RedAmber::DataFrame : 3 x 2 Vectors, 0x0000000000138a8>
+    #       y        median(x)
+    #       <string>  <double>
+    #     0 A              1.5
+    #     1 B              4.0
+    #     2 C              6.0
+    #
+    define_group_aggregation :approximate_median
+    def median(*group_keys)
+      df = approximate_median(*group_keys)
+      df.rename do
+        keys_org = keys.select { _1.start_with?('approximate_') }
+        keys_renamed = keys_org.map { _1.to_s.delete_prefix('approximate_') }
+        keys_org.zip keys_renamed
+      end
+    end
 
-    define_group_aggregation(:product)
+    # Compute minimum of values in each group for numeric columns.
+    #
+    # @!method min(*group_keys)
+    #   @macro group_aggregation
+    #   @example
+    #     dataframe.group(:y).min
+    #
+    #     # =>
+    #     #<RedAmber::DataFrame : 3 x 2 Vectors, 0x000000000018f38>
+    #       y         min(x)
+    #       <string> <uint8>
+    #     0 A              1
+    #     1 B              3
+    #     2 C              6
+    #
+    define_group_aggregation :min
 
-    define_group_aggregation(:mean)
+    # Get one value from each group.
+    #
+    # @!method one(*group_keys)
+    #   @macro group_aggregation
+    #   @example
+    #     dataframe.group(:y).one
+    #
+    #     # =>
+    #     #<RedAmber::DataFrame : 3 x 2 Vectors, 0x000000000002885c>
+    #       y         one(x)
+    #       <string> <uint8>
+    #     0 A              1
+    #     1 B              3
+    #     2 C              6
+    #
+    define_group_aggregation :one
 
-    define_group_aggregation(:min)
+    # Compute product of values in each group for numeric columns.
+    #
+    # @!method product(*group_keys)
+    #   @macro group_aggregation
+    #   @example
+    #     dataframe.group(:y).product
+    #
+    #     # =>
+    #     #<RedAmber::DataFrame : 3 x 2 Vectors, 0x000000000021a84>
+    #       y        product(x)
+    #       <string>   <uint64>
+    #     0 A                 2
+    #     1 B                60
+    #     2 C                 6
+    #
+    define_group_aggregation :product
 
-    define_group_aggregation(:max)
+    # Compute standard deviation of values in each group for numeric columns.
+    #
+    # @!method stddev(*group_keys)
+    #   @macro group_aggregation
+    #   @example
+    #     dataframe.group(:y).stddev
+    #
+    #     # =>
+    #     #<RedAmber::DataFrame : 3 x 2 Vectors, 0x00000000002be6c>
+    #       y        stddev(x)
+    #       <string>  <double>
+    #     0 A              0.5
+    #     1 B            0.082
+    #     2 C              0.0
+    #
+    define_group_aggregation :stddev
 
-    define_group_aggregation(:stddev)
+    # Compute sum of values in each group for numeric columns.
+    #
+    # @!method sum(*group_keys)
+    #   @macro group_aggregation
+    #   @example
+    #     dataframe.group(:y).sum
+    #
+    #     # =>
+    #     #<RedAmber::DataFrame : 3 x 2 Vectors, 0x000000000032a14>
+    #       y          sum(x)
+    #       <string> <uint64>
+    #     0 A               3
+    #     1 B              12
+    #     2 C               6
+    #
+    define_group_aggregation :sum
 
-    define_group_aggregation(:variance)
+    # Compute variance of values in each group for numeric columns.
+    #
+    # @!method variance(*group_keys)
+    #   @macro group_aggregation
+    #   @example
+    #     dataframe.group(:y).variance
+    #
+    #     # =>
+    #     #<RedAmber::DataFrame : 3 x 2 Vectors, 0x00000000003b1dc>
+    #       y        variance(x)
+    #       <string>    <double>
+    #     0 A               0.25
+    #     1 B              0.067
+    #     2 C                0.0
+    #
+    define_group_aggregation :variance
 
     # Returns Array of boolean filters to select each records in the Group.
     #
@@ -168,27 +437,6 @@ module RedAmber
       @filters.size
     end
 
-    # Returns each record group size as a DataFrame.
-    #
-    # @return [DataFrame]
-    #   DataFrame consists of:
-    #   - Group key columns.
-    #   - Result columns by group aggregation.
-    # @example
-    #   penguins.group(:species).group_count
-    #
-    #   # =>
-    #   #<RedAmber::DataFrame : 3 x 2 Vectors, 0x0000000000003a70>
-    #     species   group_count
-    #     <string>      <uint8>
-    #   0 Adelie            152
-    #   1 Chinstrap          68
-    #   2 Gentoo            124
-    #
-    def group_count
-      DataFrame.create(group_table)
-    end
-
     # String representation of self.
     #
     # @return [String]

data/lib/red_amber/subframes.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module RedAmber
-  # class SubFrames treats a set of subsets of a DataFrame
+  # class SubFrames treats subsets of a DataFrame
   # [Experimental feature] Class SubFrames may be removed or be changed in the future.
   class SubFrames
     include Enumerable # may change to use Forwardable.
@@ -434,7 +434,7 @@ module RedAmber
     #   @return [DataFrame]
     #     created DataFrame.
     #   @example Aggregate by key labels in arguments and values from block.
-    #     subframes.aggregate(:y, :sum_x) { [y.first, x.sum] }
+    #     subframes.aggregate(:y, :sum_x) { [y.one, x.sum] }
     #
     #     # =>
     #     #<RedAmber::DataFrame : 3 x 2 Vectors, 0x0000000000003b24>
@@ -445,7 +445,7 @@ module RedAmber
     #     2 C              6
     #
     #   @example Aggregate by key labels in an Array and values from block.
-    #     subframes.aggregate([:y, :sum_x]) { [y.first, x.sum] }
+    #     subframes.aggregate([:y, :sum_x]) { [y.one, x.sum] }
     #
     #     # =>
     #     #<RedAmber::DataFrame : 3 x 2 Vectors, 0x0000000000003b24>
@@ -457,7 +457,7 @@ module RedAmber
     #
     # @overload aggregate
     #
-    #   Aggregate SubFrames creating DataFrame with pairs of key and aggregated value
+    #   Aggregate SubFrames creating DataFrame with pairs of key and aggregated values
     #   in Hash from the block.
     #
     #   @yieldparam dataframe [DataFrame]
@@ -470,7 +470,7 @@ module RedAmber
     #     created DataFrame.
     #   @example Aggregate by key and value pairs from block.
     #     subframes.aggregate do
-    #       { y: y.first, sum_x: x.sum }
+    #       { y: y.one, sum_x: x.sum }
     #     end
     #
     #     # =>
@@ -712,7 +712,7 @@ module RedAmber
     #   @example
     #     subframes.assign(:sum_x, :frac_x) do
     #       group_sum = x.sum
-    #       [[group_sum] * size, x / s.to_f]
+    #       [[group_sum] * size, x / group_sum.to_f]
     #     end
     #
     #     # =>

data/lib/red_amber/vector.rb

@@ -180,7 +180,8 @@ module RedAmber
         end
         sio << ']'
 
-        format "#<#{self.class}(:#{type}, size=#{size}):0x%016x>\n%s\n",
+        chunked = chunked? ? ', chunked' : ''
+        format "#<#{self.class}(:#{type}, size=#{size}#{chunked}):0x%016x>\n%s\n",
                object_id, sio.string
       end
     end