red_amber 0.4.1 → 0.4.2

data/docker/notebook/red-amber.ipynb

@@ -0,0 +1,188 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# RedAmber Examples\n",
+    "\n",
+    "This notebook walks through the [README of RedAmber](https://github.com/heronshoes/red_amber#readme)."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## `RedAmber::DataFrame`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "require 'red_amber'\n",
+    "include RedAmber\n",
+    "require 'datasets-arrow'\n",
+    "\n",
+    "{RedAmber: VERSION, Datasets: Datasets::VERSION}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Example: diamonds dataset\n",
+    "\n",
+    "For the first loading of Datasets::Diamonds, it will take some time to download."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "dataset = Datasets::Diamonds.new\n",
+    "diamonds = DataFrame.new(dataset)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "df = diamonds\n",
+    "  .slice { carat > 1 } # or use #filter instead of #slice\n",
+    "  .group(:cut)\n",
+    "  .mean(:price) # `pick` prior to `group` is not required if `:price` is specified here.\n",
+    "  .sort('-mean(price)')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "usdjpy = 110.0 # when the yen was stronger\n",
+    "\n",
+    "df.rename('mean(price)': :mean_price_USD)\n",
+    "  .assign(:mean_price_JPY) { mean_price_USD * usdjpy }"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "## Example: starwars dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "uri = URI('https://vincentarelbundock.github.io/Rdatasets/csv/dplyr/starwars.csv')\n",
+    "\n",
+    "starwars = DataFrame.load(uri)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "starwars\n",
+    "  .drop(0) # delete unnecessary index column\n",
+    "  .remove { species == \"NA\" } # delete unnecessary rows\n",
+    "  .group(:species) { [count(:species), mean(:height, :mass)] }\n",
+    "  .slice { count > 1 } # or use #filter instead of slice"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## `RedAmber::Vector`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "penguins = DataFrame.new(Datasets::Penguins.new)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "penguins[:bill_length_mm]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "penguins[:bill_length_mm] < 40"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "penguins[:bill_length_mm].mean"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Ruby 3.0.2",
+   "language": "ruby",
+   "name": "ruby"
+  },
+  "language_info": {
+   "file_extension": ".rb",
+   "mimetype": "application/x-ruby",
+   "name": "ruby",
+   "version": "3.0.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

data/docker/readme.md

@@ -0,0 +1,118 @@
+# RedAmber Minimal Notebook
+
+This is a docker image containing RedAmber created from 
+[jupyter/minimal-notebook](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-minimal-notebook)
+
+## Contents
+
+- From jupyter/minimal-notebook:
+  - Based on 2023-03-13 (295612d3ade4)
+  - x86-64
+  - Ubuntu-22.04
+  - python-3.10.9
+  - lab-3.6.1
+  - notebook-6.5.3
+- System ruby-dev:
+  - Ruby 3.0.2
+- Arrow 11.0.0 for Ubuntu:
+  - libarrow-dev
+  - libarrow-glib-dev
+  - libparquet-dev
+  - libparquet-glib-dev
+- Locally installed iruby:
+  - Using Ruby 3.0.2
+- Locally installed bundler and Gemfile:
+  - RedAmber 0.4.1
+  - Others (see Gemfile)
+
+## Install
+
+```
+git clone https://github.com/heronshoes/red_amber.git
+cd  docker
+```
+
+Edit ENV variable in `.env` as you like.
+
+[note] NB_USER is fixed for `jovyan`, the common user name in Jupyter,
+can not change it in this version.
+
+If TZ is not used in your host system, define it here.
+Otherwise UTC is used in the container.
+
+TOKEN will be used for token-based authentication.
+
+```
+# Example
+TZ=Asia/Tokyo
+TOKEN='something'
+```
+
+Then build `red_amber-minimal-notebook` container. It will take a while.
+
+```
+docker-compose build
+```
+
+## Start Jupyter Lab
+
+After build, start the container. Adding `-d` option will detach it in background.
+
+```
+docker-compose up
+```
+
+You can access Jupyter Lab from `http://localhost:8888/` in your browser.
+
+- `red-amber.ipynb`:
+  - Walks through the [README of RedAmber](https://github.com/heronshoes/red_amber#readme).
+- `examples_of_red_amber.ipynb`:
+  - [Examples of RedAmber](https://github.com/heronshoes/red_amber/blob/main/docker/notebook/examples_of_red_amber.ipynb) in Notebook style.
+
+## Example in REPL
+
+You can try RedAmber in irb with pre-loaded datasets.
+
+Start `terminal` in Jupyter.
+
+For the first run,
+
+```
+source ~/.bashrc
+../example
+
+```
+
+It will take a while for the first run to fetch and prepare red-datasets cache.
+
+If irb starts you can see:
+
+```ruby
+
+    69: # Welcome to RedAmber example!
+    70: # This environment will offer these pre-loaded datasets:
+    71: #   penguins, diamonds, iris, starwars, simpsons_paradox_covid,
+    72: #   mtcars, band_members, band_instruments, band_instruments2
+    73: #   (original) import_cars, comecome, dataframe, subframes
+ => 74: binding.irb
+
+irb(main):001:0> 
+```
+
+RedAmber is already loaded in this environment with some datasets shown above.
+
+```ruby
+irb(main):002:0> dataframe
+=> 
+#<RedAmber::DataFrame : 6 x 3 Vectors, 0x0000000000003818>
+        x y        z
+  <uint8> <string> <boolean>
+0       1 A        false
+1       2 A        true
+2       3 B        false
+3       4 B        (nil)
+4       5 B        true
+5       6 C        false
+```
+
+Next time you start this environment, you can simply invoke as `../example`.

data/lib/red_amber/group.rb

@@ -60,11 +60,11 @@ module RedAmber
     #
     #   # =>
     #   #<RedAmber::Group : 0x000000000000f410>
-    #     species   group_count
-    #     <string>      <uint8>
-    #   0 Adelie            152
-    #   1 Chinstrap          68
-    #   2 Gentoo            124
+    #     species     count
+    #     <string>  <uint8>
+    #   0 Adelie        152
+    #   1 Chinstrap      68
+    #   2 Gentoo        124
     #
     def initialize(dataframe, *group_keys)
       @dataframe = dataframe
@@ -186,14 +186,14 @@ module RedAmber
     #
     #   # =>
     #   #<RedAmber::Group : 0x0000000000003a98>
-    #     species   group_count
-    #     <string>      <uint8>
-    #   0 Adelie            152
-    #   1 Chinstrap          68
-    #   2 Gentoo            124
+    #     species     count
+    #     <string>  <uint8>
+    #   0 Adelie        152
+    #   1 Chinstrap      68
+    #   2 Gentoo        124
     #
     def inspect
-      "#<#{self.class} : #{format('0x%016x', object_id)}>\n#{group_count}"
+      "#<#{self.class} : #{format('0x%016x', object_id)}>\n#{count(@group_keys)}"
     end
 
     # Summarize Group by aggregation functions from the block.
@@ -210,11 +210,11 @@ module RedAmber
     #
     #   # =>
     #   #<RedAmber::Group : 0x000000000000c314>
-    #     species   group_count
-    #     <string>      <uint8>
-    #   0 Adelie            152
-    #   1 Chinstrap          68
-    #   2 Gentoo            124
+    #     species     count
+    #     <string>  <uint8>
+    #   0 Adelie        152
+    #   1 Chinstrap      68
+    #   2 Gentoo        124
     #
     #   group.summarize { mean(:bill_length_mm) }
     #

data/lib/red_amber/subframes.rb

@@ -10,6 +10,38 @@ module RedAmber
     using RefineArray
     using RefineArrayLike
 
+    # Entity to select sub-dataframes
+    class Selectors
+      attr_reader :selectors, :size, :sizes
+
+      def initialize(selectors)
+        @selectors = selectors
+        @size = selectors.size
+        @sizes = []
+      end
+
+      def each
+        @selectors.each
+      end
+    end
+
+    # Boolean selectors of sub-dataframes
+    class Filters < Selectors
+      def sizes
+        # count true
+        @sizes = @selectors.map { |s| s.to_a.count { _1 } } # rubocop:disable Performance/Size
+      end
+    end
+
+    # Index selectors of sub-dataframes
+    class Indices < Selectors
+      def sizes
+        @sizes = @selectors.map(&:size)
+      end
+    end
+
+    private_constant :Selectors, :Filters, :Indices
+
     class << self
       # Create SubFrames from a Group.
       #
@@ -79,13 +111,8 @@ module RedAmber
       def by_indices(dataframe, subset_indices)
         instance = allocate
         instance.instance_variable_set(:@baseframe, dataframe)
-        enum =
-          Enumerator.new(subset_indices.size) do |y|
-            subset_indices.each do |i|
-              y.yield DataFrame.new_dataframe_with_schema(dataframe, dataframe.take(i))
-            end
-          end
-        instance.instance_variable_set(:@enum, enum)
+        instance.instance_variable_set(:@selectors, Indices.new(subset_indices))
+        instance.instance_variable_set(:@frames, [])
         instance
       end
 
@@ -105,13 +132,8 @@ module RedAmber
       def by_filters(dataframe, subset_filters)
         instance = allocate
         instance.instance_variable_set(:@baseframe, dataframe)
-        enum =
-          Enumerator.new(subset_filters.size) do |y|
-            subset_filters.each do |i|
-              y.yield DataFrame.new_dataframe_with_schema(dataframe, dataframe.filter(i))
-            end
-          end
-        instance.instance_variable_set(:@enum, enum)
+        instance.instance_variable_set(:@selectors, Filters.new(subset_filters))
+        instance.instance_variable_set(:@frames, [])
         instance
       end
 
@@ -130,18 +152,13 @@ module RedAmber
         case Array(dataframes)
         when [] || [nil]
           instance.instance_variable_set(:@baseframe, DataFrame.new)
+          instance.instance_variable_set(:@selectors, [])
           instance.instance_variable_set(:@frames, [])
-          enum = [].each
         else
-          enum =
-            Enumerator.new(dataframes.size) do |y|
-              dataframes.each do |i|
-                y.yield i
-              end
-            end
-          instance.instance_variable_set(:@baseframe, enum.lazy)
+          instance.instance_variable_set(:@baseframe, nil)
+          instance.instance_variable_set(:@selectors, nil)
+          instance.instance_variable_set(:@frames, dataframes)
         end
-        instance.instance_variable_set(:@enum, enum)
         instance
       end
 
@@ -261,40 +278,34 @@ module RedAmber
     #
     # @since 0.4.0
     #
-    def initialize(dataframe, subset_specifier = nil, &block)
+    def initialize(dataframe, selectors = nil, &block)
       unless dataframe.is_a?(DataFrame)
         raise SubFramesArgumentError, "not a DataFrame: #{dataframe}"
       end
 
       if block
-        unless subset_specifier.nil?
+        unless selectors.nil?
           raise SubFramesArgumentError, 'Must not specify both arguments and block.'
         end
 
-        subset_specifier = yield(dataframe)
+        selectors = yield(dataframe)
       end
 
-      if dataframe.empty? || subset_specifier.nil? || subset_specifier.empty?
+      if dataframe.empty? || selectors.nil? || selectors.empty?
         @baseframe = DataFrame.new
-        @frames = []
-        @enum = @frames.each
+        @selectors = Selectors.new([])
       else
-        @baseframe = nil
-        @enum =
-          Enumerator.new(subset_specifier.size) do |yielder|
-            subset_specifier.map do |i|
-              df =
-                if i.numeric?
-                  dataframe.take(i)
-                elsif i.boolean?
-                  dataframe.filter(i)
-                else
-                  raise SubFramesArgumentError, "illegal type: #{i}"
-                end
-              yielder.yield DataFrame.new_dataframe_with_schema(dataframe, df)
-            end
+        @baseframe = dataframe
+        @selectors =
+          if selectors[0].boolean?
+            Filters.new(selectors)
+          elsif selectors[0].numeric?
+            Indices.new(selectors)
+          else
+            raise SubFramesArgumentError, "illegal type: #{selectors}"
           end
       end
+      @frames = []
     end
 
     # Return concatenated SubFrames as a DataFrame.
@@ -305,11 +316,7 @@ module RedAmber
     # @since 0.4.0
     #
     def baseframe
-      if @baseframe.nil? || @baseframe.is_a?(Enumerator)
-        @baseframe = reduce(&:concatenate)
-      else
-        @baseframe
-      end
+      @baseframe ||= reduce(&:concatenate)
     end
     alias_method :concatenate, :baseframe
     alias_method :concat, :baseframe
@@ -384,7 +391,19 @@ module RedAmber
     def each(&block)
       return enum_for(__method__) { size } unless block
 
-      frames.each(&block)
+      if @selectors
+        @selectors.each.with_index do |selector, i|
+          if i < @frames.size
+            yield @frames[i]
+          else
+            frame = get_subframe(selector)
+            @frames << frame
+            yield frame
+          end
+        end
+      else
+        @frames.each(&block)
+      end
       nil
     end
 
@@ -916,6 +935,26 @@ module RedAmber
     #
     define_subframable_method :filter_map
 
+    # Return 0...num sub-dataframes in self.
+    #
+    # @param num [Integer, Float]
+    #   num of sub-dataframes to pick up. `num`` must be positive or zero.
+    # @return [SubFrames]
+    #   A new SubFrames.
+    #   If n == 0, it returns empty SubFrames.
+    #   If n >= size, it returns self.
+    # @since 0.4.2
+    #
+    def take(num)
+      if num.zero?
+        SubFrames.new(DataFrame.new, [])
+      elsif num >= size
+        self
+      else
+        SubFrames.by_dataframes(frames(num))
+      end
+    end
+
     # Number of subsets.
     #
     # @return [Integer]
@@ -923,7 +962,12 @@ module RedAmber
     # @since 0.4.0
     #
     def size
-      @size ||= @enum.size
+      @size ||=
+        if @selectors
+          @selectors.size
+        else
+          @frames.size
+        end
     end
 
     # Size list of subsets.
@@ -933,7 +977,12 @@ module RedAmber
     # @since 0.4.0
     #
     def sizes
-      @sizes ||= @enum.map(&:size)
+      @sizes ||=
+        if @selectors
+          @selectors.sizes
+        else
+          @frames.map(&:size)
+        end
     end
 
     # Indices at the top of each sub DataFrames.
@@ -945,10 +994,17 @@ module RedAmber
     # @since 0.4.0
     #
     def offset_indices
-      sum = 0
-      sizes.map do |size|
-        sum += size
-        sum - size
+      case @selectors
+      when Filters
+        @selectors.selectors.map do |selector|
+          selector.each.with_index.find { |x, _| x }[1]
+        end
+      else # Indices, nil
+        sum = 0
+        sizes.map do |size|
+          sum += size
+          sum - size
+        end
       end
     end
 
@@ -965,11 +1021,11 @@ module RedAmber
     # Test if self has only one subset and it is comprehensive.
     #
     # @return [true, false]
-    #   true if only member of self is equal to universal DataFrame.
+    #   true if the only member of self is equal to universal DataFrame.
     # @since 0.4.0
     #
     def universal?
-      size == 1 && @enum.first == baseframe
+      size == 1 && first == @baseframe
     end
 
     # Return string representation of self.
@@ -1012,7 +1068,7 @@ module RedAmber
     #
     # @since 0.4.0
     #
-    def to_s(limit: 16)
+    def to_s(limit: 5)
       _to_s(limit: limit)
     end
 
@@ -1064,10 +1120,10 @@ module RedAmber
     #
     # @since 0.4.0
     #
-    def inspect(limit: 16)
+    def inspect(limit: 5)
       shape =
-        if @baseframe.is_a?(Enumerator)
-          "Enumerator::Lazy:size=#{@baseframe.size}"
+        if @baseframe.nil?
+          '(Not prepared)'
         else
           baseframe.shape_str(with_id: true)
         end
@@ -1079,14 +1135,51 @@ module RedAmber
         "---\n#{_to_s(limit: limit, with_id: true)}"
     end
 
+    # Return an Array of sub DataFrames
+    #
+    # @overload frames
+    #   Returns all sub dataframes.
+    #
+    #   @return [Array<DataFrame>]
+    #     sub DataFrames.
+    #
+    # @overload frames(n_frames)
+    #   Returns partial sub dataframes.
+    #
+    #   @param n_frames [Integer]
+    #     num of dataframes to retrieve.
+    #   @return [Array<DataFrame>]
+    #     sub DataFrames.
+    #
+    # @since 0.4.2
+    #
+    def frames(n_frames = nil)
+      n_frames = size if n_frames.nil?
+
+      if @frames.size < n_frames
+        @frames = each.take(n_frames)
+      else
+        @frames.take(n_frames)
+      end
+    end
+
     private
 
-    def frames
-      @frames ||= @enum.to_a
+    # Get sub dataframe specified by 'selector'
+    def get_subframe(selector)
+      df =
+        case @selectors
+        when Filters
+          @baseframe.filter(selector)
+        when Indices
+          @baseframe.take(selector)
+        end
+      DataFrame.new_dataframe_with_schema(@baseframe, df)
     end
 
-    def _to_s(limit: 16, with_id: false)
-      a = take(limit).map do |df|
+    # Subcontractor of to_s
+    def _to_s(limit: 5, with_id: false)
+      a = each.take(limit).map do |df|
         if with_id
           "#<#{df.shape_str(with_id: with_id)}>\n" \
             "#{df.to_s(head: 2, tail: 2)}"