red_amber 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aa6f3c47b47df7271d7d150a800013c7c9d8bd75ca6066f54506c922f12eea09
-  data.tar.gz: 763f19f54a6508648fe9f1bdd0a11f678a86f554b58b71d7bed66aa5df7df2a7
+  metadata.gz: 264e7637475fd01946900335751a1592a3859e9bfa772ecc0800ab05c4d852f0
+  data.tar.gz: a57400445419698a66d6b5c94e15fa8c040f2f3930f9fbf75603ffb6e18bd9cf
 SHA512:
-  metadata.gz: 433ca52f7a62f055f327e0426426cfd86f563009e4ec4811d7cf8297152309081271b7b7625d39ffa31ecf455d352ee305d76b6d09e4d1dab0d90aa6c2bffb3e
-  data.tar.gz: 717d8618dd428d165c80420e7c35f1b7f870a059227a91bd5224f67b9cd3b8bdafcaed523fee170524738036cc9b43e914712fa01e88f7eb9ca1f0cc18c98dbf
+  metadata.gz: 0fdbcdb732e36bb866a8251800ab3fa1a714fa075234bf8cd516f2542ab6704ebfa429a7177da2bd8cd6fa6eb1158efb0d68f46f43d1dc088a9a0f0debdc5c54
+  data.tar.gz: f9c1dffaa157ecf34b0b4fec6c1d7972b4773bbf7a11101a345172d621753cd9fc3818753b329dd2906a506af294d6a96c0180a0fb4dc84c2b54bceef6b520f5

data/.rubocop.yml

@@ -44,17 +44,16 @@ Layout/LineLength:
 Layout/MultilineMethodCallIndentation:
   EnforcedStyle: indented_relative_to_receiver
 
-# avoid unused variable asignment
-Rubycw/Rubycw:
-  Exclude:
-    - 'test/**/*'
-
 # Disabled to define Vector operators
 # Offense count: 38
 Lint/BinaryOperatorWithIdenticalOperands:
   Exclude:
     - 'test/test_vector_binary_element_wise.rb'
 
+Lint/Debugger:
+  Exclude:
+    - 'bin/example'
+
 # Need for test with empty block
 # Offense count: 1
 # Configuration parameters: AllowComments, AllowEmptyLambdas.
@@ -87,6 +86,7 @@ Metrics/AbcSize:
     'drop', # 31.42
     '[]', # 33.76
     'split', # 37.35
+    'aggregate', # 38.13
   ]
 
 # Max: 25
@@ -110,6 +110,7 @@ Metrics/ClassLength:
     - 'lib/red_amber/group.rb' # 105
     - 'lib/red_amber/subframes.rb' # 110
     - 'lib/red_amber/vector.rb' # 152
+    - 'lib/red_amber/vector_binary_element_wise.rb' # 109
 
 # Only for monitoring. I will measure by PerceivedComplexity.
 # Max: 7
@@ -127,6 +128,8 @@ Metrics/CyclomaticComplexity:
     'parse_range', # 14
     'remove', # 14
     '[]', # 13
+    'drop', # 13
+    'aggregate', # 13
   ]
 
 # Max: 10
@@ -140,6 +143,7 @@ Metrics/MethodLength:
     'format_table', # 53
     'slice_by', # 38
     'assign_update', # 35
+    'drop', # 32
     'aggregate', # 31
   ]
 
@@ -187,6 +191,7 @@ Metrics/PerceivedComplexity:
     'filters', # 11
     'html_table', # 11
     'slice', # 11
+    'pick', # 11
   ]
 
 # Offense count: 1
@@ -210,6 +215,12 @@ Naming/PredicateName:
     - 'lib/red_amber/vector_functions.rb'
     - 'lib/red_amber/vector_selectable.rb'
 
+# avoid unused variable asignment
+Rubycw/Rubycw:
+  Exclude:
+    - 'test/**/*'
+    - 'bin/example'
+
 # Offense count: 16
 # This cop supports safe autocorrection (--autocorrect).
 Style/OperatorMethodCall:
@@ -223,6 +234,10 @@ Style/SlicingWithRange:
   Exclude:
     - 'test/test_data_frame_selectable.rb'
 
+Style/MixinUsage:
+  Exclude:
+    - 'bin/example'
+
 # Necessary to Vector < 0 element-wise comparison
 # Offense count: 5
 # This cop supports unsafe autocorrection (--autocorrect-all).

data/CHANGELOG.md

@@ -1,6 +1,70 @@
-## [0.4.0] - 2023-02-25
+## [0.4.1] - 2023-03-11
+
+- Breaking change
+  - Remove Vector.aggregate? method (#200)
+
+- Bug fixes
+  - Return self in DataFrame#drop when dropper is empty (reverts 746ac263) (#193)
+  - Return self in DataFrame#rename when renaming to same name (#193)
+  - Return self in DataFrame#pick when pick itself (#199)
+  - Fix column width for non-ascii elemnts in DataFrame#to_s (#193)
+    - This change uses String#width.
+  - Fix DataFrame#to_iruby when data is date32 type (#193)
+  - Fix DataFrame#shorthand to show temporal type data simply (#193)
+  - Fix Vector#rank when data is ChunkedArray (#198)
+  - Fix Vector element-wise functions with nil as scalar (#198)
+  - Support :force_order for all methods of join family (#199)
+    - Supports :force_order option to force sorting after join for all #join familiy. 
+    - This will valuable in some cases such as large dataframes.
+  - Ensure baseframe's schema for SubFrames (#200)
+
+- New features and improvements
+  - Add Vector#first, #last method (#198)
+    - This method will be used in SubFrames feature.
+  - Add Vector#modulo method (#198)
+    - The divmod function in Arrow C++ is still in draft state.
+      This method was created by combining existing functions
+  - Add Vector#quotient method (#198)
+  - Add aliases #div, #mod, #mul, #pow, #quo and #sub for Vector (#198)
+  - Add Vector#*_checked functions (#198)
+    - This functions will check numeric range overflow.
+  - Add 'tdra' and 'plain' in display mode (#193)
+    - The plain mode and default inspect will show up to 128 rows and 128 columns.
+  - Add String#width method in refinements (#193)
+    - This will be used to update DataFrame#to_s.
+  - Introduce pre-loaded REPL environment (#199)
+    - This commit will add bin/example and it will start irb environment
+      with enabled commonly used datasets such as penguins, diamonds, etc.
+  - Upgrade SubFrames#aggregate to accept block (#200)
+
+- Refactoring
+  - Use symbolized keys in refinements of Table#keys, #key? (#193)
+    - This can be treat Tables and DataFrames as same manner.
+  - Use key_name.succ in suffix of DataFrame#join (#193)
+    - This will make simple to get name candidate.
+  - Use ||= to memorize instance variables (#193)
+  - Refine vector projection to use #variables (#193)
+    - #variables is fastest when picking Vectors.
+  - Refine Vector#is_in to avoid #pack (#198)
+  - Refine Vector#index (#198)
+
+- Improve in tests/CI
+  - Tests
+    - Update benchmarks to test from older version (#193)
+    - Refine test of Vector function with scalar (#198)
+    - Refine test subframes and test_vector_selectable (#200)
+
+  - Cops
+  - CI
+
+- Documentation
+  - Update documents(small fix) (#201)
 
-:memo: Update documents for consistency
+- GitHub site
+
+- Thanks
+
+## [0.4.0] - 2023-02-25
 
 - Breaking change
   - Upgrade dependency to Arrow 11.0.0 (#188)
@@ -73,7 +137,8 @@
   - CI
     - Fix setting up Arrow by homebrew in CI (#167)
     - Fix CI error on homebrew deleting python link (#167)
-    - Set cache-version to get new C extensions in CI (#173) Thanks to @kou for suggestion.
+    - Set cache-version to get new C extensions in CI (#173)
+      - Thanks to @kou for suggestion.
 
 - Documentation
   - Update DataFrame.md about loading csv without headers (#165)

data/README.md

@@ -18,7 +18,7 @@ A simple dataframe library for Ruby.
 ## Requirements
 ### Ruby
 Supported Ruby version is >= 3.0 (since RedAmber 0.3.0).
-- I decided to remove Ruby 2.7 without waiting for EOL. See [Release note for v0.3.0](https://github.com/heronshoes/red_amber/discussions/162) for details.
+- I decided to remove support for Ruby 2.7 without waiting for its EOL. See [Release note for v0.3.0](https://github.com/heronshoes/red_amber/discussions/162) for details.
 
 ### Libraries
 ```ruby
@@ -29,7 +29,7 @@ gem 'rover-df',    '~> 0.3.0' # Optional, if you use IO from/to Rover::DataFrame
 
 ## Installation
 
-Install requirements before you install Red Amber.
+Install requirements before you install RedAmber.
 
 - Apache Arrow (~> 11.0.0)
 - Apache Arrow GLib (~> 11.0.0)
@@ -88,12 +88,12 @@ Also you can try the contents of this README interactively by [Binder](https://m
 Comparison of  basic features of RedAmber with Python 
 [pandas](https://pandas.pydata.org/),
 R [Tidyverse](https://www.tidyverse.org/) and
-Julia [Dataframes](https://dataframes.juliadata.org/stable/) is [here](doc/DataFrame_Comparison.md) (Thanks to Benson Muite).
+Julia [Dataframes](https://dataframes.juliadata.org/stable/) is in [DataFrame_Comparison.md](doc/DataFrame_Comparison.md) (Thanks to Benson Muite).
 
 ## Data frame in `RedAmber`
 
 Class `RedAmber::DataFrame` represents a set of data in 2D-shape.
-The entity is a Red Arrow's Table object. 
+Its entity is a Red Arrow's Table object. 
 
 ![dataframe model of RedAmber](https://raw.githubusercontent.com/heronshoes/red_amber/main/doc/image/dataframe_model.png)
 
@@ -115,7 +115,7 @@ then
 require 'datasets-arrow' # to load sample data
 
 dataset = Datasets::Diamonds.new
-diamonds = DataFrame.new(dataset) # from v0.2.2, should be `dataset.to_arrow` if older.
+diamonds = DataFrame.new(dataset) # before v0.2.3, should be `dataset.to_arrow`
 
 # =>
 #<RedAmber::DataFrame : 53940 x 10 Vectors, 0x000000000000f668>
@@ -174,7 +174,7 @@ df.rename('mean(price)': :mean_price_USD)
 
 ### Example: starwars dataset
 
-Next example is `starwars` dataset reading from the downloaded CSV file. Followed by minimum data cleansing.
+Next example is `starwars` dataset reading from the downloaded CSV file. Followed by minimum data cleaning.
 
 ```ruby
 uri = URI('https://vincentarelbundock.github.io/Rdatasets/csv/dplyr/starwars.csv')

data/benchmark/basic.yml

@@ -1,18 +1,18 @@
 loop_count: 3
 
 contexts:
-  - name: HEAD
-    prelude: |
-      $LOAD_PATH.unshift(File.expand_path('lib'))
-  - name: 0.3.0
+  - name: 0.1.5
     gems:
-      red_amber: 0.3.0
+      red_amber: 0.1.5
   - name: 0.2.0
     gems:
       red_amber: 0.2.0
-  - name: 0.1.5
+  - name: 0.3.0
     gems:
-      red_amber: 0.1.5
+      red_amber: 0.3.0
+  - name: HEAD
+    prelude: |
+      $LOAD_PATH.unshift(File.expand_path('lib'))
 
 prelude: |
   require 'red_amber'

data/benchmark/combine.yml

@@ -1,12 +1,12 @@
 loop_count: 3
 
 contexts:
-  - name: HEAD
-    prelude: |
-      $LOAD_PATH.unshift(File.expand_path('lib'))
   - name: 0.3.0
     gems:
       red_amber: 0.3.0
+  - name: HEAD
+    prelude: |
+      $LOAD_PATH.unshift(File.expand_path('lib'))
 
 prelude: |
   require 'red_amber'

data/benchmark/dataframe.yml

@@ -1,15 +1,15 @@
 loop_count: 3
 
 contexts:
-  - name: HEAD
-    prelude: |
-      $LOAD_PATH.unshift(File.expand_path('lib'))
-  - name: 0.3.0
-    gems:
-      red_amber: 0.3.0
   - name: 0.2.0
     gems:
       red_amber: 0.2.0
+  - name: 0.3.0
+    gems:
+      red_amber: 0.3.0
+  - name: HEAD
+    prelude: |
+      $LOAD_PATH.unshift(File.expand_path('lib'))
 
 prelude: |
   require 'red_amber'
@@ -19,8 +19,14 @@ prelude: |
 
   starwars = RedAmber::DataFrame.new(Datasets::Rdataset.new('dplyr', 'starwars').to_arrow)
 
-  uri = URI("https://raw.githubusercontent.com/heronshoes/red_amber/master/test/entity/import_cars.tsv")
-  import_cars = RedAmber::DataFrame.load(uri)
+  import_cars = RedAmber::DataFrame.load(Arrow::Buffer.new(<<~TSV), format: :tsv)
+    Year	Audi	BMW	BMW_MINI	Mercedes-Benz	VW
+    2017	28336	52527	25427	68221	49040
+    2018	26473	50982	25984	67554	51961
+    2019	24222	46814	23813	66553	46794
+    2020	22304	35712	20196	57041	36576
+    2021	22535	35905	18211	51722	35215
+  TSV
 
   ds = Datasets::Rdataset.new('openintro', 'simpsons_paradox_covid')
   simpsons_paradox_covid = RedAmber::DataFrame.new(ds.to_arrow)
@@ -43,7 +49,7 @@ benchmark:
       .group(:species) { [count(:species), mean(:height, :mass)] }
       .slice { v(:count) > 1 }
 
-  'D03: Inport cars test': |
+  'D03: Import cars test': |
     import_cars
       .to_long(:Year, name: :Manufacturer, value: :Num_of_imported)
       .to_wide(name: :Manufacturer, value: :Num_of_imported)

data/benchmark/group.yml

@@ -1,15 +1,15 @@
 loop_count: 3
 
 contexts:
-  - name: HEAD
-    prelude: |
-      $LOAD_PATH.unshift(File.expand_path('lib'))
-  - name: 0.3.0
-    gems:
-      red_amber: 0.3.0
   - name: 0.2.2
     gems:
       red_amber: 0.2.2
+  - name: 0.3.0
+    gems:
+      red_amber: 0.3.0
+  - name: HEAD
+    prelude: |
+      $LOAD_PATH.unshift(File.expand_path('lib'))
 
 prelude: |
   require 'red_amber'

data/benchmark/reshape.yml

@@ -1,15 +1,15 @@
 loop_count: 3
 
 contexts:
-  - name: HEAD
-    prelude: |
-      $LOAD_PATH.unshift(File.expand_path('lib'))
-  - name: 0.3.0
-    gems:
-      red_amber: 0.3.0
   - name: 0.2.2
     gems:
       red_amber: 0.2.2
+  - name: 0.3.0
+    gems:
+      red_amber: 0.3.0
+  - name: HEAD
+    prelude: |
+      $LOAD_PATH.unshift(File.expand_path('lib'))
 
 prelude: |
   require 'red_amber'

data/benchmark/vector.yml

@@ -1,15 +1,15 @@
 loop_count: 10
 
 contexts:
-  - name: HEAD
-    prelude: |
-      $LOAD_PATH.unshift(File.expand_path('lib'))
-  - name: 0.3.0
-    gems:
-      red_amber: 0.3.0
   - name: 0.2.0
     gems:
       red_amber: 0.2.0
+  - name: 0.3.0
+    gems:
+      red_amber: 0.3.0
+  - name: HEAD
+    prelude: |
+      $LOAD_PATH.unshift(File.expand_path('lib'))
 
 prelude: |
   require 'red_amber'

data/lib/red_amber/data_frame.rb

@@ -33,6 +33,23 @@ module RedAmber
         instance.instance_variable_set(:@table, table)
         instance
       end
+
+      # Return new DataFrame for specified schema and value.
+      #
+      # @param dataframe_for_schema [Dataframe]
+      #   schema of this dataframe will be used.
+      # @param dataframe_for_value [DataFrame]
+      #   column values of thes dataframe will be used.
+      # @return [DataFrame]
+      #   created DataFrame.
+      # @since 0.4.1
+      #
+      def new_dataframe_with_schema(dataframe_for_schema, dataframe_for_value)
+        DataFrame.create(
+          Arrow::Table.new(dataframe_for_schema.table.schema,
+                           dataframe_for_value.table.columns)
+        )
+      end
     end
 
     # Creates a new DataFrame.
@@ -194,7 +211,7 @@ module RedAmber
     #   `key => Vector` pairs for each columns.
     #
     def variables
-      @variables || @variables = init_instance_vars(:variables)
+      @variables ||= init_instance_vars(:variables)
     end
     alias_method :vars, :variables
 
@@ -204,7 +221,7 @@ module RedAmber
     #   keys in an Array.
     #
     def keys
-      @keys || @keys = init_instance_vars(:keys)
+      @keys ||= init_instance_vars(:keys)
     end
     alias_method :column_names, :keys
     alias_method :var_names, :keys
@@ -240,7 +257,7 @@ module RedAmber
     #   abbreviated Red Arrow data type names.
     #
     def types
-      @types || @types = @table.columns.map do |column|
+      @types ||= @table.columns.map do |column|
         column.data.value_type.nick.to_sym
       end
     end
@@ -251,7 +268,7 @@ module RedAmber
     #   an Array of Red Arrow data type Classes.
     #
     def type_classes
-      @data_types || @data_types = @table.columns.map { |column| column.data_type.class }
+      @type_classes ||= @table.columns.map { |column| column.data_type.class }
     end
 
     # Returns Vectors in an Array.
@@ -260,7 +277,7 @@ module RedAmber
     #   an Array of Vector.
     #
     def vectors
-      @vectors || @vectors = init_instance_vars(:vectors)
+      @vectors ||= init_instance_vars(:vectors)
     end
 
     # Returns column-oriented data in a Hash.
@@ -682,7 +699,7 @@ module RedAmber
 
     # Catch variable (column) key as method name.
     def method_missing(name, *args, &block)
-      return v(name) if args.empty? && key?(name)
+      return variables[name] if args.empty? && key?(name)
 
       super
     end
@@ -723,11 +740,9 @@ module RedAmber
     end
 
     def name_unnamed_keys
-      return unless @table.key?('')
+      return unless @table.key?(:'')
 
-      # We can't use #keys because it causes mismatch of @table and @keys
-      keys = @table.schema.fields.map { |f| f.name.to_sym }
-      unnamed = (:unnamed1..).find { |e| !keys.include?(e) }
+      unnamed = (:unnamed1..).find { |name| !@table.key?(name) }
       fields =
         @table.schema.fields.map do |field|
           if field.name.empty?