red_amber 0.4.2 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 498f580bc6fc79e36b199cc3ec62c38638e4df903e956a7a78947d9091748d0c
-  data.tar.gz: 24c3f25ff47b6bba0af26d1a1c77a80a56c2115e0bd76d26ba3f769f71d3557b
+  metadata.gz: b921353dbfcaf634a2e026f541caaf914125482c956fa05886f6a542f6ac2e35
+  data.tar.gz: 737fae720227e8e3ef36c2c3142bdb7096b051fd51363a2978079e766081d320
 SHA512:
-  metadata.gz: '085caa83703f4b9be0a3baae3318f0bcb606bbf264347ad3db431b29fb59c87e05b142ab1fd7e8a0af0a3d64b0b81055f1e85d82414f773d01456cf5a386df25'
-  data.tar.gz: 901e7eda6560eb2a9ab378d672b39d3ba08ae47012d8465ce541a465d744ce45d93a451ef2bff8fdf838fc1040b12facc58c87d95c6f6bedb89c5c138b95005c
+  metadata.gz: 8f45b7c45725b2da1d9459edaa3ae59a2355e771055c281eb612356703b727315b64ed6963ce02493e6c8ef94194e848fc19b1cf4bbd0620b44ba2e24c1953b0
+  data.tar.gz: ca9f134d488168f0d2396f55b118fa7db78d2f6969286896ba2139d891a8a09f8a7fde69141c2c226212748c16f93ece4737f65465f9ea0616e500177806c314

data/.devcontainer/Dockerfile

@@ -0,0 +1,75 @@
+# [Choice] debian-11, debian-10, ubuntu-22.04, ubuntu-20.04, ubuntu-18.04
+ARG VARIANT=ubuntu-22.04
+
+FROM mcr.microsoft.com/devcontainers/base:${VARIANT}
+
+# Set env for tracking that we're running in a devcontainer
+ENV DEVCONTAINER=true
+
+RUN set -e; \
+    apt-get update; \
+    apt-get install -y \
+    # To build Ruby
+    autoconf \
+    bison \
+    rustc \
+    libssl-dev \
+    libyaml-dev \
+    libreadline6-dev \
+    zlib1g-dev \
+    libgmp-dev \
+    libncurses5-dev \
+    libffi-dev \
+    libgdbm6 \
+    libgdbm-dev \
+    libdb-dev \
+    uuid-dev \
+    # To install IRuby
+    libczmq-dev \
+    libzmq3-dev
+
+# Install Apache Arrow
+ARG APACHE_ARROW_VERSION=12.0.1-1
+ARG arrow_deb_tmp=/tmp/apache-arrow-apt-source-latest.deb
+ARG arrow_apt_source=https://apache.jfrog.io/artifactory/arrow/ubuntu/pool/jammy/main/a/apache-arrow-apt-source/apache-arrow-apt-source_${APACHE_ARROW_VERSION}_all.deb
+RUN set -e; \
+    apt-get update; \
+    curl -sfSL -o ${arrow_deb_tmp} ${arrow_apt_source}; \
+    apt-get install -y --no-install-recommends ${arrow_deb_tmp}; \
+    rm -f ${arrow_deb_tmp}; \
+    apt-get update; \
+    apt-get install -y \
+    libarrow-dev \
+    libarrow-glib-dev \
+    libarrow-dataset-dev \
+    libarrow-flight-dev \
+    libparquet-dev \
+    libparquet-glib-dev \
+    libgandiva-dev \
+    libgandiva-glib-dev
+
+# This Dockerfile adds a non-root user with sudo access. Use the "remoteUser"
+ARG USERNAME=vscode
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+
+USER $USERNAME
+
+# Install rbenv
+ARG RBENV_RUBY=3.2.2
+RUN set -e; \
+    git clone https://github.com/rbenv/rbenv.git $HOME/.rbenv; \
+    echo 'eval "$($HOME/.rbenv/bin/rbenv init -)"' >> $HOME/.profile; \
+    echo 'eval "$($HOME/.rbenv/bin/rbenv init -)"' >> $HOME/.bashrc; \
+    git clone https://github.com/rbenv/ruby-build.git $HOME/.rbenv/plugins/ruby-build
+
+# Install Ruby
+# Append `RUBY_CONFIGURE_OPTS=--disable-install-doc ` before rbenv to disable documents
+RUN set -e; \
+    $HOME/.rbenv/bin/rbenv install --verbose $RBENV_RUBY; \
+    $HOME/.rbenv/bin/rbenv global $RBENV_RUBY
+
+# Install IRuby
+RUN set -e; \
+    $HOME/.rbenv/bin/rbenv exec gem install iruby; \
+    $HOME/.rbenv/bin/rbenv exec iruby register --force

data/.devcontainer/devcontainer.json

@@ -0,0 +1,38 @@
+{
+    "name": "RedAmber",
+    "build": {
+        "dockerfile": "Dockerfile",
+        "cacheFrom": "ghcr.io/red-data-tools/red-amber"
+    },
+    "features": {
+        "ghcr.io/devcontainers/features/python:1": {
+            "installTools": true,
+            "installJupyterlab": true,
+            "version": "3.11"
+        },
+        // We don't use Ruby feature here
+        // "ghcr.io/devcontainers/features/ruby:1": {},
+        "ghcr.io/rocker-org/devcontainer-features/quarto-cli:1": {
+            "installTinyTex": true,
+            "version": "latest"
+        },
+        "ghcr.io/devcontainers/features/github-cli:1": {}
+    },
+    // VS Code extentions for Ruby
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                "rebornix.Ruby",
+                "shopify.ruby-lsp"
+            ]
+        }
+    },
+    // Use init process to deal with zombie process
+    "init": true,
+    // set TZ from local machine's environment defaulting to 'UTC' if not supplied.
+    "containerEnv": {
+        "RUBYLIB": "/workspaces/red_amber/lib",
+        "TZ": "${localEnv:TZ:UTC}"
+    },
+    "onCreateCommand": ".devcontainer/onCreateCommand.sh"
+}

data/.devcontainer/onCreateCommand.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -e
+
+# Install language and set timezone
+# You should change here if you use another
+sudo apt-get update
+sudo apt-get install -y language-pack-ja
+
+echo 'export LANG=ja_JP.UTF-8' >> $HOME/.bashrc
+echo 'export LANG=ja_JP.UTF-8' >> $HOME/.profile
+echo 'export TZ=Asia/Tokyo' >> $HOME/.bashrc
+echo 'export TZ=Asia/Tokyo' >> $HOME/.profile
+
+# Install HaranoAjiFonts
+mkdir -p $HOME/.fonts
+git clone https://github.com/trueroad/HaranoAjiFonts.git $HOME/.fonts/HaranoAjiFonts
+
+# Install gems
+bundle install
+
+# Create Jupyter Notebooks
+rake quarto:convert

data/.rubocop.yml

@@ -52,7 +52,7 @@ Lint/BinaryOperatorWithIdenticalOperands:
 
 Lint/Debugger:
   Exclude:
-    - 'docker/example'
+    - 'bin/example'
 
 # Need for test with empty block
 # Offense count: 1
@@ -76,7 +76,8 @@ Metrics/AbcSize:
   Max: 30
   CountRepeatedAttributes: false
   AllowedMethods: [
-    'join', # 51.87
+    'join_merge_keys', # 54.18
+    'join', # 53.1
     'dataframe_info', # 46.5
     'format_table', # 84.62
     'to_long', # 33.66
@@ -87,6 +88,9 @@ Metrics/AbcSize:
     '[]', # 33.76
     'split', # 37.35
     'aggregate', # 38.13
+    'filters', # 33.91
+    'merge_keys', # 32.17
+    'rename_keys', # 31.64
   ]
 
 # Max: 25
@@ -139,10 +143,12 @@ Metrics/MethodLength:
   Max: 30
   AllowedMethods: [
     'join', # 47
-    'dataframe_info', # 33
+    'join_merge_keys', # 41
     'format_table', # 53
     'slice_by', # 38
     'assign_update', # 35
+    'summarize', # 35
+    'dataframe_info', # 33
     'drop', # 32
     'aggregate', # 31
   ]
@@ -219,7 +225,7 @@ Naming/PredicateName:
 Rubycw/Rubycw:
   Exclude:
     - 'test/**/*'
-    - 'docker/example'
+    - 'bin/example'
 
 # Offense count: 16
 # This cop supports safe autocorrection (--autocorrect).
@@ -236,7 +242,7 @@ Style/SlicingWithRange:
 
 Style/MixinUsage:
   Exclude:
-    - 'docker/example'
+    - 'bin/example'
 
 # Necessary to Vector < 0 element-wise comparison
 # Offense count: 5

data/CHANGELOG.md

@@ -1,10 +1,134 @@
+## [0.5.1] - 2023-08-18
+
+Docker environment is replaced by Dev Container,
+and Jupyter Notebooks will be created from qmd files.
+
+- Breaking change
+
+- Bug fixes
+  - Fix timestamp test to set TZ locally (#249)
+  - Fix regexp for beginning of String (#251)
+  - Fix loading bin/Gemfile locally in bin/jupyter script (#261)
+
+- New features and improvements
+  - Support sort and null_placement options in Vector#rank (#265)
+  - Add Vector#find_substring method (#270)
+  - Add Group#one method (#274)
+  - Add Group#all and #any method (#274)
+  - Add Group#median method (#274)
+  - Add Group#count_uniq method (#274)
+  - Introduce Dev Container environment
+    - Introduce Devcontainer environment (#253)
+    - Change lifecycle script from postCreate to onCreate (#253)
+    - Move example to bin (#253)
+    - Fix Python and Ruby versions in Dev Container (#254)
+    - Add locale and timezone settings (#256)
+    - Add quarto from devcontainer feature (#259)
+    - Install HaranoAjiFonts as default Tex font (#259)
+
+- Refactoring
+  - Rename boolean methods in VectorStringFunction (#263)
+  - Refine Vector#inspect to show wheather chunked or not (#267)
+  - Add an alias Group#count_all for #group_count (#274)
+
+- Improve in tests/CI
+  - Create rake commands for Notebook convert/test (#269)
+  - Fix rubocop warning of forwarding arguments in assign_update (#269)
+  - Use rake to start example script (#269)
+  - Add test in Vector#rank to cover illegal rank option error (#271)
+  - Add bundle install to Rakefile (#276)
+  - Use Dockerfile to create dev container (#276)
+  - Save image to ghcr in ci (#276)
+
+- Documentation and Example
+  - YARD
+    - Update Docker Environment (#245)
+    - Refine jupyter notebook environment (#253)
+    - Refine yard in Group aggregations (#274)
+    - Fix yard of Vector#rank (#269)
+    - Fix yard of Group (#269)
+  - Notebook
+    - Start source management for jupyter notebook by qmd (#259)
+    - Don't create ipynb if it exists (#261)
+    - Add Group methods (125 in total) (#269)
+    - Add ArrowFunction (126 in total) (#269)
+    - Add DataFrame#auto_cast (127 in total) (#269)
+    - Update required version in examples notebook (#269)
+    - Update examples_of_red_amber (#269)
+    - Update red-amber.qmd (#269)
+
+- GitHub site
+  - Fix broken link in README/README.ja by Viktorius Suwandi (#262)
+  - Change description in gemspec (#254)
+  - Add documents for Dev Container (#254)
+
+- Thanks
+  - Viktorius Suwandi
+
+## [0.5.0] - 2023-05-24
+
+- Breaking change
+  - Use non keyword argument in #sub_by_value (#219)
+  - Upgrade dependency to Arrow 12.0.0 (#238)
+    - right_join will output columns as same order as Red Arrow.
+    - DataFrame#join will not force ordering of original column by default
+    - Join with type, such as full_join, sort after join by default
+
+- Bug fixes
+  - Use truncate in Vector#sample(float) (#229)
+  - Support options in DataFrame#tdra (#231)
+  - Fix printing table with non-ascii strings (#233)
+  - Fix join for Arrow 12.0.0
+
+- New features and improvements
+  - Add a singleton method Vector.[] (#218)
+  - Add an alias #sub_group (#219)
+  - Accept Group#summarize{Hash} to rename aggregated columns (#219)
+  - Add Group#group_frame (#219)
+  - Add Vector#cast (#224)
+  - Add Vector#fill_nil(value) (#226)
+  - Add Vector#one (#227)
+  - Add Vector#mode (#228)
+  - Add DataFrame#propagate (#235)
+  - Add DataFrame#sample (#237)
+  - Add DataFrame#shuffle (#237)
+  - Support RankOptions in Vector#rank (#239)
+  - Introduce MatchSubstringOptions family in Vector (#241)
+    - Introduce Vector#match_substring?
+    - Add Vector#end_with?, #start_with? method
+    - Add Vector#match_like?
+    - Add Vector#count_substring method
+
+- Refactoring
+  - Refine Group and SubFrames function (#219)
+    - Refine Group#group_count
+    - Use Acero in Group#filters
+    - Refine Group#filters, not using Acero
+    - Refine Group#summarize(array)
+  - Use Acero for renaming columns in join (#238)
+  - Use index kernel with IndexOptions introduced in 12.0.0 (#240)
+
+- Improve in tests/CI
+  - Use Fedra 39 Rawhide in CI (#238)
+
+- Documentation and Example
+  - Add missing yard documents for SubFrames::Selectors (#219)
+  - Update docker/example (#219)
+  - Update Gemfile in docker (#219)
+  - Add README.ja.md (#242)
+
+- GitHub site
+  - Update link of Red Data Tools Chat to matrix (#242)
+
+- Thanks
+
 ## [0.4.2] - 2023-04-02
 
 - Breaking change
 
 - Bug fixes
  - Fix Vector#modulo, #fdiv, #remainder (#203)
- 
+
 - New features and improvements
   - Update SubFrames#take to return SubFrames (#212)
 
@@ -49,7 +173,7 @@
   - Fix Vector#rank when data is ChunkedArray (#198)
   - Fix Vector element-wise functions with nil as scalar (#198)
   - Support :force_order for all methods of join family (#199)
-    - Supports :force_order option to force sorting after join for all #join familiy. 
+    - Supports :force_order option to force sorting after join for all #join familiy.
     - This will valuable in some cases such as large dataframes.
   - Ensure baseframe's schema for SubFrames (#200)
 
@@ -451,11 +575,11 @@
       - Move binder support to heronshoes/docker-stacks repository.
       - Update README notebook on binder.
       - Add examples_of_RedAmber notebook on binder.
-    
+
     - Start to use discussions.
 
 - Thanks
-  
+
   - Kenta Murata
 
 ## [0.2.1] - 2022-09-07
@@ -504,7 +628,7 @@
   - Update Jupyter Notebook `71 examples of RedAmber`
 
 - Thanks
-  
+
   - Kenta Murata
 
 ## [0.2.0] - 2022-08-15
@@ -519,7 +643,7 @@
   - Remove optional `require` for rover (#55)
     Fix DataFrame.new for argument with Rover::DataFrame.
   - Fix occasional failure in CI (#59)
-    Sometimes the CI test fails. I added -dev dependency 
+    Sometimes the CI test fails. I added -dev dependency
     in Arrow install by apt, not doing in bundler.
 
   - Fix calling :take in V#[] (#56)
@@ -537,7 +661,7 @@
     - Upgrade to Arrow 9.0.0 (#59)
     - Add Vector#quantile method (#59)
       Arrow::QuantileOptions has supported in Arrow GLib 9.0.0 (ARROW-16623, Thanks!)
-  
+
     - Add Vector#quantiles (#62)
 
     - Add DataFrame#each_row (#56)
@@ -548,7 +672,7 @@
     - Refine DataFrame.new to use pattern match
     - Use pattern match in DataFrame#assign
     - Use pattern match in DataFrame#rename
-  
+
   - Accept Array for renamer/assigner in #rename/#assign (#61)
     - Accept assigner by Arrays in DataFrame#assign
     - Accept renamer pairs by Arrays in DataFrame#rename
@@ -563,15 +687,15 @@
     - Intorduce DataFrame#to_wide method
 
   - Others
- 
+
     - Add alias sort_index for array_sort_indices (#59)
     - Enable :width option in DataFrame#to_s (#62)
     - Add options to DataFrame#format_table (#62)
 
   - Update Documents
-  
+
     - Add Yard doc for some methods
-  
+
     - Update Jupyter notebook '61 Examples of Red Amber' (#65)
 
 ## [0.1.8] - 2022-08-04 (experimental)
@@ -634,7 +758,7 @@
       - Show nils.
       - Show data types.
     - Refine documents to use new formatter output
-  
+
   - Simplify options of Vector functions (#46)
     Vector functions with options use optional argument opt in previous code.
 
@@ -646,7 +770,7 @@
     - Add methods to Group
 
   - Move parquet and rover to development dependency (#49)
-  
+
   - Refine text in `DataFrame#to_iruby` (#40)
 
   - Add badges in Github site
@@ -743,7 +867,7 @@
     - Add gem and status badges in README. (#42) [Patch by kojix2]
 
 - Thanks
-  
+
   - kojix2
 
 ## [0.1.5] - 2022-06-12 (experimental)
@@ -784,7 +908,7 @@
     - Change to use DataFrame#map_indices in #[]
 
   - Add rounding functions with opts (#21)
-    -  With options :mode and :n_digits 
+    -  With options :mode and :n_digits
     -  :n_digits also can be specified with :multiple option in `Vector#round_to_multiple`
     - `Vector#round`
     - `Vector#ceil`
@@ -852,7 +976,7 @@
     - Add example about TDR (#4)
     - Separate README to create DataFrame and Vector documents (#12)
     - Add DataFrame model concept image to README (#12)
-  
+
   - GitHub site
     - Switched to use merge on GitHub (not to push merged master) (#1)
     - Create lifetime issue #3 to show the goal of this project (#3)
@@ -877,7 +1001,7 @@
 
   - `Vector`
     - Add categorization functions
-      
+
       This is an important step to support `slice` method and NA treatment features.
       -  `is_finite`
       -  `is_inf`

data/Gemfile

@@ -7,7 +7,7 @@ gemspec
 group :test do
   gem 'rake'
 
-  gem 'red-parquet', '~> 11.0.0'
+  gem 'red-parquet', '~> 12.0.0'
   gem 'rover-df', '~> 0.3.0'
 
   gem 'rubocop'
@@ -15,14 +15,13 @@ group :test do
   gem 'rubocop-rake'
   gem 'rubocop-rubycw', require: false
 
-  gem 'iruby'
-  gem 'test-unit'
-  gem 'webrick'
-  gem 'yard'
-
   gem 'benchmark_driver'
+  gem 'iruby'
   gem 'red-arrow-numo-narray'
   gem 'red-datasets-arrow'
   gem 'simplecov'
   gem 'simplecov-json'
+  gem 'test-unit'
+  gem 'webrick'
+  gem 'yard'
 end