red_amber 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 00ba2e99b2b1d6f977b2e2e5c7d60b9313972cf3e831918606e5388d51442137
+  data.tar.gz: f0fc831937bff5fede4ee0f0537b0ef5fdfb8a1faa8a57082a197a627562252c
+SHA512:
+  metadata.gz: 7bc020b8663c3523426461e3bd54642d4eb85a86296a8db3f5d94315091ee4475ec8b910fb87165c5d029e35fa9dc45f119bea6278e023d3cc63ad011388fbfb
+  data.tar.gz: 78dd55182b40ee9bec769efdbcac23adb85ad93bbafe3f74c4ded9d56ab40e39da0ce1e34a841d5705e6b94fea85312057e360140965fa217243eede0d238eb5

data/.rubocop.yml

@@ -0,0 +1,89 @@
+# We will use cops to detect bugs in an early stage
+# Feel free to use .rubocop_todo.yml by --auto-gen-config
+inherit_from: .rubocop_todo.yml
+
+require:
+  - rubocop-performance
+  - rubocop-rubycw
+  - rubocop-rake
+
+AllCops:
+  # drop support for < 2.7
+  TargetRubyVersion: 2.7
+  # accept new cops if any
+  NewCops: enable
+
+# ===
+
+# alias is hard to see separately 
+Style/Alias:
+  EnforcedStyle: prefer_alias_method
+
+# For consistency and portability
+Style/TrailingCommaInArrayLiteral:
+  EnforcedStyleForMultiline: comma
+Style/TrailingCommaInHashLiteral:
+  EnforcedStyleForMultiline: comma
+# Should not set for Style/TrailingCommaInArguments
+
+# ===
+
+# To let you know the possibility of refactoring ===
+#
+# avoid unused variable asignment
+Rubycw/Rubycw:
+  Exclude:
+    - 'test/**/*'
+Lint/UselessAssignment:
+  Exclude:
+    - 'test/**/*'
+
+# Disabled to define Vector operators
+Lint/BinaryOperatorWithIdenticalOperands:
+  Exclude:
+    - 'test/test_vector_function.rb'
+
+# Max: 120
+Layout/LineLength:
+  Max: 100
+  Exclude:
+    - 'test/**/*'
+
+# <= 17 satisfactory
+# 18..30 unsatisfactory
+# > 30 dangerous
+Metrics/AbcSize:
+  Max: 19
+  Exclude:
+    - 'lib/red_amber/data_frame_output.rb' # Max: 78
+
+# Max: 25
+Metrics/BlockLength:
+  Max: 25
+  Exclude:
+    - 'test/**/*'
+    - '*.gemspec'
+
+# Max: 100
+Metrics/ClassLength:
+  Max: 100
+  Exclude:
+    - 'test/**/*'
+
+# Max: 7
+Metrics/CyclomaticComplexity:
+  Max: 10
+  Exclude:
+    - 'lib/red_amber/data_frame_output.rb' # Max: 11
+
+# Max: 10
+Metrics/MethodLength:
+  Max: 18
+  Exclude:
+    - 'lib/red_amber/data_frame_output.rb' # Max: 35
+
+# Max: 8
+Metrics/PerceivedComplexity:
+  Max: 9
+  Exclude:
+    - 'lib/red_amber/data_frame_output.rb' # Max: 12

data/.rubocop_todo.yml

@@ -0,0 +1,21 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2022-04-27 00:29:57 UTC using RuboCop version 1.27.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+# This cop supports unsafe auto-correction (--auto-correct-all).
+Style/SlicingWithRange:
+  Exclude:
+    - 'lib/red_amber/data_frame_selectable.rb'
+
+# Offense count: 1
+# This cop supports unsafe auto-correction (--auto-correct-all).
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: forbid_for_all_comparison_operators, forbid_for_equality_operators_only, require_for_all_comparison_operators, require_for_equality_operators_only
+Style/YodaCondition:
+  Exclude:
+    - 'lib/red_amber/data_frame.rb'

data/CHANGELOG.md

@@ -0,0 +1,34 @@
+## [0.1.2] - Unreleased
+
+- Add support for Arrow 8.0.0
+- `DataFrame`
+  - Introduce updating
+  - Introduce NA support
+  - Add slice method
+- `Vector`
+  - Add NaN support for functions
+  - More functions
+
+## [0.1.1] - 2022-05-06 (experimental)
+
+- Release on rubygem.org
+- Introduce class `DataFrame`
+  -  New from Hash, schema/rows, `Arrow::Table`, `Rover::DataFrame`
+  -  Load from file, string, URI
+  -  Save to file, string, URI
+  -  Methods for basic properties
+  -  Rich inspect method
+  -  Basic selecting by `#[]`
+- Introduce class `Vector`
+  -  New from a column in a `DataFlame`
+  -  New from `Arrow::Array`, `Arrow::ChunkedArray`, `Array`
+  -  Methods for basic properties
+  -  Function support
+     -  Unary aggregations
+     -  Unary element-wises
+     -  Binary element-wises
+     -  Some operators defined
+
+## [0.1.0] - 2022-04-15 (unreleased)
+
+- Initial version

data/Gemfile

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gemspec
+
+group :test do
+  gem 'rake'
+
+  gem 'rubocop'
+  gem 'rubocop-performance', require: false
+  gem 'rubocop-rake'
+  gem 'rubocop-rubycw', require: false
+
+  gem 'test-unit'
+  gem 'webrick'
+end

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2022 Hirokazu SUZUKI (heronshoes)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,336 @@
+# RedAmber
+
+A simple dataframe library for Ruby (experimental)
+
+- Powered by [Red Arrow](https://github.com/apache/arrow/tree/master/ruby/red-arrow)
+- Simple API similar to [Rover-df](https://github.com/ankane/rover)
+
+## Requirements
+
+```ruby
+gem 'red-arrow',   '~> 7.0.0'
+gem 'red-parquet', '~> 7.0.0' # if you use IO from/to parquet
+gem 'rover-df',    '~> 0.3.0' # if you use IO from/to Rover::DataFrame
+```
+
+## Installation
+
+Add this line to your Gemfile:
+
+```ruby
+gem 'red_amber'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install red_amber
+
+## `RedAmber::DataFrame`
+
+### Constructors and saving
+
+- [x] `new` from a columnar Hash
+  - `RedAmber::DataFrame.new(x: [1, 2, 3])`
+
+- [x] `new` from a schema (by Hash) and rows (by Array)
+  - `RedAmber::DataFrame.new({:x=>:uint8}, [[1], [2], [3]])`
+
+- [x] `new` from an Arrow::Table
+  - `RedAmber::DataFrame.new(Arrow::Table.new(x: [1, 2, 3]))`
+
+- [x] `new` from a Rover::DataFrame
+  - `RedAmber::DataFrame.new(Rover::DataFrame.new(x: [1, 2, 3]))`
+
+- [ ] `load` (class method)
+
+     - [x] from a [`.arrow`, `.arrows`, `.csv`, `.csv.gz`, `.tsv`] file
+       - `RedAmber::DataFrame.load("test/entity/with_header.csv")`
+
+     - [x] from a string buffer
+
+     - [x] from a URI
+       - `RedAmber::DataFrame.load(URI("https://github.com/heronshoes/red_amber/blob/master/test/entity/with_header.csv"))`
+
+     - [ ] from a parquet file
+
+- [ ] `save` (instance method)
+
+     - [x] to a [`.arrow`, `.arrows`, `.csv`, `.csv.gz`, `.tsv`] file
+
+     - [x] to a string buffer
+
+     - [x] to a URI
+
+     - [ ] to a parquet file
+
+### Properties
+
+- [x] `table`
+
+  Reader of Arrow::Table object inside.
+
+- [x] `n_rows`, `nrow`, `size`, `length`
+  
+  Returns num of rows (data size).
+ 
+- [x] `n_columns`, `ncol`, `width`
+  
+  Returns num of columns (num of vectors).
+ 
+- [x] `shape`
+ 
+  Returns shape in an Array[n_rows, n_cols].
+ 
+- [x] `column_names`, `keys`
+  
+  Returns num of column names by an Array.
+
+- [x] `types(class_name: false)`
+  
+  Returns types of columns by an Array.
+  If `class_name: true` returns an Array of `Arrow::DataType`.
+
+- [x] `vectors`
+
+  Returns an Array of Vectors.
+
+- [x] `to_h`
+
+  Returns column-oriented data in a Hash.
+
+- [x] `to_a`, `raw_records`
+
+  Returns an array of row-oriented data without header. If you need a column-oriented full array, use `.to_h.to_a`
+
+- [x] `schema`
+
+  Returns column name and data type in a Hash.
+
+- [x] `==`
+ 
+- [x] `empty?`
+
+### Output
+
+- [x] `to_s`
+
+- [ ] summary, describe
+
+- [x] `to_rover`
+
+  Returns a `Rover::DataFrame`.
+
+- [x] `inspect(tally_level: 5, max_element: 5)`
+
+  Shows some information about self.
+
+  - tally_level: max level to use tally mode
+  - max_element: max num of element to show values in each row
+
+### Selecting
+
+- [x] Selecting columns by `[]`
+
+  `[key]`, `[keys]`, `[keys[index]]`
+
+- [x] Selecting rows by `[]`
+
+  `[index]`, `[range]`, `[array]`
+
+- [x] Selecting rows from top or bottom
+
+  `head(n=5)`, `tail(n=5)`, `first(n=1)`, `last(n=1)`
+
+- [ ] slice
+
+### Updating
+
+- [ ] Add a new column
+
+- [ ] Update a single element
+
+- [ ] Update multiple elements
+
+- [ ] Update all elements
+
+- [ ] Update elements matching a condition
+
+- [ ] Clamp
+
+- [ ] Delete columns
+
+- [ ] Rename a column
+
+- [ ] Sort rows
+
+- [ ] Clear data
+
+### Treat na data
+
+- [ ] Drop na (NaN, nil)
+
+- [ ] Replace na with value
+
+- [ ] Interpolate na with convolution array
+
+### Combining DataFrames
+
+- [ ] Add rows
+
+- [ ] Add columns
+
+- [ ] Inner join
+
+- [ ] Left join
+
+### Encoding
+
+- [ ] One-hot encoding
+
+### Iteration (not impremented)
+
+### Filtering (not impremented)
+
+
+## `RedAmber::Vector`
+### Constructor
+
+- [x] Create from a column in a DataFrame
+
+- [x] New from an Array
+
+### Properties
+
+- [x] `to_s`
+
+- [x] `values`, `to_a`, `entries`
+
+- [x] `size`, `length`, `n_rows`, `nrow`
+
+- [x] `type`
+
+- [ ] `each`
+
+- [ ] `chunked?`
+
+- [ ] `n_chunks`
+
+- [ ] `each_chunk`
+
+- [x] `tally`
+
+- [ ] `n_nulls`
+
+### Functions
+#### Unary aggregations: vector.func => Scalar
+
+| Method    |Boolean|Numeric|String|Remarks|
+| ------------ | --- | --- | --- | ----- |
+|[x] `all`     | [x] |     |     |       |
+|[x] `any`     | [x] |     |     |       |
+|[x] `approximate_median`| | [x] |     |     |
+|[x] `count`         | [x] | [x] | [x] |     |
+|[x] `count_distinct`| [x] | [x] | [x] |     |
+|[x] `count_uniq`    | [x] | [x] | [x] |an alias of `count_distinct`|
+|[ ] `index`   |     |     |     |       |
+|[x] `max`     | [x] | [x] | [x] |       |
+|[x] `mean`    | [x] | [x] |     |       |
+|[x] `min`     | [x] | [x] | [x] |       |
+|[ ] `min_max` |     |     |     |       |
+|[ ] `mode`    |     |     |     |       |
+|[x] `product` | [x] | [x] |     |       |
+|[ ] `quantile`|     |     |     |       |
+|[x] `stddev`  |     | [x] |     |       |
+|[x] `sum`     | [x] | [x] |     |       |
+|[ ] `tdigest` |     |     |     |       |
+|[x] `variance`|     | [x] |     |       |
+
+#### Unary element-wise: vector.func => Vector
+
+| Method    |Boolean|Numeric|String|Remarks|
+| ------------ | --- | --- | --- | ----- |
+|[x] `-@`      |     | [x] |     |as `-vector`|
+|[x] `negate`  |     | [x] |     |`-@`   |
+|[x] `abs`     |     | [x] |     |       |
+|[ ] `acos`    |     | [ ] |     |       |
+|[ ] `asin`    |     | [ ] |     |       |
+|[x] `atan`    |     | [x] |     |       |
+|[ ] `ceil`    |     | [x] |     |       |
+|[x] `cos`     |     | [x] |     |       |
+|[ ] `floor`   |     | [x] |     |       |
+|[ ] `ln`      |     | [ ] |     |       |
+|[ ] `log10`   |     | [ ] |     |       |
+|[ ] `log1p`   |     | [ ] |     |       |
+|[ ] `log2`    |     | [ ] |     |       |
+|[x] `sign`    |     | [x] |     |       |
+|[x] `sin`     |     | [x] |     |       |
+|[x] `tan`     |     | [x] |     |       |
+|[ ] `trunc`   |     | [x] |     |       |
+
+#### Binary element-wise: vector.func(vector) => Vector
+
+| Method          |Boolean|Numeric|String|Remarks|
+| ------------------ | --- | --- | --- | ----- |
+|[x] `add`           |     | [x] |     | `+`   |
+|[x] `atan2`         |     | [x] |     |       |
+|[x] `and`           | [x] |     |     |       |
+|[x] `and_kleene`    | [x] |     |     |       |
+|[x] `and_not`       | [x] |     |     |       |
+|[x] `and_not_kleene`| [x] |     |     |       |
+|[x] `bit_wise_and`  |     |([x])|     |`&`, integer only|
+|[ ] `bit_wise_not`  |     |([x])|     |`!`, integer only|
+|[x] `bit_wise_or`   |     |([x])|     |`|`, integer only|
+|[x] `bit_wise_xor`  |     |([x])|     |`^`, integer only|
+|[x] `divide`        |     | [x] |     | `/`   |
+|[x] `equal`         | [x] | [x] | [x] |`==`, alias `eq`|
+|[x] `greater`       | [x] | [x] | [x] |`>`, alias `gt`|
+|[x] `greater_equal` | [x] | [x] | [x] |`>=`, alias `ge`|
+|[x] `less`          | [x] | [x] | [x] |`<`, alias `lt`|
+|[x] `less_equal`    | [x] | [x] | [x] |`<=`, alias `le`|
+|[ ] `logb`          |     | [ ] |     |       |
+|[ ] `mod`           |     | [ ] |     |       |
+|[x] `multiply`      |     | [x] |     | `*`   |
+|[x] `not_equal`     | [x] | [x] | [x] |`!=`, alias `ne`|
+|[x] `or`            | [x] |     |     |       |
+|[x] `or_kleene`     | [x] |     |     |       |
+|[x] `power`         |     | [x] |     | `**`  |
+|[x] `subtract`      |     | [x] |     | `-`   |
+|[x] `shift_left`    |     |([x])|     |`<<`, integer only|
+|[x] `shift_right`   |     |([x])|     |`>>`, integer only|
+|[x] `xor`           | [x] |     |     |       |
+
+##### (Not impremented)
+- [ ] invert, round, round_to_multiple
+- [ ] sort, sort_index
+- [ ] minmax, var, median, quantile
+- [ ] argmin, argmax
+- [ ] (array functions)
+- [ ] (strings functions)
+- [ ] (temporal functions)
+- [ ] (conditional functions)
+- [ ] (index functions)
+- [ ] (other functions)
+
+### Coerce (not impremented)
+
+### Updating (not impremented)
+
+### DSL in a block for faster calculation ?
+
+
+## Development
+
+```
+git clone https://github.com/heronshoes/red_amber.git
+cd red_amber
+bundle install
+bundle exec rake test
+```
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/**/test_*.rb']
+end
+
+require 'rubocop/rake_task'
+
+RuboCop::RakeTask.new
+
+task default: %i[test rubocop]

data/doc/CODE_OF_CONDUCT.md

@@ -0,0 +1,84 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at 63298319+heronshoes@users.noreply.github.com. All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior,  harassment of an individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0,
+available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at https://www.contributor-covenant.org/translations.

data/lib/red_amber/data_frame.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module RedAmber
+  # data frame class
+  #   @table   : holds Arrow::Table object
+  class DataFrame
+    # mix-in
+    include DataFrameSelectable
+    include DataFrameOutput
+
+    def initialize(*args)
+      # accepts: DataFrame.new, DataFrame.new([]), DataFrame.new(nil)
+      #   returns empty DataFrame
+      @table = Arrow::Table.new({}, [])
+      # bug in gobject-introspection: ruby-gnome/ruby-gnome#1472
+      #  [Arrow::Table] == [nil] shows ArgumentError
+      #  temporary use yoda condition to workaround
+      return if args.empty? || args == [[]] || [nil] == args
+
+      if args.size > 1
+        @table = Arrow::Table.new(*args)
+      else
+        arg = args[0]
+        @table =
+          case arg
+          when Arrow::Table then arg
+          when DataFrame then arg.table
+          when Rover::DataFrame then Arrow::Table.new(arg.to_h)
+          when Hash
+            args << [] if arg.empty? # create empty df from DataFrame.new({})
+            Arrow::Table.new(*args)
+          else
+            raise DataFrameTypeError, "invalid argument: #{args}"
+          end
+      end
+    end
+
+    def self.load(path, options = {})
+      DataFrame.new(Arrow::Table.load(path, options))
+    end
+
+    attr_reader :table
+
+    def save(output, options = {})
+      @table.save(output, options)
+    end
+
+    # Properties ===
+    def n_rows
+      @table.n_rows
+    end
+    alias_method :nrow, :n_rows
+    alias_method :size, :n_rows
+    alias_method :length, :n_rows
+
+    def n_columns
+      @table.n_columns
+    end
+    alias_method :ncol, :n_columns
+    alias_method :width, :n_columns
+
+    def shape
+      [n_rows, n_columns]
+    end
+
+    def column_names
+      @table.columns.map { |column| column.name.to_sym }
+    end
+    alias_method :keys, :column_names
+    alias_method :header, :column_names
+
+    def types(class_name: false)
+      @table.columns.map do |column|
+        r = column.data_type
+        class_name ? r.class : r.to_s.to_sym
+      end
+    end
+
+    def vectors
+      @table.columns.map do |column|
+        Vector.new(column.data)
+      end
+    end
+
+    def to_h
+      @table.columns.each_with_object({}) do |column, result|
+        result[column.name.to_sym] = column.entries
+      end
+    end
+
+    def to_a
+      # output an array of row-oriented data without header
+      # if you need column-oriented array, use `.to_h.to_a`
+      @table.raw_records
+    end
+    alias_method :raw_records, :to_a
+
+    def schema
+      keys.zip(types).to_h
+    end
+
+    def ==(other)
+      other.is_a?(DataFrame) && @table == other.table
+    end
+
+    def empty?
+      @table.columns.empty?
+    end
+
+    def to_rover
+      Rover::DataFrame.new(to_h)
+    end
+
+    # def to_parquet() end
+  end
+end

data/lib/red_amber/data_frame_output.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require 'stringio'
+
+module RedAmber
+  # mix-ins for the class DataFrame
+  module DataFrameOutput
+    def to_s
+      @table.to_s
+    end
+
+    # def describe() end
+
+    # def summary() end
+
+    def inspect_raw
+      format "#<#{self.class}:0x%016x>\n#{self}", object_id
+    end
+
+    # - tally_level: max level to use tally mode
+    # - max_element: max element to show values in each row
+    # TODO: Is it better to change name other than `inspect` ?
+    # TODO: Add na count capability
+    # TODO: Fall back to inspect_raw when treating large dataset
+    # TODO: Refactor code to smaller methods
+    def inspect(tally_level: 5, max_element: 5)
+      return '#<RedAmber::DataFrame (empty)>' if empty?
+
+      stringio = StringIO.new # output string buffer
+
+      # 1st row: show shape of the dataframe
+      r = pl(nrow)
+      c = pl(ncol)
+      stringio.puts \
+        "#{self.class} : #{nrow} observation#{r}(row#{r}) of #{ncol} variable#{c}(column#{c})"
+
+      # 2nd row: show var counts by type
+      type_groups = types(class_name: true).map { |t| type_group(t) }
+
+      stringio.puts "Variable#{pl(ncol)} : #{var_type_count(type_groups).join(', ')}"
+
+      # 3rd row: print header of rows
+      levels = vectors.map { |v| v.to_a.uniq.size }
+      row_headers = { idx: '#', key: 'key', type: 'type', levels: 'level', data: 'data_preview' }
+      # find longest word to adjust column width
+      w_idx = ncol.to_s.size
+      w_key = (keys.map { |key| key.size + 1 } << row_headers[:key].size).max
+      w_type = (types.map(&:size) << row_headers[:type].size).max
+      w_row = (levels.map { |l| l.to_s.size } << row_headers[:levels].size).max
+      stringio.printf("%-#{w_idx}s %-#{w_key}s %-#{w_type}s %-#{w_row}s %s\n", *row_headers.values)
+
+      # (4) show details for each column (vector)
+      vectors.each.with_index(1) do |vector, i|
+        key = keys[i - 1]
+        type = types[i - 1]
+        type_group = type_groups[i - 1]
+        data_tally = vector.tally
+
+        str = format("%#{w_row}d ", data_tally.size)
+        str <<
+          case type_group
+          when :numeric, :string, :boolean
+            if data_tally.size <= tally_level && data_tally.size != nrow
+              data_tally.to_s
+            else
+              reduced_vector_presentation(vector, nrow, max_element)
+            end
+            #  c = vector.is_na.tally[1]   # release when `#is_na` impremented
+            #  str << " #{c} NaN#{pl(c)}" if c&.>(0)  # safely call c>0
+          else
+            reduced_vector_presentation(vector, nrow, max_element)
+          end
+
+        stringio.printf("%#{w_idx}d %-#{w_key}s %-#{w_type}s %s\n", i, ":#{key}", type, str)
+      end
+
+      stringio.string
+    end
+
+    private # =====
+
+    def pl(num)
+      num > 1 ? 's' : ''
+    end
+
+    def type_group(type)
+      if Arrow::NumericDataType >= type
+        :numeric
+      elsif Arrow::StringDataType >= type
+        :string
+      elsif Arrow::BooleanDataType >= type
+        :boolean
+      elsif Arrow::TemporalDataType >= type
+        :temporal
+      else
+        :other
+      end
+    end
+
+    def var_type_count(type_groups)
+      tg = type_groups.tally
+      a = []
+      a << "#{tg[:numeric]} numeric" if tg[:numeric]
+      a << "#{tg[:string]} string#{pl(tg[:string])}" if tg[:string]
+      a << "#{tg[:boolean]} boolean" if tg[:boolean]
+      a << "#{tg[:temporal]} temporal" if tg[:temporal]
+      a
+    end
+
+    def reduced_vector_presentation(vector, nrow, max_element)
+      a = vector.to_a.take(max_element)
+      a << '...' if nrow > max_element
+      "[#{a.join(', ')}]"
+    end
+  end
+end

data/lib/red_amber/data_frame_selectable.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module RedAmber
+  # mix-ins for the class DataFrame
+  module DataFrameSelectable
+    # select columns: [symbol] or [string]
+    # select rows: [array of index], [range]
+    def [](*args)
+      raise DataFrameArgumentError, 'Empty dataframe' if empty?
+      raise DataFrameArgumentError, 'Empty argument' if args.empty?
+
+      # expand Range like [1..3, 4] to [1, 2, 3, 4]
+      expanded =
+        args.each_with_object([]) do |e, a|
+          e.is_a?(Range) ? a.concat(e.to_a) : a.append(e)
+        end
+
+      return select_rows(expanded) if integers?(expanded)
+      return select_columns(expanded.map(&:to_sym)) if sym_or_str?(expanded)
+
+      raise DataFrameArgumentError, "invalid argument #{args}"
+    end
+
+    def head(n_rows = 5)
+      raise DataFrameArgumentError, "index is out of range #{n_rows}" if n_rows.negative?
+
+      self[0...[n_rows, size].min]
+    end
+
+    def tail(n_rows = 5)
+      raise DataFrameArgumentError, "index is out of range #{n_rows}" if n_rows.negative?
+
+      self[-[n_rows, size].min..-1]
+    end
+
+    def first(n_rows = 1)
+      head(n_rows)
+    end
+
+    def last(n_rows = 1)
+      tail(n_rows)
+    end
+
+    private # =====
+
+    def select_columns(keys)
+      if keys.one?
+        Vector.new(@table[*keys].data)
+      else
+        DataFrame.new(@table[keys])
+      end
+    end
+
+    def select_rows(indeces)
+      if out_of_range?(indeces)
+        raise DataFrameArgumentError, "invalid index: #{indeces} for [0..#{size - 1}]"
+      end
+
+      a = indeces.map { |i| @table.slice(i).to_a }
+      DataFrame.new(@table.schema, a)
+    end
+
+    def out_of_range?(indeces)
+      indeces.max >= size || indeces.min < -size
+    end
+
+    def integers?(enum)
+      enum.all?(Integer)
+    end
+
+    def sym_or_str?(enum)
+      enum.all? { |e| e.is_a?(Symbol) || e.is_a?(String) }
+    end
+  end
+end

data/lib/red_amber/vector.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module RedAmber
+  # Columnar data object
+  #   @data : holds Arrow::ChunkedArray
+  class Vector
+    # mix-in
+    include VectorFunctions
+
+    # chunked_array may come from column.data
+    def initialize(array)
+      case array
+      when Vector
+        @data = array.data
+      when Arrow::Array, Arrow::ChunkedArray
+        @data = array
+      when Array
+        @data = Arrow::Array.new(array)
+      else
+        raise ArgumentError, 'Unknown array in argument'
+      end
+    end
+
+    attr_reader :data
+
+    def to_s
+      @data.to_a.inspect
+    end
+
+    def inspect
+      format "#<#{self.class}(:#{type}, size=#{size}):0x%016x>\n#{self}", object_id
+    end
+
+    def values
+      @data.values
+    end
+    alias_method :to_a, :values
+    alias_method :entries, :values
+
+    def size
+      # only defined :length in Arrow?
+      @data.length
+    end
+    alias_method :length, :size
+    alias_method :n_rows, :size
+    alias_method :nrow, :size
+
+    def type
+      @data.value_type.nick.to_sym
+    end
+
+    # def each() end
+
+    def chunked?
+      @data.is_a? Arrow::ChunkedArray
+    end
+
+    def n_chunks
+      chunked? ? @data.n_chunks : 0
+    end
+
+    # def each_chunk() end
+
+    def tally
+      values.tally
+    end
+
+    def n_nulls
+      @data.n_nulls
+    end
+  end
+end

data/lib/red_amber/vector_functions.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+module RedAmber
+  # mix-ins for class Vector
+  module VectorFunctions
+    # Available functions in Arrow are shown by `Arrow::Function.all.map(&:name)`
+    # reference: https://arrow.apache.org/docs/cpp/compute.html
+
+    # [Unary aggregations]: vector.func => Scalar
+    unary_aggregations =
+      %i[all any approximate_median count count_distinct max mean min \
+         product stddev sum variance]
+    unary_aggregations.each do |function|
+      define_method(function) { exec_func(function, other: nil, options: { aggregate: true }) }
+    end
+    alias_method :count_uniq, :count_distinct
+
+    # option(s) required
+    # index
+
+    # Returns other than value
+    # min_max
+    # mode
+    # quantile
+    # tdigest
+
+    # [Unary element-wise]: vector.func => Vector
+    unary_element_wise = %i[abs atan ceil cos floor sign sin tan trunc]
+    unary_element_wise.each do |function|
+      define_method(function) { exec_func(function, other: nil, options: {}) }
+    end
+
+    # [Unary element-wise with operator]: vector.func => Vector
+    unary_element_wise_op = {
+      negate: '-@',
+    }
+    unary_element_wise_op.each do |function, operator|
+      define_method(function) { exec_func(function, other: nil, options: {}) }
+      define_method(operator) { exec_func(function, other: nil, options: {}) }
+    end
+
+    # bit_wise_not => '!', invert, round, round_to_multiple
+
+    # NaN support needed
+    # %i[acos asin ln log10 log1p log2]
+
+    # With numerical range check
+    # %i[abs_checked acos_checked asin_checked cos_checked ln_checked \
+    #    log10_checked log1p_checked log2_checked sin_checked tan_checked]
+
+    # [Binary element-wise]: vector.func(other) => Vector
+    binary_element_wise = %i[atan2 and and_kleene and_not and_not_kleene or or_kleene xor]
+    binary_element_wise.each do |function|
+      define_method(function) do |other|
+        exec_func(function, other: other, options: {})
+      end
+    end
+
+    # NaN support needed
+    # logb
+
+    # With numerical range check
+    # %i[add_checked divide_checked logb_checked multiply_checked power_checked subtract_checked \
+    #    shift_left_checked shift_right_checked]
+
+    # [Binary element-wise with operator]: vector.func(other) => Vector
+    binary_element_wise_op = {
+      add: '+',
+      divide: '/',
+      multiply: '*',
+      power: '**',
+      subtract: '-',
+
+      bit_wise_and: '&',
+      bit_wise_or: '|',
+      bit_wise_xor: '^',
+      shift_left: '<<',
+      shift_right: '>>',
+
+      equal: '==',
+      greater: '>',
+      greater_equal: '>=',
+      less: '<',
+      less_equal: '<=',
+      not_equal: '!=',
+    }
+    binary_element_wise_op.each do |function, operator|
+      define_method(function) do |other|
+        exec_func(function, other: other, options: {})
+      end
+      define_method(operator) do |other|
+        exec_func(function, other: other, options: {})
+      end
+    end
+    alias_method :eq, :equal
+    alias_method :ge, :greater_equal
+    alias_method :gt, :greater
+    alias_method :le, :less_equal
+    alias_method :lt, :less
+    alias_method :ne, :not_equal
+
+    # mod: '%',
+
+    # (array functions)
+    # array_filter, array_sort_indices, array_take
+    # dictionary_encode, hash_all, hash_any, hash_approximate_median,
+    # hash_count, hash_count_distinct, hash_distinct, hash_max, hash_mean, hash_min,
+    # hash_min_max, hash_product, hash_stddev, hash_sum, hash_tdigest, hash_variance,
+    # partition_nth_indices,
+    # quarter, quarters_between, unique,
+    # value_counts
+
+    # (strings)
+    # ascii_capitalize, ascii_center, ascii_is_alnum, ascii_is_alpha, ascii_is_decimal,
+    # ascii_is_lower, ascii_is_printable, ascii_is_space, ascii_is_title, ascii_is_upper,
+    # ascii_lower, ascii_lpad, ascii_ltrim, ascii_ltrim_whitespace, ascii_reverse,
+    # ascii_rpad, ascii_rtrim, ascii_rtrim_whitespace, ascii_split_whitespace,
+    # ascii_swapcase, ascii_title, ascii_trim, ascii_trim_whitespace, ascii_upper,
+    # binary_join, binary_join_element_wise, binary_length, binary_repeat,
+    # binary_replace_slice, binary_reverse, count_substring, count_substring_regex,
+    # ends_with, extract_regex, find_substring, find_substring_regex,
+    # match_like, match_substring, match_substring_regex, replace_substring,
+    # replace_substring_regex, split_pattern, split_pattern_regex, starts_with,
+    # string_is_ascii, utf8_capitalize, utf8_center, utf8_is_alnum, utf8_is_alpha,
+    # utf8_is_decimal, utf8_is_digit, utf8_is_lower, utf8_is_numeric, utf8_is_printable,
+    # utf8_is_space, utf8_is_title, utf8_is_upper, utf8_length, utf8_lower, utf8_lpad,
+    # utf8_ltrim, utf8_ltrim_whitespace, utf8_normalize, utf8_replace_slice, utf8_reverse,
+    # utf8_rpad, utf8_rtrim, utf8_rtrim_whitespace, utf8_slice_codeunits, utf8_split_whitespace,
+    # utf8_swapcase, utf8_title, utf8_trim, utf8_trim_whitespace, utf8_upper
+
+    # (temporal)
+    # assume_timezone, ceil_temporal, day, day_of_week, day_of_year, day_time_interval_between,
+    # days_between, floor_temporal, hour, hours_between, iso_calendar, iso_week, iso_year,
+    # microsecond, microseconds_between, millisecond, milliseconds_between, minute,
+    # minutes_between, month, month_day_nano_interval_between, month_interval_between,
+    # nanosecond, nanoseconds_between, round_temporal, second, seconds_between, strftime,
+    # strptime, subsecond, us_week, week, weeks_between, year, year_month_day, years_between
+
+    # (onditional)
+    # case_when, cast, if_else
+
+    # (indices)
+    # choose, index_in, index_in_meta_binary, indices_nonzero
+
+    # (others)
+    # coalesce, drop_null, fill_null_backward, fill_null_forward,
+    # filter, is_finite, is_in, is_in_meta_binary, is_inf, is_nan, is_null, is_valid,
+    # list_element, list_flatten, list_parent_indices, list_value_length, make_struct,
+    # max_element_wise, min_element_wise, random, replace_with_mask, select_k_unstable,
+    # sort_indices, struct_field, take
+
+    private # =======
+
+    def exec_func(function, other: nil, options: {})
+      func = Arrow::Function.find(function)
+      output =
+        case other
+        when nil
+          func.execute([data])
+        when Arrow::Array, Arrow::ChunkedArray, Arrow::Scalar, Numeric
+          func.execute([data, other])
+        when Vector
+          func.execute([data, other.data])
+        when Rover::Vector
+          func.execute([data, other.to_a])
+        else
+          raise ArgumentError, "operand is not supported: #{other.class}"
+        end
+      options[:aggregate] ? output.value : Vector.new(output.value)
+    end
+  end
+end

data/lib/red_amber/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module RedAmber
+  VERSION = '0.1.1'
+end

data/lib/red_amber.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'arrow'
+require 'rover-df'
+
+require_relative 'red_amber/data_frame_output'
+require_relative 'red_amber/data_frame_selectable'
+require_relative 'red_amber/data_frame'
+require_relative 'red_amber/vector_functions'
+require_relative 'red_amber/vector'
+require_relative 'red_amber/version'
+
+module RedAmber
+  class Error < StandardError; end
+
+  class DataFrameArgumentError < ArgumentError; end
+  class DataFrameTypeError < TypeError; end
+end

data/red_amber.gemspec

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative 'lib/red_amber/version'
+
+Gem::Specification.new do |spec|
+  spec.name = 'red_amber'
+  spec.version = RedAmber::VERSION
+  spec.authors = ['Hirokazu SUZUKI (heronshoes)']
+  spec.email = ['63298319+heronshoes@users.noreply.github.com']
+
+  spec.summary = 'Simple data frames for Ruby'
+  spec.description = 'Powered by Red Arrow and simple API similar to Rover-df'
+  spec.homepage = 'https://github.com/heronshoes/red_amber'
+  spec.license = 'MIT'
+  spec.required_ruby_version = '>= 2.7'
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/heronshoes/red_amber'
+  spec.metadata['changelog_uri'] = 'https://github.com/heronshoes/red_amber/blob/main/CHANGELOG.md'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) ||
+        f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
+    end
+  end
+  spec.bindir = 'exe'
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'red-arrow', '~> 7.0.0'
+  spec.add_dependency 'red-parquet', '~> 7.0.0'
+  spec.add_dependency 'rover-df', '~> 0.3.0'
+
+  # Development dependency has gone to the Gemfile (rubygems/bundler#7237)
+
+  spec.metadata['rubygems_mfa_required'] = 'true'
+
+  # For more information and examples about making a new gem, check out our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end

data/sig/red_amber.rbs

@@ -0,0 +1,4 @@
+module RedAmber
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification
+name: red_amber
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Hirokazu SUZUKI (heronshoes)
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2022-05-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: red-arrow
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+- !ruby/object:Gem::Dependency
+  name: red-parquet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+- !ruby/object:Gem::Dependency
+  name: rover-df
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+description: Powered by Red Arrow and simple API similar to Rover-df
+email:
+- 63298319+heronshoes@users.noreply.github.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rubocop.yml"
+- ".rubocop_todo.yml"
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- doc/CODE_OF_CONDUCT.md
+- lib/red_amber.rb
+- lib/red_amber/data_frame.rb
+- lib/red_amber/data_frame_output.rb
+- lib/red_amber/data_frame_selectable.rb
+- lib/red_amber/vector.rb
+- lib/red_amber/vector_functions.rb
+- lib/red_amber/version.rb
+- red_amber.gemspec
+- sig/red_amber.rbs
+homepage: https://github.com/heronshoes/red_amber
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/heronshoes/red_amber
+  source_code_uri: https://github.com/heronshoes/red_amber
+  changelog_uri: https://github.com/heronshoes/red_amber/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.7'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.7
+signing_key:
+specification_version: 4
+summary: Simple data frames for Ruby
+test_files: []