red_amber 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4d18eedf5de7fd06fe52e8a82ad38fe12d590dc10929c96872e557b9e946f785
-  data.tar.gz: dda93f0af421096410e00ecf2261e8846a236634bd96ae9941d1b5cd49cd5eb2
+  metadata.gz: ae6a6696e0f01ae7d621d11542e203803ba117fc1ee3d286a1444b3c4ac746fc
+  data.tar.gz: 722d4ad538fe4f0c85db4911e773e1f87eb03f47fa63c954529bf04babc55d8c
 SHA512:
-  metadata.gz: 7c1b1edd6c1f6f3f275ea765c4bc8765327c88a36120a4c5a66dd8afa59f5913db4a5b436d80378554e03403bab823edf7467beea0f44e2803e36f3e9677a065
-  data.tar.gz: 949fd15d2076d4e53fb141375bde282228c7f6566e137047344134c54964fe77fd2f9757b0bdc324eb3cfa14091f2ae928e0e844d28f3ebbcfa17fc7d388bbd0
+  metadata.gz: 96887abfbdd44330e80a6a97f91597c00706fc99492d086683702f1d3e757331e90fb275e5796a2b7b3228b476f8c7799ab22727411baedb79ae39acafd2d3f0
+  data.tar.gz: c020bba60734fccdeb4a18efecb98260f70fa76c5b8ba2c7f2830d2dac6de66e9776a1fa2ffc5d396e7270b29c5e4dd9737dee70b5dcc47dd3113aaafe4f4d22

data/.rubocop.yml

@@ -56,7 +56,9 @@ Metrics/AbcSize:
   Max: 30
   Exclude:
     - 'lib/red_amber/data_frame_displayable.rb' # Max: 55
-    - 'lib/red_amber/vector_compensable.rb' # Max: 36
+    - 'lib/red_amber/data_frame_selectable.rb' # Max: 51
+    - 'lib/red_amber/vector_updatable.rb' # Max: 36
+    - 'lib/red_amber/vector_selectable.rb' # Max: 33
 
 # Max: 25
 Metrics/BlockLength:
@@ -66,15 +68,18 @@ Metrics/BlockLength:
 
 # Max: 100
 Metrics/ClassLength:
-  Max: 120
+  Max: 100
   Exclude:
     - 'test/**/*'
+    - 'lib/red_amber/data_frame.rb' #Max: 131
+    - 'lib/red_amber/vector.rb' #Max: 102
 
 # Max: 7
 Metrics/CyclomaticComplexity:
   Max: 12
   Exclude:
-    - 'lib/red_amber/vector_compensable.rb' # Max: 14
+    - 'lib/red_amber/data_frame_selectable.rb' # Max: 14
+    - 'lib/red_amber/vector_updatable.rb' # Max: 14
 
 # Max: 10
 Metrics/MethodLength:
@@ -86,20 +91,34 @@ Metrics/MethodLength:
 Metrics/ModuleLength:
   Max: 100
   Exclude:
+    - 'lib/red_amber/data_frame_selectable.rb' # Max: 141
     - 'lib/red_amber/vector_functions.rb' # Max: 114
 
 # Max: 8
 Metrics/PerceivedComplexity:
   Max: 13
   Exclude:
-    - 'lib/red_amber/vector_compensable.rb' # Max: 15
+    - 'lib/red_amber/data_frame_selectable.rb' # Max: 14
+    - 'lib/red_amber/vector_updatable.rb' # Max: 15
+
+Naming/FileName:
+  Exclude:
+    - 'lib/red-amber.rb'
 
-# Necessary to define is_na
+# Necessary to define is_na, is_in, etc.
 Naming/PredicateName:
   Exclude:
     - 'lib/red_amber/vector_functions.rb'
+    - 'lib/red_amber/vector.rb'
+    - 'lib/red_amber/vector_selectable.rb'
 
 # Necessary to test when range.end == -1
 Style/SlicingWithRange:
   Exclude:
     - 'test/test_data_frame_selectable.rb'
+
+# Necessary to Vector < 0 element-wise comparison
+Style/NumericPredicate:
+  Exclude:
+    - 'lib/red_amber/data_frame_selectable.rb'
+    - 'lib/red_amber/vector_selectable.rb'

data/CHANGELOG.md

@@ -1,30 +1,115 @@
-## [0.2.0] - unreleased
+## - unreleased
 
 - Document
   - YARD support
 
-- DataFrame#join features
+- `datasets-red-amber` gem
+- `red-amber` gem
 
-## [0.1.6] - Unreleased
+- `Vector#divmod`
+  - Introduce if Arrow's function is ready
 
-- Feedback something to Red Data Tools
+## - Unreleased, will be after Arrow 9.0.0 released
 
 - `DataFrame`
   - Introduce `summary` or ``describe`
-  - Add `Quantile` by own code?
-  - Improve dataframe obs. manipuration methods to accept float as a index (#10)
-  - Improve as more performant by benchmark check.
+    - `Quantile` will be available
 
-- `Vector`
-  - Support more functions
-  - Support coerece
+## [0.1.7] - Unreleased, may be 2022-07-10
 
+- Feedback something to Red Data Tools
+- Support more functions
+- Improve as more performant
 - More examples of frequently needed tasks
 
+- New `Group` API
+- `DataFrame#join features
+
+## [0.1.6] - 2022-06-26 (experimental)
+
+- Bug fixes
+  - Fix mime-type of empty DataFrame in `#to_iruby` (#31)
+  - Fix mime setting in `DataFrame#to_iruby` (#36)
+  - Fix unmatched return val in Selectable (#34)
+  - Fix to return same error as `#[]` in `DataFrame#slice` (#34)
+
+- New features and improvements
+  - Introduce Jupyter support (#29, #30, #31, #32)
+    - Add `DataFrame#to_html (changed to use #to_iruby)
+    - Add feature to show nil in to_iruby
+      - nil is expressed as (nil)
+      - empty string('') is ""
+      - blank spaces are " "
+
+  - Enable to change DataFrame display mode by ENV (#36)
+    - Support ENV['RED_AMBER_OUTPUT_STYLE'] to change display mode in `#inspect` and `#to_iruby`
+      - ENV['RED_AMBER_OUTPUT_STYLE'] = 'table' # => Table mode
+      - ENV['RED_AMBER_OUTPUT_STYLE'] = nil or other than 'table' # => TDR mode
+
+  - Support `require 'red-amber'`, as well (#34)
+
+  - Refine Vector slicing methods (#31)
+    - Introduce `Vector#take` method
+    - Introduce `Vector#filter` method
+    - Improve `Vector#[]` to overload take and filter
+    - Introduce `Vector#drop_nil` method
+    - Introduce `Vector#if_else` method
+    - Intorduce `Vector#is_in` method
+    - Add alias `Vector#all?`, `#any?` methods (#32)
+    - Add `Vector#has_nil?` method(#32)
+    - Add `Vector#empty?` method
+    - Add `Vector#primitive_invert` method
+    - Refactor `Vector#take`, `#filter`
+    - Move `Vector#if_else` from function to Updatable
+    - Move if_else test to updatable
+    - Rename updatable in test
+    - Remove method `Vector#take_out_element_wise`
+    - Rename inner metthod name
+
+  - Refine DataFrame slicing methods (#31)
+    - Introduce `DataFrame#take method
+      - #take is implemented as vector calculation by #if_else
+    - Introduce `DataFrame#fliter method
+    - Change `DataFrame#[] to use take and filter
+      - Float indices is acceptable (#10)
+      - Negative index (like Array) is also acceptable
+
+  - Further refinement in DataFrame slicing methods (#34)
+   -  Improve `DataFrame#[]`, `#slice`, `#remove` by a new engine
+      -  It parses arguments to Vector internally.
+      -  Used Kernel#Array to simplify code (#16) .
+    - recycle: Move `DataFrame#slice`, `#remove` to Selectable
+    - Refine `DataFrame#take`, `#filter` (undocumented)
+
+  - Introduce coerce in Vector (#35)
+    - Introduce `Vector#coerce`
+      - Now we can `-1 * Vector.new([1, 2, 3])`
+    - Add `Vector#to_ary` method
+      - Now we can `[1, 2] + Vector.new([3, 4, 5])`
+
+  - Other new feature or refinements
+    - Common
+      - Refactor helper as common for DataFrame and Vector (#35)
+      - Change name row/col to obs/var (#34)
+      - Rename internal function name (#34)
+      - Delete unused methods (#34)
+    - DataFrame
+      - Change to return instance variable in `#to_arrow`, `#keys` and `#key_index` (#34)
+      - Change to return an Array in `DataFrame#indices` (#35)
+    - Vector
+      - Introduce `Vector#replace` method
+      - Accept Range and expanded Array in `Vector#new`
+      - Add `Vector#indices` method (#35)
+      - Add `Vector#index` method (#35)
+      - Rename VectorCompensable to *Updatable (#33)
+
+  - Documentation
+    - Fix typo in DataFrame.md
+
 ## [0.1.5] - 2022-06-12 (experimental)
 
 - Bug fixes
-  - Fix DF#tdr to display timestamp type (#19)
+  - Fix DataFrame#tdr to display timestamp type (#19)
   - Add TZ setting in CI test to pass temporal tests (#19)
   - Fix example in document of #load(csv_from_URI) (#23)
 
@@ -38,7 +123,7 @@
     - Add `Vector#temporal?` to check if temporal type
     - Refine around DataFrame#variables
     - Refine init of instance variables
-    - Refine DataFrame#type_classes, V#ectortype_class
+    - Refine DataFrame#type_classes, Vector#ectortype_class
     - Refine DataFrame#tdr to shorten temporal data
 
   - Add supports to make up for missing values (#20)
@@ -86,7 +171,7 @@
 
 - Bug fixes
   - Fix missing support for scalar argument (#1)
-  - Fix type name of boolean in DF#types to be same as Vector#type (#6, #7)
+  - Fix type name of boolean in DataFrame#types to be same as Vector#type (#6, #7)
   - Fix zero picking to return empty DataFrame (#8)
   - Fix code at both args and a block given (#8)
 

data/Gemfile

@@ -12,6 +12,7 @@ group :test do
   gem 'rubocop-rake'
   gem 'rubocop-rubycw', require: false
 
+  gem 'iruby'
   gem 'test-unit'
   gem 'webrick'
 

data/README.md

@@ -1,6 +1,6 @@
 # RedAmber
 
-A simple dataframe library for Ruby (experimental)
+A simple dataframe library for Ruby (experimental).
 
 - Powered by [Red Arrow](https://github.com/apache/arrow/tree/master/ruby/red-arrow)
 - Inspired by the dataframe library [Rover-df](https://github.com/ankane/rover)
@@ -42,17 +42,56 @@ Or install it yourself as:
 gem install red_amber
 ```
 
+(From v0.1.6)
+
+RedAmber uses TDR mode for `#inspect` and `#to_iruby` by default. If you prefer Table mode, please set the environment variable
+`RED_AMBER_OUTPUT_MODE` to `"table"`. See [TDR section](#TDR) for detail.
+
 ## `RedAmber::DataFrame`
 
-Represents a set of data in 2D-shape.
+Represents a set of data in 2D-shape. The entity is a Red Arrow's Table object. 
 
 ```ruby
-require 'red_amber'
+require 'red_amber' # require 'red-amber' is also OK.
 require 'datasets-arrow'
 
 arrow = Datasets::Penguins.new.to_arrow
 penguins = RedAmber::DataFrame.new(arrow)
-penguins.tdr
+penguins.table
+
+# =>
+#<Arrow::Table:0x111271098 ptr=0x7f9118b3e0b0>
+	species	island	bill_length_mm	bill_depth_mm	flipper_length_mm	body_mass_g	sex	year
+  0	Adelie 	Torgersen	     39.100000	    18.700000	              181	       3750	male	2007
+  1	Adelie 	Torgersen	     39.500000	    17.400000	              186	       3800	female	2007
+  2	Adelie 	Torgersen	     40.300000	    18.000000	              195	       3250	female	2007
+  3	Adelie 	Torgersen	        (null)	       (null)	           (null)	     (null)	(null)	2007
+  4	Adelie 	Torgersen	     36.700000	    19.300000	              193	       3450	female	2007
+  5	Adelie 	Torgersen	     39.300000	    20.600000	              190	       3650	male	2007
+  6	Adelie 	Torgersen	     38.900000	    17.800000	              181	       3625	female	2007
+  7	Adelie 	Torgersen	     39.200000	    19.600000	              195	       4675	male	2007
+  8	Adelie 	Torgersen	     34.100000	    18.100000	              193	       3475	(null)	2007
+  9	Adelie 	Torgersen	     42.000000	    20.200000	              190	       4250	(null)	2007
+...
+334	Gentoo 	Biscoe	     46.200000	    14.100000	              217	       4375	female	2009
+335	Gentoo 	Biscoe	     55.100000	    16.000000	              230	       5850	male	2009
+336	Gentoo 	Biscoe	     44.500000	    15.700000	              217	       4875	(null)	2009
+337	Gentoo 	Biscoe	     48.800000	    16.200000	              222	       6000	male	2009
+338	Gentoo 	Biscoe	     47.200000	    13.700000	              214	       4925	female	2009
+339	Gentoo 	Biscoe	        (null)	       (null)	           (null)	     (null)	(null)	2009
+340	Gentoo 	Biscoe	     46.800000	    14.300000	              215	       4850	female	2009
+341	Gentoo 	Biscoe	     50.400000	    15.700000	              222	       5750	male	2009
+342	Gentoo 	Biscoe	     45.200000	    14.800000	              212	       5200	female	2009
+343	Gentoo 	Biscoe	     49.900000	    16.100000	              213	       5400	male	2009
+```
+
+By default, RedAmber shows self by compact transposed style. This unfamiliar style (TDR) is designed for
+the exploratory data processing. It keeps Vectors as row vectors, shows keys and types at a glance, shows levels 
+for the 'factor-like' variables and shows the number of abnormal values like NaN and nil.
+
+```ruby
+penguins
+
 # =>
 RedAmber::DataFrame : 344 x 8 Vectors
 Vectors : 5 numeric, 3 strings
@@ -139,9 +178,19 @@ Vectors accepts some [functional methods from Arrow](https://arrow.apache.org/do
 
 See [Vector.md](doc/Vector.md) for details.
 
-## TDR concept
+## TDR
+
+I named the data frame representation style in the model above as TDR (Transposed DataFrame Representation). 
+
+This library can be used with both TDR mode and usual Table mode.
+If you set the environment variable `RED_AMBER_OUTPUT_MODE` to `"table"`, output style by `inspect` and `to_iruby` is the Table mode. Other value including nil will output TDR style.
+
+You can switch the mode in Ruby like this.
+```ruby
+ENV['RED_AMBER_OUTPUT_STYLE'] = 'table' # => Table mode
+```
 
-I named the data frame representation style in the model above as TDR (Transposed DataFrame Representation). See [TDR.md](doc/tdr.md) for details.
+For more detail information about TDR, see [TDR.md](doc/tdr.md).
 
 ## Development
 

data/doc/DataFrame.md

@@ -4,11 +4,13 @@ Class `RedAmber::DataFrame` represents 2D-data. A `DataFrame` consists with:
 - A collection of data which have same data type within. We call it `Vector`.
 - A label is attached to `Vector`. We call it `key`.
 - A `Vector` and associated `key` is grouped as a `variable`.
-- `variable`s with same vector length are aligned and arranged to be a `DaTaFrame`.
+- `variable`s with same vector length are aligned and arranged to be a `DataFrame`.
 - Each `Vector` in a `DataFrame` contains a set of relating data at same position. We call it `observation`.
 
 ![dataframe model image](doc/../image/dataframe_model.png)
 
+(No change in this model in v0.1.6 .)
+
 ## Constructors and saving
 
 ### `new` from a Hash
@@ -52,7 +54,7 @@ Class `RedAmber::DataFrame` represents 2D-data. A `DataFrame` consists with:
 - from a URI
 
   ```ruby
-  uri = URI("uri = URI("https://raw.githubusercontent.com/mwaskom/seaborn-data/master/penguins.csv")
+  uri = URI("https://raw.githubusercontent.com/mwaskom/seaborn-data/master/penguins.csv")
   RedAmber::DataFrame.load(uri)
   ```
 
@@ -147,9 +149,9 @@ Class `RedAmber::DataFrame` represents 2D-data. A `DataFrame` consists with:
 
 - Returns an Array of Vectors.
 
-### `indexes`, `indices`
+### `indices`, `indexes`
 
-- Returns all indexes in a Range.
+- Returns all indexes in an Array.
 
 ### `to_h`
 
@@ -179,6 +181,10 @@ Class `RedAmber::DataFrame` represents 2D-data. A `DataFrame` consists with:
 
 - Returns a `Rover::DataFrame`.
 
+### `to_iruby`
+
+- Show the DataFrame as a Table in Jupyter Notebook or Jupyter Lab with IRuby.
+
 ### `tdr(limit = 10, tally: 5, elements: 5)`
 
   - Shows some information about self in a transposed style.
@@ -280,6 +286,9 @@ Class `RedAmber::DataFrame` represents 2D-data. A `DataFrame` consists with:
   An end-less or a begin-less Range can be used to represent indeces.
 
 - Select obs. by indeces in an Array: `df[1, 2]`
+
+- You can use float indices.
+
 - Mixed case: `df[2, 0..]`
 
   ```ruby
@@ -423,9 +432,11 @@ Class `RedAmber::DataFrame` represents 2D-data. A `DataFrame` consists with:
 
   ![slice method image](doc/../image/dataframe/slice.png)
 
-- Keys as arguments
+- Indices as arguments
 
-    `slice(indeces)` accepts indeces as arguments. Indeces should be an Integer or a Range of Integer.
+    `slice(indeces)` accepts indices as arguments. Indices should be Integers, Floats or Ranges of Integers.
+
+    Negative index from the tail like Ruby's Array is also acceptable.
 
     ```ruby
     # returns 5 obs. at start and 5 obs. from end
@@ -457,7 +468,7 @@ Class `RedAmber::DataFrame` represents 2D-data. A `DataFrame` consists with:
      ... 5 more Vectors ...
     ```
 
-- Keys or booleans by a block
+- Indices or booleans by a block
 
     `slice {block}` is also acceptable. We can't use both arguments and a block at a same time. The block should return indeces or a boolean Array with a same length as `size`. Block is called in the context of self.
 
@@ -469,6 +480,7 @@ Class `RedAmber::DataFrame` represents 2D-data. A `DataFrame` consists with:
       max = vector.mean + vector.std
       vector.to_a.map { |e| (min..max).include? e }
     end
+
     # =>
     #<RedAmber::DataFrame : 204 x 8 Vectors, 0x000000000000f30c>
     Vectors : 5 numeric, 3 strings
@@ -509,7 +521,7 @@ Class `RedAmber::DataFrame` represents 2D-data. A `DataFrame` consists with:
 
   ![remove method image](doc/../image/dataframe/remove.png)
 
-- Keys as arguments
+- Indices as arguments
 
     `remove(indeces)` accepts indeces as arguments. Indeces should be an Integer or a Range of Integer.
 
@@ -548,7 +560,7 @@ Class `RedAmber::DataFrame` represents 2D-data. A `DataFrame` consists with:
     8 :year              uint16     3 {2007=>103, 2008=>113, 2009=>117}    
     ```
 
-- Keys or booleans by a block
+- Indices or booleans by a block
 
     `remove {block}` is also acceptable. We can't use both arguments and a block at a same time. The block should return indeces or a boolean Array with a same length as `size`. Block is called in the context of self.
 
@@ -748,6 +760,8 @@ Class `RedAmber::DataFrame` represents 2D-data. A `DataFrame` consists with:
 
 ### `group(aggregating_keys, function, target_keys)`
 
+  (This is a temporary API and may change in the future version.)
+
   Create grouped dataframe by `aggregation_keys` and apply `function` to each group and returns in `target_keys`. Aggregated key name is `function(key)` style.
 
   (The current implementation is not intuitive. Needs improvement.)