red_amber 0.2.1 → 0.2.2

data/doc/Vector.md

@@ -7,7 +7,7 @@ Class `RedAmber::Vector` represents a series of data in the DataFrame.
 ### Create from a column in a DataFrame
   
   ```ruby
-  df = RedAmber::DataFrame.new(x: [1, 2, 3])
+  df = DataFrame.new(x: [1, 2, 3])
   df[:x]
   # =>
   #<RedAmber::Vector(:uint8, size=3):0x000000000000f4ec>
@@ -17,13 +17,13 @@ Class `RedAmber::Vector` represents a series of data in the DataFrame.
 ### New from an Array
 
   ```ruby
-  vector = RedAmber::Vector.new([1, 2, 3])
+  vector = Vector.new([1, 2, 3])
   # or
-  vector = RedAmber::Vector.new(1, 2, 3)
+  vector = Vector.new(1, 2, 3)
   # or
-  vector = RedAmber::Vector.new(1..3)
+  vector = Vector.new(1..3)
   # or
-  vector = RedAmber::Vector.new(Arrow::Array([1, 2, 3])
+  vector = Vector.new(Arrow::Array.new([1, 2, 3])
 
   # =>
   #<RedAmber::Vector(:uint8, size=3):0x000000000000f514>
@@ -61,7 +61,7 @@ Class `RedAmber::Vector` represents a series of data in the DataFrame.
 
 ### `type_class`
 
-### `each`
+### `each`, `map`, `collect`
 
   If block is not given, returns Enumerator.
 
@@ -78,7 +78,7 @@ Class `RedAmber::Vector` represents a series of data in the DataFrame.
   - `limit` sets size limit to display a long array.
 
     ```ruby
-    vector = RedAmber::Vector.new((1..50).to_a)
+    vector = Vector.new((1..50).to_a)
     # =>
     #<RedAmber::Vector(:uint8, size=50):0x000000000000f528>
     [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, ... ]
@@ -95,8 +95,8 @@ Class `RedAmber::Vector` represents a series of data in the DataFrame.
 - Negative index is also OK like the Ruby's primitive Array.
 
 ```ruby
-array = RedAmber::Vector.new(%w[A B C D E])
-indices = RedAmber::Vector.new([0.1, -0.5, -5.1])
+array = Vector.new(%w[A B C D E])
+indices = Vector.new([0.1, -0.5, -5.1])
 array.take(indices)
 # or
 array[indices]
@@ -106,7 +106,7 @@ array[indices]
 ["A", "E", "A"]
 ```
 
-### `filter(booleans)`, `[](booleans)`
+### `filter(booleans)`, `select(booleans)`, `[](booleans)`
 
 - Acceptable class for booleans:
   - An array of true, false, or nil
@@ -114,7 +114,7 @@ array[indices]
   - Arrow::BooleanArray
 
 ```ruby
-array = RedAmber::Vector.new(%w[A B C D E])
+array = Vector.new(%w[A B C D E])
 booleans = [true, false, nil, false, true]
 array.filter(booleans)
 # or
@@ -124,6 +124,7 @@ array[booleans]
 #<RedAmber::Vector(:string, size=2):0x000000000000f21c>
 ["A", "E"]
 ```
+`filter` and `select` also accepts a block.
 
 ## Functions
 
@@ -158,7 +159,7 @@ Options can be used as follows.
 See the [document of C++ function](https://arrow.apache.org/docs/cpp/compute.html) for detail.
 
 ```ruby
-double = RedAmber::Vector.new([1, 0/0.0, -1/0.0, 1/0.0, nil, ""])
+double = Vector.new([1, 0/0.0, -1/0.0, 1/0.0, nil, ""])
 #=>
 #<RedAmber::Vector(:double, size=6):0x000000000000f910>
 [1.0, NaN, -Infinity, Infinity, nil, 0.0]
@@ -168,7 +169,7 @@ double.count(mode: :only_valid) #=> 5, default
 double.count(mode: :only_null) #=> 1
 double.count(mode: :all) #=> 6
 
-boolean = RedAmber::Vector.new([true, true, nil])
+boolean = Vector.new([true, true, nil])
 #=>
 #<RedAmber::Vector(:boolean, size=3):0x000000000000f924>
 [true, true, nil]
@@ -215,7 +216,7 @@ Examples of options for `#round`;
 - `round_mode` Specify rounding mode.
 
 ```ruby
-double = RedAmber::Vector.new([15.15, 2.5, 3.5, -4.5, -5.5])
+double = Vector.new([15.15, 2.5, 3.5, -4.5, -5.5])
 # => [15.15, 2.5, 3.5, -4.5, -5.5]
 double.round
 # => [15.0, 2.0, 4.0, -4.0, -6.0]
@@ -293,7 +294,7 @@ double.round(n_digits: -1)
   array = [0.0/0, Float::NAN]
   array.tally #=> {NaN=>1, NaN=>1}
 
-  vector = RedAmber::Vector.new(array)
+  vector = Vector.new(array)
   vector.tally #=> {NaN=>2}
   vector.value_counts #=> {NaN=>2}
   ```
@@ -310,7 +311,7 @@ double.round(n_digits: -1)
 ## Coerce
 
 ```ruby
-vector = RedAmber::Vector.new(1,2,3)
+vector = Vector.new(1,2,3)
 # => 
 #<RedAmber::Vector(:uint8, size=3):0x00000000000decc4>            
 [1, 2, 3]                                                         
@@ -340,12 +341,13 @@ vector * -1
 - Accepts Scalar, Range  of Integer, Vector, Array, Arrow::Array as a specifier
 - Accepts Scalar, Vector, Array and Arrow::Array as a replacer.
 - Boolean specifiers specify the position of replacer in true.
+  - If booleans.any is false, no replacement happen and return self.
 - Index specifiers specify the position of replacer in indices.
 - replacer specifies the values to be replaced.
   - The number of true in booleans must be equal to the length of replacer
 
 ```ruby
-vector = RedAmber::Vector.new([1, 2, 3])
+vector = Vector.new([1, 2, 3])
 booleans = [true, false, true]
 replacer = [4, 5]
 vector.replace(booleans, replacer)
@@ -379,7 +381,7 @@ vector.replace(booleans, replacer)
 ```ruby
 booleans = [true, false, nil]
 replacer = -1
-vec.replace(booleans, replacer)
+vector.replace(booleans, replacer)
 => 
 #<RedAmber::Vector(:int8, size=3):0x00000000000304d0>
 [-1, 2, nil]
@@ -390,17 +392,7 @@ vec.replace(booleans, replacer)
 ```ruby
 booleans = [true, false, true]
 replacer = [nil]
-vec.replace(booleans, replacer)
-=> 
-#<RedAmber::Vector(:int8, size=3):0x00000000000304d0>
-[nil, 2, nil]
-```
-
-- If no replacer specified, it is same as to specify nil.
-
-```ruby
-booleans = [true, false, true]
-vec.replace(booleans)
+vector.replace(booleans, replacer)
 => 
 #<RedAmber::Vector(:int8, size=3):0x00000000000304d0>
 [nil, 2, nil]
@@ -409,7 +401,7 @@ vec.replace(booleans)
 - An example to replace 'NA' to nil.
 
 ```ruby
-vector = RedAmber::Vector.new(['A', 'B', 'NA'])
+vector = Vector.new(['A', 'B', 'NA'])
 vector.replace(vector == 'NA', nil)
 # =>
 #<RedAmber::Vector(:string, size=3):0x000000000000f8ac>
@@ -421,7 +413,7 @@ vector.replace(vector == 'NA', nil)
 Specified indices are used 'as sorted'. Position in indices and replacer may not have correspondence.
 
 ```ruby
-vector = RedAmber::Vector.new([1, 2, 3])
+vector = Vector.new([1, 2, 3])
 indices = [2, 1]
 replacer = [4, 5]
 vector.replace(indices, replacer)
@@ -437,7 +429,7 @@ Propagate the last valid observation forward (or backward).
 Or preserve nil if all previous values are nil or at the end.
 
 ```ruby
-integer = RedAmber::Vector.new([0, 1, nil, 3, nil])
+integer = Vector.new([0, 1, nil, 3, nil])
 integer.fill_nil_forward
 # =>
 #<RedAmber::Vector(:uint8, size=5):0x000000000000f960>
@@ -459,7 +451,7 @@ Choose values based on self. Self must be a boolean Vector.
 This example will normalize negative indices to positive ones.
 
 ```ruby
-indices = RedAmber::Vector.new([1, -1, 3, -4])
+indices = Vector.new([1, -1, 3, -4])
 array_size = 10
 normalized_indices = (indices < 0).if_else(indices + array_size, indices)
 
@@ -474,7 +466,7 @@ For each element in self, return true if it is found in given `values`, false ot
 By default, nulls are matched against the value set. (This will be changed in SetLookupOptions: not impremented.)
 
 ```ruby
-vector = RedAmber::Vector.new %W[A B C D]
+vector = Vector.new %W[A B C D]
 values = ['A', 'C', 'X']
 vector.is_in(values)
 
@@ -486,7 +478,7 @@ vector.is_in(values)
 `values` are casted to the same Class of Vector.
 
 ```ruby
-vector = RedAmber::Vector.new([1, 2, 255])
+vector = Vector.new([1, 2, 255])
 vector.is_in(1, -1)
 
 # =>
@@ -499,7 +491,7 @@ vector.is_in(1, -1)
 Shift vector's values by specified `amount`. Shifted space is filled by value `fill`.
 
 ```ruby
-vector = RedAmber::Vector.new([1, 2, 3, 4, 5])
+vector = Vector.new([1, 2, 3, 4, 5])
 vector.shift
 
 # =>

data/lib/red_amber/data_frame.rb

@@ -7,6 +7,7 @@ module RedAmber
     # mix-in
     include DataFrameDisplayable
     include DataFrameIndexable
+    include DataFrameLoadSave
     include DataFrameReshaping
     include DataFrameSelectable
     include DataFrameVariableOperation
@@ -37,6 +38,13 @@ module RedAmber
         # DataFrame.new, DataFrame.new([]), DataFrame.new({}), DataFrame.new(nil)
         #   returns empty DataFrame
         @table = Arrow::Table.new({}, [])
+      in [->(x) { x.respond_to?(:to_arrow) } => arrowable]
+        table = arrowable.to_arrow
+        unless table.is_a?(Arrow::Table)
+          raise DataFrameTypeError,
+                "to_arrow must return an Arrow::Table but #{table.class}: #{arrowable}"
+        end
+        @table = table
       in [Arrow::Table => table]
         @table = table
       in [DataFrame => dataframe]
@@ -52,10 +60,9 @@ module RedAmber
         @table = Arrow::Table.new(*args)
       end
       name_unnamed_keys
-    end
 
-    def self.load(path, options = {})
-      DataFrame.new(Arrow::Table.load(path, options))
+      duplicated_keys = keys.tally.select { |_k, v| v > 1 }.keys
+      raise DataFrameArgumentError, "duplicate keys: #{duplicated_keys}" unless duplicated_keys.empty?
     end
 
     attr_reader :table
@@ -64,10 +71,6 @@ module RedAmber
       @table
     end
 
-    def save(output, options = {})
-      @table.save(output, options)
-    end
-
     # Returns the number of rows.
     #
     # @return [Integer] Number of rows.
@@ -215,17 +218,6 @@ module RedAmber
       Rover::DataFrame.new(to_h)
     end
 
-    def to_iruby
-      require 'iruby'
-      return ['text/plain', '(empty DataFrame)'] if empty?
-
-      if ENV.fetch('RED_AMBER_OUTPUT_MODE', 'Table') == 'TDR'
-        size <= 5 ? ['text/plain', tdr_str(tally: 0)] : ['text/plain', tdr_str]
-      else
-        ['text/html', html_table]
-      end
-    end
-
     def group(*group_keys, &block)
       g = Group.new(self, group_keys)
       g = g.summarize(&block) if block
@@ -260,25 +252,6 @@ module RedAmber
       ary[%i[variables keys vectors].index(var)]
     end
 
-    def html_table
-      reduced = size > 8 ? self[0..4, -4..-1] : self
-
-      converted = reduced.assign do
-        vectors.select.with_object({}) do |vector, assigner|
-          if vector.has_nil?
-            assigner[vector.key] = vector.to_a.map do |e|
-              e = e.nil? ? '<i>(nil)</i>' : e.to_s # nil
-              e = '""' if e.empty? # empty string
-              e.sub(/(\s+)/, '"\1"') # blank spaces
-            end
-          end
-        end
-      end
-
-      html = IRuby::HTML.table(converted.to_h, maxrows: 8, maxcols: 15)
-      "#{self.class} <#{size} x #{n_keys} vector#{pl(n_keys)}> #{html}"
-    end
-
     def name_unnamed_keys
       return unless @table[:'']
 

data/lib/red_amber/data_frame_displayable.rb

@@ -37,8 +37,12 @@ module RedAmber
     alias_method :describe, :summary
 
     def inspect
-      if ENV.fetch('RED_AMBER_OUTPUT_MODE', 'Table') == 'TDR'
+      mode = ENV.fetch('RED_AMBER_OUTPUT_MODE', 'Table')
+      case mode.upcase
+      when 'TDR'
         "#<#{shape_str(with_id: true)}>\n#{dataframe_info(3)}"
+      when 'MINIMUM'
+        shape_str
       else
         "#<#{shape_str(with_id: true)}>\n#{self}"
       end
@@ -55,6 +59,23 @@ module RedAmber
       "#{shape_str}\n#{dataframe_info(limit, tally_level: tally, max_element: elements)}"
     end
 
+    def to_iruby
+      require 'iruby'
+      return ['text/plain', '(empty DataFrame)'] if empty?
+
+      mode = ENV.fetch('RED_AMBER_OUTPUT_MODE', 'Table')
+      case mode.upcase
+      when 'PLAIN'
+        ['text/plain', inspect]
+      when 'MINIMUM'
+        ['text/plain', shape_str]
+      when 'TDR'
+        size <= 5 ? ['text/plain', tdr_str(tally: 0)] : ['text/plain', tdr_str]
+      else # 'TABLE'
+        ['text/html', html_table]
+      end
+    end
+
     private # =====
 
     def shape_str(with_id: false)
@@ -98,7 +119,7 @@ module RedAmber
             else
               [shorthand(vector, size, max_element)]
             end
-        sio.printf header_format, i + 1, key, type, data_tally.size, a.join(', ')
+        sio.printf header_format, i, key, type, data_tally.size, a.join(', ')
       end
       sio.string
     end
@@ -156,7 +177,7 @@ module RedAmber
       original = self
       indices = size > head + tail ? [*0..head, *(size - tail)...size] : [*0...size]
       df = slice(indices).assign do
-        assigner = { INDEX_KEY => indices.map { |i| (i + 1).to_s } }
+        assigner = { INDEX_KEY => indices.map(&:to_s) }
         vectors.each_with_object(assigner) do |v, a|
           a[v.key] = v.to_a.map do |e|
             if e.nil?
@@ -220,5 +241,37 @@ module RedAmber
         "%#{width}s"
       end
     end
+
+    def html_table
+      reduced = size > 8 ? self[0..4, -4..-1] : self
+
+      converted = reduced.assign do
+        vectors.select.with_object({}) do |vector, assigner|
+          assigner[vector.key] = vector.map do |element|
+            case element
+            in TrueClass
+              '<i>(true)</i>'
+            in FalseClass
+              '<i>(false)</i>'
+            in NilClass
+              '<i>(nil)</i>'
+            in ''
+              '""'
+            in String
+              element.sub(/^(\s+)$/, '"\1"') # blank spaces
+            in Float
+              format('%g', element)
+            in Integer
+              format('%d', element)
+            else
+              element
+            end
+          end
+        end
+      end
+
+      html = IRuby::HTML.table(converted.to_h, maxrows: 8, maxcols: 15)
+      "#{self.class} <#{size} x #{n_keys} vector#{pl(n_keys)}> #{html}"
+    end
   end
 end

data/lib/red_amber/data_frame_loadsave.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module RedAmber
+  # mix-ins for the class DataFrame
+  module DataFrameLoadSave
+    # Enable `self.load` as class method of DataFrame
+    def self.included(klass)
+      klass.extend ClassMethods
+    end
+
+    # Enable `self.load` as class method of DataFrame
+    module ClassMethods
+      # Load DataFrame via Arrow::Table.load
+      def load(path, options = {})
+        DataFrame.new(Arrow::Table.load(path, options))
+      end
+    end
+
+    # Save DataFrame
+    def save(output, options = {})
+      @table.save(output, options)
+    end
+
+    # Save and reload to cast automatically
+    #   Via tsv format file temporally as default
+    #
+    #   experimental feature
+    def auto_cast(format: :tsv)
+      return self if empty?
+
+      tempfile = Arrow::ResizableBuffer.new(1024)
+      save(tempfile, format: format)
+      DataFrame.load(tempfile, format: format)
+    end
+  end
+end

data/lib/red_amber/data_frame_reshaping.rb

@@ -9,16 +9,17 @@ module RedAmber
     #   to transepose into keys.
     #   If it is not specified, keys[0] is used.
     # @param new_key [Symbol] key name of transposed index column.
-    #   If it is not specified, :N is used. If it already exists, :N1 or :N1.succ is used.
+    #   If it is not specified, :NAME is used. If it already exists, :NAME1 or :NAME1.succ is used.
     # @return [DataFrame] trnsposed DataFrame
-    def transpose(key: keys.first, name: :N)
+    def transpose(key: keys.first, name: :NAME)
       raise DataFrameArgumentError, "Self does not include: #{key}" unless keys.include?(key)
 
       # Find unused name
       new_keys = self[key].to_a.map { |e| e.to_s.to_sym }
-      name = (:N1..).find { |k| !new_keys.include?(k) } if new_keys.include?(name)
+      name = (:NAME1..).find { |k| !new_keys.include?(k) } if new_keys.include?(name)
 
-      hash = { name => (keys - [key]) }
+      names = (keys - [key]).map { |x| x&.to_s }
+      hash = { name => names }
       i = keys.index(key)
       each_row do |h|
         k = h.values[i]
@@ -33,7 +34,7 @@ module RedAmber
     # @param name [Symbol, String] key of the column which is come **from values**.
     # @param value [Symbol, String] key of the column which is come **from values**.
     # @return [DataFrame] long DataFrame.
-    def to_long(*keep_keys, name: :N, value: :V)
+    def to_long(*keep_keys, name: :NAME, value: :VALUE)
       not_included = keep_keys - keys
       raise DataFrameArgumentError, "Not have keys #{not_included}" unless not_included.empty?
 
@@ -55,6 +56,7 @@ module RedAmber
           end
         end
       end
+      hash[name] = hash[name].map { |x| x&.to_s }
       DataFrame.new(hash)
     end
 
@@ -63,7 +65,7 @@ module RedAmber
     # @param name [Symbol, String] key of the column which will be expanded **to key names**.
     # @param value [Symbol, String] key of the column which will be expanded **to values**.
     # @return [DataFrame] wide DataFrame.
-    def to_wide(name: :N, value: :V)
+    def to_wide(name: :NAME, value: :VALUE)
       name = name.to_sym
       raise DataFrameArgumentError, "Invalid key: #{name}" unless keys.include?(name)
 

data/lib/red_amber/data_frame_variable_operation.rb

@@ -15,16 +15,22 @@ module RedAmber
       return DataFrame.new if picker.empty? || picker == [nil]
 
       key_vector = Vector.new(keys)
-      picker_vector = parse_to_vector(picker)
-
-      picker = key_vector.filter(*picker_vector).to_a if picker_vector.boolean?
-      picker = key_vector.take(*picker_vector).to_a if picker_vector.numeric?
+      vec = parse_to_vector(picker, vsize: n_keys)
+
+      ary =
+        if vec.boolean?
+          key_vector.filter(*vec).to_a
+        elsif vec.numeric?
+          key_vector.take(*vec).to_a
+        elsif vec.string? || vec.dictionary?
+          picker
+        else
+          raise DataFrameArgumentError, "Invalid argument #{args}"
+        end
 
       # DataFrame#[] creates a Vector with single key is specified.
       # DataFrame#pick creates a DataFrame with single key.
-      return DataFrame.new(@table[picker]) if sym_or_str?(picker)
-
-      raise DataFrameArgumentError, "Invalid argument #{args}"
+      DataFrame.new(@table[ary])
     end
 
     # drop some variables to create remainer sub DataFrame
@@ -38,24 +44,24 @@ module RedAmber
       dropper.flatten!
 
       key_vector = Vector.new(keys)
-      dropper_vector = parse_to_vector(dropper)
-
-      picker =
-        if dropper_vector.boolean?
-          key_vector.filter(*dropper_vector.primitive_invert).each.map(&:to_sym)
-        elsif dropper_vector.numeric?
-          keys - key_vector.take(*dropper_vector).each.map(&:to_sym)
-        else
+      vec = parse_to_vector(dropper, vsize: n_keys)
+
+      ary =
+        if vec.boolean?
+          key_vector.filter(*vec.primitive_invert).each.map(&:to_sym) # Array
+        elsif vec.numeric?
+          keys - key_vector.take(*vec).each.map(&:to_sym) # Array
+        elsif vec.string? || vec.dictionary?
           keys - dropper
+        else
+          raise DataFrameArgumentError, "Invalid argument #{args}"
         end
 
-      return DataFrame.new if picker.empty?
+      return DataFrame.new if ary.empty?
 
       # DataFrame#[] creates a Vector with single key is specified.
       # DataFrame#drop creates a DataFrame with single key.
-      return DataFrame.new(@table[picker]) if sym_or_str?(picker)
-
-      raise DataFrameArgumentError, "Invalid argument #{args}"
+      DataFrame.new(@table[ary])
     end
 
     # rename variables to create a new DataFrame

data/lib/red_amber/group.rb

@@ -54,9 +54,11 @@ module RedAmber
       raise GroupArgumentError, "#{d} is not a key of\n #{@dataframe}." unless summary_keys.empty? || d.empty?
 
       df = RedAmber::DataFrame.new(@group.send(func, *summary_keys))
-      df = df[@group_keys, df.keys - @group_keys]
-      # if counts are the same (no nil included), aggregate count columns.
-      df = df[df.keys[0..1]].rename(df.keys[1], :count) if func == :count && df.to_h.values[1..].uniq.size == 1
+      df = df.pick(@group_keys, df.keys - @group_keys)
+      # if counts are the same (and do not include NaN or nil), aggregate count columns.
+      if func == :count && df.pick(@group_keys.size..).to_h.values.uniq.size == 1
+        df = df.pick(0..@group_keys.size).rename { [keys[-1], :count] }
+      end
       df
     end
   end