red_amber 0.2.3 → 0.3.0

data/benchmark/vector.yml

@@ -0,0 +1,60 @@
+loop_count: 10
+
+contexts:
+  - name: HEAD
+    prelude: |
+      $LOAD_PATH.unshift(File.expand_path('lib'))
+  - name: 0.2.0
+    gems:
+      red_amber: 0.2.0
+
+prelude: |
+  require 'red_amber'
+  include RedAmber
+  require 'datasets-arrow'
+
+  ds = Datasets::Rdatasets.new('nycflights13', 'flights')
+  flights = RedAmber::DataFrame.new(ds.to_arrow)
+  df = flights.slice { flights[:month] <= 6 }
+
+  tailnum_vector = df[:tailnum]
+  distance_vector = df[:distance]
+
+  strings = tailnum_vector.to_a
+  arrow_array = tailnum_vector.data
+  integers = df[:dep_delay].to_a
+  boolean_vector = df[:air_time].is_nil
+  index_vector = Vector.new(0...boolean_vector.size).filter(boolean_vector)
+  replacer = index_vector.data.map(&:to_s)
+  booleans = boolean_vector.to_a
+
+benchmark:
+  'V01: Vector.new from integer Array': |
+    Vector.new(integers)
+
+  'V02: Vector.new from string Array': |
+    Vector.new(strings)
+
+  'V03: Vector.new from boolean Vector': |
+    Vector.new(boolean_vector)
+
+  'V04: Vector#sum': |
+    distance_vector.mean
+
+  'V05: Vector#*': |
+    distance_vector * 1.852
+
+  'V06: Vector#[booleans]': |
+    tailnum_vector[booleans]
+
+  'V07: Vector#[boolean_vector]': |
+    tailnum_vector[boolean_vector]
+
+  'V08: Vector#[index_vector]': |
+    tailnum_vector[index_vector]
+
+  'V09: Vector#replace': |
+    tailnum_vector.replace(booleans, replacer)
+
+  'V10: Vector#replace with broad casting': |
+    tailnum_vector.replace(booleans, 'x')

data/doc/DataFrame.md

@@ -1302,7 +1302,10 @@ When the option `keep_key: true` used, the column `key` will be preserved.
   - `join_keys` are keys shared by self and other to match with them.
   - If `join_keys` are empty, common keys in self and other are chosen (natural join).
   - If (common keys) > `join_keys`, duplicated keys are renamed by `suffix`. 
+  - If you want to match the columns with different names,
+    use Hash for `join_keys` such as `{ left: :KEY1, right: KEY2}`.
 
+  These are dataframes to use in the examples of joins.
   ```ruby
   df = DataFrame.new(
     KEY: %w[A B C],

data/doc/Vector.md

@@ -513,3 +513,91 @@ vector.shift(fill: Float::NAN)
 #<RedAmber::Vector(:double, size=5):0x0000000000011d3c>                    
 [NaN, 1.0, 2.0, 3.0, 4.0]
 ```
+
+### `split_to_columns(sep = ' ', limit = 0)`
+
+Split string type Vector with any ASCII whitespace as separator.
+Returns an Array of Vectors.
+
+```ruby
+vector = Vector.new(['a b', 'c d', 'e f'])
+vector.split_to_columns
+
+#=> 
+[#<RedAmber::Vector(:string, size=3):0x00000000000363a8>                                
+["a", "c", "e"]                                    
+,                                                  
+ #<RedAmber::Vector(:string, size=3):0x00000000000363bc>
+["b", "d", "f"]                                    
+]
+```
+It will be used for column splitting in DataFrame.
+
+```ruby
+df = DataFrame.new(year_month: %w[2022-01 2022-02 2022-03])
+  .assign(:year, :month) { year_month.split_to_columns('-') }
+  .drop(:year_month)
+
+#=>
+#<RedAmber::DataFrame : 3 x 2 Vectors, 0x000000000000f974>
+  year     month
+  <string> <string>
+0 2022     01
+1 2022     02
+2 2022     03
+```
+
+### `split_to_rows(sep = ' ', limit = 0)`
+
+Split string type Vector with any ASCII whitespace as separator.
+Returns an flattend into rows by Vector.
+
+```ruby
+vector = Vector.new(['a b', 'c d', 'e f'])
+vector.split_to_rows
+
+#=>
+#<RedAmber::Vector(:string, size=6):0x000000000002ccf4>
+["a", "b", "c", "d", "e", "f"]
+```
+
+### `merge(other, sep: ' ')`
+
+Merge String or other string Vector to self using aseparator.
+Self must be a string Vector.
+Returns merged string Vector.
+
+```ruby
+# with vector
+vector = Vector.new(%w[a c e])
+other = Vector.new(%w[b d f])
+vector.merge(other)
+
+#=>
+#<RedAmber::Vector(:string, size=3):0x0000000000038b80>
+["a b", "c d", "e f"]
+```
+
+If other is a String it will be broadcasted.
+
+```ruby
+# with vector
+vector = Vector.new(%w[a c e])
+
+#=>
+#<RedAmber::Vector(:string, size=3):0x00000000000446b0>
+["a x", "c x", "e x"]
+```
+
+You can specify separator string by :sep.
+
+```ruby
+# with vector
+vector = Vector.new(%w[a c e])
+other = Vector.new(%w[b d f])
+vector.merge(other, sep: '')
+
+#=>
+#<RedAmber::Vector(:string, size=3):0x0000000000038b80>
+["ab", "cd", "ef"]
+```

data/lib/red_amber/data_frame.rb

@@ -14,65 +14,111 @@ module RedAmber
     include DataFrameVariableOperation
     include Helper
 
-    # Creates a new RedAmber::DataFrame.
+    using RefineArrowTable
+    using RefineHash
+
+    # Quicker DataFrame construction from a `Arrow::Table`.
     #
-    # @overload initialize(hash)
+    # @param table [Arrow::Table] A table to have in the DataFrame.
+    # @return [DataFrame] Initialized DataFrame.
     #
-    #   @params hash [Hash]
+    # @note This method will allocate table directly and may be used in the method.
+    # @note `table` must have unique keys.
+    def self.create(table)
+      instance = allocate
+      instance.instance_variable_set(:@table, table)
+      instance
+    end
+
+    # Creates a new DataFrame.
     #
     # @overload initialize(table)
+    #   Initialize DataFrame by an `Arrow::Table`
+    #
+    #   @param table [Arrow::Table]
+    #     A table to have in the DataFrame.
+    #
+    # @overload initialize(arrowable)
+    #   Initialize DataFrame by a `#to_arrow` responsible object.
+    #
+    #   @param arrowable [#to_arrow]
+    #     Any object which responds to `#to_arrow`.
+    #     `#to_arrow` must return `Arrow::Table`.
+    #
+    #   @note `RedAmber::DataFrame` itself is readable by this.
+    #   @note Hash is refined to respond to `#to_arrow` in this class.
+    #
+    # @overload initialize(rover_like)
+    #   Initialize DataFrame by a `Rover::DataFrame`-like `#to_h` responsible object.
     #
-    #   @params table [Arrow::Table]
+    #   @param rover_like [#to_h]
+    #     Any object which responds to `#to_h`.
+    #     `#to_h` must return a Hash which is convertable by `Arrow::Table.new`.
     #
-    # @overload initialize(dataframe)
+    #   @note `Rover::DataFrame` is readable by this.
     #
-    #   @params dataframe [RedAmber::DataFrame, Rover::DataFrame]
+    # @overload initialize()
+    #   Create empty DataFrame
     #
-    # @overload initialize(null)
+    #   @example DataFrame.new
     #
-    #   @params null [NilClass] No arguments.
+    # @overload initialize(empty)
+    #   Create empty DataFrame
+    #
+    #   @param empty [nil, [], {}]
+    #
+    #   @example DataFrame.new([]), DataFrame.new({}), DataFrame.new(nil)
+    #
+    # @overload initialize(args)
+    #
+    #   @param args [values]
+    #     Accepts any argments which is valid for `Arrow::Table.new(args)`. See
+    #     {https://github.com/apache/arrow/blob/master/ruby/red-arrow/lib/arrow/table.rb
     #
     def initialize(*args)
-      @variables = @keys = @vectors = @types = @data_types = nil
       case args
       in nil | [nil] | [] | {} | [[]] | [{}]
-        # DataFrame.new, DataFrame.new([]), DataFrame.new({}), DataFrame.new(nil)
-        #   returns empty DataFrame
         @table = Arrow::Table.new({}, [])
-      in [->(x) { x.respond_to?(:to_arrow) } => arrowable]
+      in [Arrow::Table => table]
+        @table = table
+      in [arrowable] if arrowable.respond_to?(:to_arrow)
         table = arrowable.to_arrow
         unless table.is_a?(Arrow::Table)
           raise DataFrameTypeError,
                 "to_arrow must return an Arrow::Table but #{table.class}: #{arrowable}"
         end
         @table = table
-      in [Arrow::Table => table]
-        @table = table
-      in [rover_or_hash]
+      in [rover_like] if rover_like.respond_to?(:to_h)
         begin
-          # Accepts Rover::DataFrame or Hash
-          @table = Arrow::Table.new(rover_or_hash.to_h)
+          # Accepts Rover::DataFrame
+          @table = Arrow::Table.new(rover_like.to_h)
         rescue StandardError
-          raise DataFrameTypeError, "invalid argument: #{rover_or_hash}"
+          raise DataFrameTypeError, "to_h must return Arrowable object: #{rover_like}"
         end
       else
-        @table = Arrow::Table.new(*args)
+        begin
+          @table = Arrow::Table.new(*args)
+        rescue StandardError
+          raise DataFrameTypeError, "invalid argument to create Arrow::Table: #{args}"
+        end
       end
-      name_unnamed_keys
 
-      duplicated_keys = keys.tally.select { |_k, v| v > 1 }.keys
-      raise DataFrameArgumentError, "duplicate keys: #{duplicated_keys}" unless duplicated_keys.empty?
+      name_unnamed_keys
+      check_duplicate_keys(keys)
     end
 
+    # Returns the table having within.
+    #
+    # @return [Arrow::Table] The table within.
+    #
     attr_reader :table
 
-    def to_arrow
-      @table
-    end
+    alias_method :to_arrow, :table
 
     # Returns the number of rows.
     #
     # @return [Integer] Number of rows.
+    #
     def size
       @table.n_rows
     end
@@ -83,6 +129,7 @@ module RedAmber
     # Returns the number of columns.
     #
     # @return [Integer] Number of columns.
+    #
     def n_keys
       @table.n_columns
     end
@@ -95,6 +142,7 @@ module RedAmber
     # @return [Array]
     #   Number of rows and number of columns in an array.
     #   Same as [size, n_keys].
+    #
     def shape
       [size, n_keys]
     end
@@ -102,7 +150,8 @@ module RedAmber
     # Returns a Hash of key and Vector pairs in the columns.
     #
     # @return [Hash]
-    #   key => Vector pairs for each columns.
+    #   `key => Vector` pairs for each columns.
+    #
     def variables
       @variables || @variables = init_instance_vars(:variables)
     end
@@ -112,6 +161,7 @@ module RedAmber
     #
     # @return [Array]
     #   Keys in an Array.
+    #
     def keys
       @keys || @keys = init_instance_vars(:keys)
     end
@@ -123,6 +173,7 @@ module RedAmber
     # @param key [Symbol, String] Key to test.
     # @return [Boolean]
     #   Returns true if self has key in Symbol.
+    #
     def key?(key)
       keys.include?(key.to_sym)
     end
@@ -133,6 +184,7 @@ module RedAmber
     # @param key [Symbol, String] key to know.
     # @return [Integer]
     #   Index of key in the Array keys.
+    #
     def key_index(key)
       keys.find_index(key.to_sym)
     end
@@ -143,14 +195,18 @@ module RedAmber
     #
     # @return [Array]
     #   Abbreviated Red Arrow data type names.
+    #
     def types
-      @types || @types = @table.columns.map { |column| column.data.value_type.nick.to_sym }
+      @types || @types = @table.columns.map do |column|
+        column.data.value_type.nick.to_sym
+      end
     end
 
     # Returns an Array of Classes of data type.
     #
     # @return [Array]
     #   An Array of Red Arrow data type Classes.
+    #
     def type_classes
       @data_types || @data_types = @table.columns.map { |column| column.data_type.class }
     end
@@ -158,50 +214,94 @@ module RedAmber
     # Returns Vectors in an Array.
     #
     # @return [Array]
-    #   An Array of RedAmber::Vector s.
+    #   An Array of `RedAmber::Vector`s.
+    #
     def vectors
       @vectors || @vectors = init_instance_vars(:vectors)
     end
 
-    # Returns row indices (start...(size+start)) in an Array.
+    # Returns row indices (start...(size+start)) in a Vector.
     #
     # @param start [Object]
-    #   Object which have #succ method.
+    #   Object which have `#succ` method.
+    #
     # @return [Array]
-    #   An Array of indices of the row.
+    #   A Vector of row indices.
+    #
     # @example
     #   (when self.size == 5)
-    #   - indices #=> [0, 1, 2, 3, 4]
-    #   - indices(1) #=> [1, 2, 3, 4, 5]
-    #   - indices('a') #=> ['a', 'b', 'c', 'd', 'e']
+    #   - indices #=> Vector[0, 1, 2, 3, 4]
+    #   - indices(1) #=> Vector[1, 2, 3, 4, 5]
+    #   - indices('a') #=> Vector['a', 'b', 'c', 'd', 'e']
+    #
     def indices(start = 0)
       Vector.new((start..).take(size))
     end
     alias_method :indexes, :indices
 
+    # Returns column-oriented data in a Hash.
+    #
+    # @return [Hash] A Hash of 'key => column_in_an_array'.
+    #
     def to_h
       variables.transform_values(&:to_a)
     end
 
+    # Returns a row-oriented array without header.
+    #
+    # @return [Array] Row-oriented data without header.
+    #
+    # @note If you need column-oriented array, use `.to_h.to_a`.
+    #
     def to_a
-      # output an array of row-oriented data without header
-      # if you need column-oriented array, use `.to_h.to_a`
       @table.raw_records
     end
     alias_method :raw_records, :to_a
 
+    # Returns column name and data type in a Hash.
+    #
+    # @return [Hash] Column name and data type.
+    #
+    # @example
+    #   RedAmber::DataFrame.new(x: [1, 2, 3], y: %w[A B C]).schema
+    #   # => {:x=>:uint8, :y=>:string}
+    #
     def schema
       keys.zip(types).to_h
     end
 
+    # Compare DataFrames.
+    #
+    # @return [true, false]
+    #   True if other is a DataFrame and table is same.
+    #   Otherwise return false.
+    #
     def ==(other)
       other.is_a?(DataFrame) && @table == other.table
     end
 
+    # Check if it is a empty DataFrame.
+    #
+    # @return [true, false] True if it has no columns.
+    #
     def empty?
       variables.empty?
     end
 
+    # Enumerate for each row.
+    #
+    # @overload each_row
+    #   Returns Enumerator when no block given.
+    #
+    #   @return [Enumerator] Enumerator of each rows.
+    #
+    # @overload each_row(&block)
+    #   Yields with key and row pairs.
+    #
+    #   @yield [key_row_pairs] Yields with key and row pairs.
+    #   @yieldparam [Hash] Key and row pairs.
+    #   @yieldreturn [Integer] Size of the DataFrame.
+    #
     def each_row
       return enum_for(:each_row) unless block_given?
 
@@ -214,6 +314,10 @@ module RedAmber
       end
     end
 
+    # Returns self in a `Rover::DataFrame`.
+    #
+    # @return [Rover::DataFrame] A `Rover::DataFrame`.
+    #
     def to_rover
       require 'rover'
       Rover::DataFrame.new(to_h)
@@ -226,7 +330,7 @@ module RedAmber
     end
 
     def method_missing(name, *args, &block)
-      return v(name) if args.empty?
+      return v(name) if args.empty? && key?(name)
 
       super
     end
@@ -241,20 +345,31 @@ module RedAmber
 
     # initialize @variable, @keys, @vectors and return one of them
     def init_instance_vars(var)
-      ary = @table.columns.each_with_object([{}, [], []]) do |column, (variables, keys, vectors)|
-        v = Vector.new(column.data)
-        k = column.name.to_sym
-        v.key = k
-        variables[k] = v
-        keys << k
-        vectors << v
-      end
+      ary =
+        @table.columns
+              .each_with_object([{}, [], []]) do |column, (variables, keys, vectors)|
+          v = Vector.create(column.data)
+          k = column.name.to_sym
+          v.key = k
+          variables[k] = v
+          keys << k
+          vectors << v
+        end
+
       @variables, @keys, @vectors = ary
       ary[%i[variables keys vectors].index(var)]
     end
 
+    def check_duplicate_keys(array)
+      org = array.dup
+      return unless array.uniq!
+
+      raise DataFrameArgumentError,
+            "duplicate keys: #{org.tally.select { |_k, v| v > 1 }.keys}"
+    end
+
     def name_unnamed_keys
-      return unless @table[:'']
+      return unless @table.key?('')
 
       # We can't use #keys because it causes mismatch of @table and @keys
       keys = @table.schema.fields.map { |f| f.name.to_sym }