red_amber 0.1.8 → 0.2.0

data/lib/red_amber/data_frame.rb

@@ -1,35 +1,55 @@
 # frozen_string_literal: true
 
 module RedAmber
-  # data frame class
-  #   @table   : holds Arrow::Table object
+  # Class to represent a data frame.
+  # Variable @table holds an Arrow::Table object.
   class DataFrame
     # mix-in
     include DataFrameDisplayable
     include DataFrameIndexable
+    include DataFrameReshaping
     include DataFrameSelectable
     include DataFrameVariableOperation
     include Helper
 
+    # Creates a new RedAmber::DataFrame.
+    #
+    # @overload initialize(hash)
+    #
+    #   @params hash [Hash]
+    #
+    # @overload initialize(table)
+    #
+    #   @params table [Arrow::Table]
+    #
+    # @overload initialize(dataframe)
+    #
+    #   @params dataframe [RedAmber::DataFrame, Rover::DataFrame]
+    #
+    # @overload initialize(null)
+    #
+    #   @params null [NilClass] No arguments.
+    #
     def initialize(*args)
       @variables = @keys = @vectors = @types = @data_types = nil
-      if args.empty? || args[0] == [] || args[0] == {} || args[0].nil?
+      case args
+      in nil | [nil] | [] | {} | [[]] | [{}]
         # DataFrame.new, DataFrame.new([]), DataFrame.new({}), DataFrame.new(nil)
         #   returns empty DataFrame
         @table = Arrow::Table.new({}, [])
-      elsif args.size > 1
-        @table = Arrow::Table.new(*args)
+      in [Arrow::Table => table]
+        @table = table
+      in [DataFrame => dataframe]
+        @table = dataframe.table
+      in [rover_or_hash]
+        begin
+          # Accepts Rover::DataFrame or Hash
+          @table = Arrow::Table.new(rover_or_hash.to_h)
+        rescue StandardError
+          raise DataFrameTypeError, "invalid argument: #{rover_or_hash}"
+        end
       else
-        arg = args[0]
-        @table =
-          case arg
-          when Arrow::Table then arg
-          when DataFrame then arg.table
-          when Rover::DataFrame then Arrow::Table.new(arg.to_h)
-          when Hash then Arrow::Table.new(arg)
-          else
-            raise DataFrameTypeError, "invalid argument: #{arg}"
-          end
+        @table = Arrow::Table.new(*args)
       end
       name_unnamed_keys
     end
@@ -48,56 +68,101 @@ module RedAmber
       @table.save(output, options)
     end
 
+    # Returns the number of rows.
+    #
+    # @return [Integer] Number of rows.
     def size
       @table.n_rows
     end
     alias_method :n_rows, :size
     alias_method :n_obs, :size
 
+    # Returns the number of columns.
+    #
+    # @return [Integer] Number of columns.
     def n_keys
       @table.n_columns
     end
     alias_method :n_cols, :n_keys
     alias_method :n_vars, :n_keys
 
+    # Returns the numbers of rows and columns.
+    #
+    # @return [Array]
+    #   Number of rows and number of columns in an array.
+    #   Same as [size, n_keys].
     def shape
       [size, n_keys]
     end
 
+    # Returns a Hash of key and Vector pairs in the columns.
+    #
+    # @return [Hash]
+    #   key => Vector pairs for each columns.
     def variables
       @variables || @variables = init_instance_vars(:variables)
     end
     alias_method :vars, :variables
 
+    # Returns an Array of keys.
+    #
+    # @return [Array]
+    #   Keys in an Array.
     def keys
       @keys || @keys = init_instance_vars(:keys)
     end
     alias_method :column_names, :keys
     alias_method :var_names, :keys
 
+    # Returns true if self has a specified key in the argument.
+    #
+    # @param key [Symbol, String] Key to test.
+    # @return [Boolean]
+    #   Returns true if self has key in Symbol.
     def key?(key)
       keys.include?(key.to_sym)
     end
     alias_method :has_key?, :key?
 
+    # Returns index of specified key in the Array keys.
+    #
+    # @param key [Symbol, String] key to know.
+    # @return [Integer]
+    #   Index of key in the Array keys.
     def key_index(key)
       keys.find_index(key.to_sym)
     end
     alias_method :find_index, :key_index
     alias_method :index, :key_index
 
+    # Returns abbreviated type names in an Array.
+    #
+    # @return [Array]
+    #   Abbreviated Red Arrow data type names.
     def types
       @types || @types = @table.columns.map { |column| column.data.value_type.nick.to_sym }
     end
 
+    # Returns an Array of Classes of data type.
+    #
+    # @return [Array]
+    #   An Array of Red Arrow data type Classes.
     def type_classes
       @data_types || @data_types = @table.columns.map { |column| column.data_type.class }
     end
 
+    # Returns Vectors in an Array.
+    #
+    # @return [Array]
+    #   An Array of RedAmber::Vector s.
     def vectors
       @vectors || @vectors = init_instance_vars(:vectors)
     end
 
+    # Returns row indices (0...size) in an Array.
+    #
+    # @return [Array]
+    #   An Array of all indices of rows.
     def indices
       (0...size).to_a
     end
@@ -126,6 +191,18 @@ module RedAmber
       variables.empty?
     end
 
+    def each_row
+      return enum_for(:each_row) unless block_given?
+
+      size.times do |i|
+        key_row_pairs =
+          vectors.each_with_object({}) do |v, h|
+            h[v.key] = v.data[i]
+          end
+        yield key_row_pairs
+      end
+    end
+
     def to_rover
       require 'rover'
       Rover::DataFrame.new(to_h)

data/lib/red_amber/data_frame_displayable.rb

@@ -7,15 +7,34 @@ module RedAmber
   module DataFrameDisplayable
     INDEX_KEY = :index_key_for_format_table
 
-    def to_s
+    def to_s(width: 80)
       return '' if empty?
 
-      format_table(width: 80)
+      format_table(width: width)
     end
 
-    # def describe() end
-
-    # def summary() end
+    # Show statistical summary by a new DatFrame.
+    #   Make stats for numeric columns only.
+    #   NaNs are ignored.
+    #   Counts also show non-NaN counts.
+    #
+    # @return [DataFrame] a new dataframe.
+    def summary
+      num_keys = keys.select { |key| self[key].numeric? }
+
+      DataFrame.new(
+        variables: num_keys,
+        count: num_keys.map { |k| self[k].count },
+        mean: num_keys.map { |k| self[k].mean },
+        std: num_keys.map { |k| self[k].std },
+        min: num_keys.map { |k| self[k].min },
+        '25%': num_keys.map { |k| self[k].quantile(0.25) },
+        median: num_keys.map { |k| self[k].median },
+        '75%': num_keys.map { |k| self[k].quantile(0.75) },
+        max: num_keys.map { |k| self[k].max }
+      )
+    end
+    alias_method :describe, :summary
 
     def inspect
       if ENV.fetch('RED_AMBER_OUTPUT_MODE', 'Table') == 'TDR'
@@ -133,11 +152,7 @@ module RedAmber
       a
     end
 
-    def format_table(width: 80)
-      head = 5
-      tail = 3
-      n_digit = 1
-
+    def format_table(width: 80, head: 5, tail: 3, n_digit: 2)
       original = self
       indices = size > head + tail ? [*0...head, *(size - tail)...size] : [*0...size]
       df = slice(indices).assign do

data/lib/red_amber/data_frame_reshaping.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module RedAmber
+  # mix-ins for the class DataFrame
+  module DataFrameReshaping
+    # Transpose a wide DataFrame.
+    #
+    # @param key [Symbol, FalseClass] key of the index column
+    #   to transepose into keys.
+    #   If it is false, keys[0] is used.
+    # @param new_key [Symbol, FalseClass] key name of transposed index column.
+    #   If it is false, :name is used. If it already exists, :name1.succ is used.
+    # @return [DataFrame] trnsposed DataFrame
+    def transpose(key: keys.first, new_key: :name)
+      raise DataFrameArgumentError, "Not include: #{key}" unless keys.include?(key)
+
+      # Find unused name
+      new_keys = self[key].to_a.map { |e| e.to_s.to_sym }
+      new_key = (:name1..).find { |k| !new_keys.include?(k) } if new_keys.include?(new_key)
+
+      hash = { new_key => (keys - [key]) }
+      i = keys.index(key)
+      each_row do |h|
+        k = h.values[i]
+        hash[k] = h.values - [k]
+      end
+      DataFrame.new(hash)
+    end
+
+    # Reshape wide DataFrame to a longer DataFrame.
+    #
+    # @param keep_keys [Array] keys to keep.
+    # @param name [Symbol, String] key of the column which is come **from values**.
+    # @param value [Symbol, String] key of the column which is come **from values**.
+    # @return [DataFrame] long DataFrame.
+    def to_long(*keep_keys, name: :name, value: :value)
+      not_included = keep_keys - keys
+      raise DataFrameArgumentError, "Not have keys #{not_included}" unless not_included.empty?
+
+      name = name.to_sym
+      raise DataFrameArgumentError, "Invalid key: #{name}" if keep_keys.include?(name)
+
+      value = value.to_sym
+      raise DataFrameArgumentError, "Invalid key: #{value}" if keep_keys.include?(value)
+
+      hash = Hash.new { |h, k| h[k] = [] }
+      l = keys.size - keep_keys.size
+      each_row do |row|
+        row.each do |k, v|
+          if keep_keys.include?(k)
+            hash[k].concat([v] * l)
+          else
+            hash[name] << k
+            hash[value] << v
+          end
+        end
+      end
+      DataFrame.new(hash)
+    end
+
+    # Reshape long DataFrame to a wide DataFrame.
+    #
+    # @param name [Symbol, String] key of the column which will be expanded **to key names**.
+    # @param value [Symbol, String] key of the column which will be expanded **to values**.
+    # @return [DataFrame] wide DataFrame.
+    def to_wide(name: :name, value: :value)
+      name = name.to_sym
+      raise DataFrameArgumentError, "Invalid key: #{name}" unless keys.include?(name)
+
+      value = value.to_sym
+      raise DataFrameArgumentError, "Invalid key: #{value}" unless keys.include?(value)
+
+      hash = Hash.new { |h, k| h[k] = {} }
+      keep_keys = keys - [name, value]
+      each_row do |row|
+        keeps, converts = row.partition { |k, _| keep_keys.include?(k) }
+        h = converts.to_h
+        hash[keeps.to_h][h[name].to_s.to_sym] = h[value]
+      end
+      ks = hash.first[0].keys + hash.first[1].keys
+      vs = hash.map { |k, v| k.values + v.values }.transpose
+      DataFrame.new(ks.zip(vs))
+    end
+  end
+end

data/lib/red_amber/data_frame_variable_operation.rb

@@ -44,64 +44,106 @@ module RedAmber
       raise DataFrameArgumentError, "Invalid argument #{args}"
     end
 
-    # rename variables to create new DataFrame
-    def rename(*args, &block)
-      renamer = args
+    # rename variables to create a new DataFrame
+    def rename(*renamer, &block)
       if block
-        raise DataFrameArgumentError, 'Must not specify both arguments and a block' unless args.empty?
+        raise DataFrameArgumentError, 'Must not specify both arguments and a block' unless renamer.empty?
 
-        renamer = instance_eval(&block)
+        renamer = [instance_eval(&block)]
       end
-      renamer = [renamer].flatten
-      return self if renamer.empty?
+      case renamer
+      in [] | [nil] | [{}] | [[]]
+        return self
+      in [Hash => key_pairs]
+      # noop
+      in [ (Symbol | String) => from, (Symbol | String) => to]
+        key_pairs = { from => to }
+      in [Array => array_in_array]
+        key_pairs = try_convert_to_hash(array_in_array)
+      in [Array, *] => array_in_array1
+        key_pairs = try_convert_to_hash(array_in_array1)
+      else
+        raise DataFrameArgumentError, "Invalid argument #{renamer}"
+      end
+      rename_by_hash(key_pairs)
+    end
 
-      return rename_by_hash([renamer].to_h) if renamer.size == 2 && sym_or_str?(renamer) # rename(from, to)
-      return rename_by_hash(renamer[0]) if renamer.one? && renamer[0].is_a?(Hash) # rename({from => to})
+    # assign variables to create a new DataFrame
+    def assign(*assigner, &block)
+      appender, fields, arrays = assign_update(*assigner, &block)
+      return self if appender.is_a?(DataFrame)
 
-      raise DataFrameArgumentError, "Invalid argument #{args}"
+      append_to_fields_and_arrays(appender, fields, arrays, append_to_left: false) unless appender.empty?
+
+      DataFrame.new(Arrow::Table.new(Arrow::Schema.new(fields), arrays))
+    end
+
+    def assign_left(*assigner, &block)
+      appender, fields, arrays = assign_update(*assigner, &block)
+      return self if appender.is_a?(DataFrame)
+
+      append_to_fields_and_arrays(appender, fields, arrays, append_to_left: true) unless appender.empty?
+
+      DataFrame.new(Arrow::Table.new(Arrow::Schema.new(fields), arrays))
     end
 
-    # assign variables to create new DataFrame
-    def assign(*args, &block)
-      assigner = args
+    private
+
+    def assign_update(*assigner, &block)
       if block
-        raise DataFrameArgumentError, 'Must not specify both arguments and a block' unless args.empty?
+        raise DataFrameArgumentError, 'Must not specify both arguments and a block' unless assigner.empty?
 
-        assigner = instance_eval(&block)
+        assigner = [instance_eval(&block)]
+      end
+      case assigner
+      in [] | [nil] | [{}] | [[]]
+        return self
+      in [Hash => key_array_pairs]
+      # noop
+      in [(Symbol | String) => key, (Vector | Array | Arrow::Array) => array]
+        key_array_pairs = { key => array }
+      in [Array => array_in_array]
+        key_array_pairs = try_convert_to_hash(array_in_array)
+      in [Array, *] => array_in_array1
+        key_array_pairs = try_convert_to_hash(array_in_array1)
+      else
+        raise DataFrameArgumentError, "Invalid argument #{assigner}"
       end
-      assigner = [assigner].flatten
-      return self if assigner.empty? || assigner == [nil]
-
-      raise DataFrameArgumentError, "Invalid argument #{args}" unless assigner.one? && assigner[0].is_a?(Hash)
 
       updater = {}
       appender = {}
-      assigner[0].each do |key, value|
+      key_array_pairs.each do |key, array|
         if keys.include? key
-          updater[key] = value
+          updater[key] = array
         else
-          appender[key] = value
+          appender[key] = array
         end
       end
-      fields, arrays = update_fields_and_arrays(updater)
-      append_to_fields_and_arrays(appender, fields, arrays) unless appender.empty?
-
-      DataFrame.new(Arrow::Table.new(Arrow::Schema.new(fields), arrays))
+      [appender, *update_fields_and_arrays(updater)]
     end
 
-    private
+    def try_convert_to_hash(array)
+      array.to_h
+    rescue TypeError
+      [array].to_h
+    rescue TypeError # rubocop:disable Lint/DuplicateRescueException
+      raise DataFrameArgumentError, "Invalid argument in Array #{array}"
+    end
 
     def rename_by_hash(key_pairs)
-      fields = keys.map do |key|
-        new_key = key_pairs[key]
-        if new_key
-          Arrow::Field.new(new_key.to_sym, @table[key].data_type)
-        else
-          @table.schema[key]
+      not_existing_keys = key_pairs.keys - keys
+      raise DataFrameArgumentError, "Not existing: #{not_existing_keys}" unless not_existing_keys.empty?
+
+      fields =
+        keys.map do |key|
+          new_key = key_pairs[key]
+          if new_key
+            Arrow::Field.new(new_key.to_sym, @table[key].data_type)
+          else
+            @table.schema[key]
+          end
         end
-      end
-      schema = Arrow::Schema.new(fields)
-      DataFrame.new(Arrow::Table.new(schema, @table.columns))
+      DataFrame.new(Arrow::Table.new(Arrow::Schema.new(fields), @table.columns))
     end
 
     def update_fields_and_arrays(updater)
@@ -120,13 +162,20 @@ module RedAmber
       [fields, arrays]
     end
 
-    def append_to_fields_and_arrays(appender, fields, arrays)
-      appender.each do |key, data|
+    def append_to_fields_and_arrays(appender, fields, arrays, append_to_left: false)
+      enum = append_to_left ? appender.reverse_each : appender.each
+      enum.each do |key, data|
         raise DataFrameArgumentError, "Data size mismatch (#{data.size} != #{size})" if data.size != size
 
         a = Arrow::Array.new(data.is_a?(Vector) ? data.to_a : data)
-        fields << Arrow::Field.new(key.to_sym, a.value_data_type)
-        arrays << Arrow::ChunkedArray.new([a])
+
+        if append_to_left
+          fields.unshift(Arrow::Field.new(key.to_sym, a.value_data_type))
+          arrays.unshift(Arrow::ChunkedArray.new([a]))
+        else
+          fields << Arrow::Field.new(key.to_sym, a.value_data_type)
+          arrays << Arrow::ChunkedArray.new([a])
+        end
       end
     end
 

data/lib/red_amber/group.rb

@@ -3,6 +3,10 @@
 module RedAmber
   # group class
   class Group
+    # Creates a new Group object.
+    #
+    # @param dataframe [DataFrame] dataframe to be grouped.
+    # @param group_keys [Array<>] keys for grouping.
     def initialize(dataframe, *group_keys)
       @dataframe = dataframe
       @table = @dataframe.table
@@ -50,7 +54,7 @@ module RedAmber
       raise GroupArgumentError, "#{d} is not a key of\n #{@dataframe}." unless summary_keys.empty? || d.empty?
 
       df = RedAmber::DataFrame.new(@group.send(func, *summary_keys))
-      df = df[df.keys[-1], df.keys[0...-1]]
+      df = df[@group_keys, df.keys - @group_keys]
       # if counts are the same (no nil included), aggregate count columns.
       df = df[df.keys[0..1]].rename(df.keys[1], :count) if func == :count && df.to_h.values[1..].uniq.size == 1
       df

data/lib/red_amber/vector_functions.rb

@@ -39,9 +39,53 @@ module RedAmber
 
     # Returns other than value
     # - mode
-    # - quantile
     # - tdigest
 
+    # Return quantile
+    #   0.5 quantile (median) is returned by default.
+    #   Or return quantile for specified probability (prob).
+    #   If quantile lies between two data points, interpolated value is
+    #   returned based on selected interpolation method.
+    #   Nils and NaNs are ignored.
+    #   Nil is returned if there are no valid data point.
+    #
+    # @param prob [Float] probability.
+    # @param interpolation [Symbol] specifies interpolation method to use,
+    #   when the quantile lies between the data i and j.
+    #   - Default value is :linear, which returns i + (j - i) * fraction.
+    #   - :lower returns i.
+    #   - :higher returns j.
+    #   - :nearest returns i or j, whichever is closer.
+    #   - :midpoint returns (i + j) / 2.
+    # @param skip_nils [Boolean] wheather to ignore nil.
+    # @param min_count [Integer] min count.
+    # @return [Float] quantile.
+    def quantile(prob = 0.5, interpolation: :linear, skip_nils: true, min_count: 0)
+      raise VectorArgumentError, "Invalid: probability #{prob} must be between 0 and 1" unless (0..1).cover? prob
+
+      datum = find(:quantile).execute([data],
+                                      q: prob,
+                                      interpolation: interpolation,
+                                      skip_nulls: skip_nils,
+                                      min_count: min_count)
+      datum.value.to_a.first
+    end
+
+    # Return quantiles in a DataFrame
+    #
+    def quantiles(probs = [1.0, 0.75, 0.5, 0.25, 0.0], interpolation: :linear, skip_nils: true, min_count: 0)
+      if probs.empty? || !probs.all? { |q| (0..1).cover?(q) }
+        raise VectorArgumentError, "Invarid probavilities #{probs}"
+      end
+
+      DataFrame.new(
+        probs: probs,
+        quantiles: probs.map do |q|
+          quantile(q, interpolation: interpolation, skip_nils: skip_nils, min_count: min_count)
+        end
+      )
+    end
+
     # [Unary element-wise]: vector.func => vector
     unary_element_wise =
       %i[abs array_sort_indices atan bit_wise_not ceil cos fill_null_backward fill_null_forward floor is_finite
@@ -63,6 +107,7 @@ module RedAmber
 
     alias_method :sort_indexes, :array_sort_indices
     alias_method :sort_indices, :array_sort_indices
+    alias_method :sort_index, :array_sort_indices
 
     alias_method :uniq, :unique
 

data/lib/red_amber/vector_selectable.rb

@@ -111,7 +111,7 @@ module RedAmber
 
       index_array = Arrow::UInt64ArrayBuilder.build(normalized_indices.data) # round to integer array
 
-      datum = find(:array_take).execute([data, index_array])
+      datum = find(:take).execute([data, index_array]) # :array_take will fail with ChunkedArray
       Vector.new(datum.value)
     end
 

data/lib/red_amber/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RedAmber
-  VERSION = '0.1.8'
+  VERSION = '0.2.0'
 end

data/lib/red_amber.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
 require 'arrow'
-require 'rover-df'
 
 require_relative 'red_amber/helper'
 require_relative 'red_amber/data_frame_displayable'
 require_relative 'red_amber/data_frame_indexable'
+require_relative 'red_amber/data_frame_reshaping'
 require_relative 'red_amber/data_frame_selectable'
 require_relative 'red_amber/data_frame_variable_operation'
 require_relative 'red_amber/data_frame'

data/red_amber.gemspec

@@ -30,7 +30,7 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'red-arrow', '>= 8.0.0'
+  spec.add_dependency 'red-arrow', '>= 9.0.0'
 
   # Development dependency has gone to the Gemfile (rubygems/bundler#7237)
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red_amber
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.2.0
 platform: ruby
 authors:
 - Hirokazu SUZUKI (heronshoes)
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-08-03 00:00:00.000000000 Z
+date: 2022-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: red-arrow
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 8.0.0
+        version: 9.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 8.0.0
+        version: 9.0.0
 description: RedAmber is a simple dataframe library inspired by Rover-df and powered
   by Red Arrow.
 email:
@@ -69,6 +69,7 @@ files:
 - lib/red_amber/data_frame.rb
 - lib/red_amber/data_frame_displayable.rb
 - lib/red_amber/data_frame_indexable.rb
+- lib/red_amber/data_frame_reshaping.rb
 - lib/red_amber/data_frame_selectable.rb
 - lib/red_amber/data_frame_variable_operation.rb
 - lib/red_amber/group.rb