red_amber 0.1.7 → 0.2.1

data/lib/red_amber/data_frame.rb

@@ -1,39 +1,57 @@
 # frozen_string_literal: true
 
 module RedAmber
-  # data frame class
-  #   @table   : holds Arrow::Table object
+  # Class to represent a data frame.
+  # Variable @table holds an Arrow::Table object.
   class DataFrame
     # mix-in
     include DataFrameDisplayable
     include DataFrameIndexable
+    include DataFrameReshaping
     include DataFrameSelectable
     include DataFrameVariableOperation
     include Helper
 
+    # Creates a new RedAmber::DataFrame.
+    #
+    # @overload initialize(hash)
+    #
+    #   @params hash [Hash]
+    #
+    # @overload initialize(table)
+    #
+    #   @params table [Arrow::Table]
+    #
+    # @overload initialize(dataframe)
+    #
+    #   @params dataframe [RedAmber::DataFrame, Rover::DataFrame]
+    #
+    # @overload initialize(null)
+    #
+    #   @params null [NilClass] No arguments.
+    #
     def initialize(*args)
       @variables = @keys = @vectors = @types = @data_types = nil
-      # bug in gobject-introspection: ruby-gnome/ruby-gnome#1472
-      #  [Arrow::Table] == [nil] shows ArgumentError
-      #  temporary use yoda condition to workaround
-      if args.empty? || args == [[]] || args == [{}] || [nil] == args
+      case args
+      in nil | [nil] | [] | {} | [[]] | [{}]
         # DataFrame.new, DataFrame.new([]), DataFrame.new({}), DataFrame.new(nil)
         #   returns empty DataFrame
         @table = Arrow::Table.new({}, [])
-      elsif args.size > 1
-        @table = Arrow::Table.new(*args)
+      in [Arrow::Table => table]
+        @table = table
+      in [DataFrame => dataframe]
+        @table = dataframe.table
+      in [rover_or_hash]
+        begin
+          # Accepts Rover::DataFrame or Hash
+          @table = Arrow::Table.new(rover_or_hash.to_h)
+        rescue StandardError
+          raise DataFrameTypeError, "invalid argument: #{rover_or_hash}"
+        end
       else
-        arg = args[0]
-        @table =
-          case arg
-          when Arrow::Table then arg
-          when DataFrame then arg.table
-          when Rover::DataFrame then Arrow::Table.new(arg.to_h)
-          when Hash then Arrow::Table.new(arg)
-          else
-            raise DataFrameTypeError, "invalid argument: #{arg}"
-          end
+        @table = Arrow::Table.new(*args)
       end
+      name_unnamed_keys
     end
 
     def self.load(path, options = {})
@@ -50,58 +68,110 @@ module RedAmber
       @table.save(output, options)
     end
 
+    # Returns the number of rows.
+    #
+    # @return [Integer] Number of rows.
     def size
       @table.n_rows
     end
     alias_method :n_rows, :size
     alias_method :n_obs, :size
 
+    # Returns the number of columns.
+    #
+    # @return [Integer] Number of columns.
     def n_keys
       @table.n_columns
     end
     alias_method :n_cols, :n_keys
     alias_method :n_vars, :n_keys
 
+    # Returns the numbers of rows and columns.
+    #
+    # @return [Array]
+    #   Number of rows and number of columns in an array.
+    #   Same as [size, n_keys].
     def shape
       [size, n_keys]
     end
 
+    # Returns a Hash of key and Vector pairs in the columns.
+    #
+    # @return [Hash]
+    #   key => Vector pairs for each columns.
     def variables
       @variables || @variables = init_instance_vars(:variables)
     end
     alias_method :vars, :variables
 
+    # Returns an Array of keys.
+    #
+    # @return [Array]
+    #   Keys in an Array.
     def keys
       @keys || @keys = init_instance_vars(:keys)
     end
     alias_method :column_names, :keys
     alias_method :var_names, :keys
 
+    # Returns true if self has a specified key in the argument.
+    #
+    # @param key [Symbol, String] Key to test.
+    # @return [Boolean]
+    #   Returns true if self has key in Symbol.
     def key?(key)
-      @keys.include?(key.to_sym)
+      keys.include?(key.to_sym)
     end
     alias_method :has_key?, :key?
 
+    # Returns index of specified key in the Array keys.
+    #
+    # @param key [Symbol, String] key to know.
+    # @return [Integer]
+    #   Index of key in the Array keys.
     def key_index(key)
-      @keys.find_index(key.to_sym)
+      keys.find_index(key.to_sym)
     end
     alias_method :find_index, :key_index
     alias_method :index, :key_index
 
+    # Returns abbreviated type names in an Array.
+    #
+    # @return [Array]
+    #   Abbreviated Red Arrow data type names.
     def types
       @types || @types = @table.columns.map { |column| column.data.value_type.nick.to_sym }
     end
 
+    # Returns an Array of Classes of data type.
+    #
+    # @return [Array]
+    #   An Array of Red Arrow data type Classes.
     def type_classes
       @data_types || @data_types = @table.columns.map { |column| column.data_type.class }
     end
 
+    # Returns Vectors in an Array.
+    #
+    # @return [Array]
+    #   An Array of RedAmber::Vector s.
     def vectors
       @vectors || @vectors = init_instance_vars(:vectors)
     end
 
-    def indices
-      (0...size).to_a
+    # Returns row indices (start...(size+start)) in an Array.
+    #
+    # @param start [Object]
+    #   Object which have #succ method.
+    # @return [Array]
+    #   An Array of indices of the row.
+    # @example
+    #   (when self.size == 5)
+    #   - indices #=> [0, 1, 2, 3, 4]
+    #   - indices(1) #=> [1, 2, 3, 4, 5]
+    #   - indices('a') #=> ['a', 'b', 'c', 'd', 'e']
+    def indices(start = 0)
+      (start..).take(size)
     end
     alias_method :indexes, :indices
 
@@ -128,6 +198,18 @@ module RedAmber
       variables.empty?
     end
 
+    def each_row
+      return enum_for(:each_row) unless block_given?
+
+      size.times do |i|
+        key_row_pairs =
+          vectors.each_with_object({}) do |v, h|
+            h[v.key] = v.data[i]
+          end
+        yield key_row_pairs
+      end
+    end
+
     def to_rover
       require 'rover'
       Rover::DataFrame.new(to_h)
@@ -144,8 +226,22 @@ module RedAmber
       end
     end
 
-    def group(*group_keys)
-      Group.new(self, group_keys)
+    def group(*group_keys, &block)
+      g = Group.new(self, group_keys)
+      g = g.summarize(&block) if block
+      g
+    end
+
+    def method_missing(name, *args, &block)
+      return v(name) if args.empty?
+
+      super
+    end
+
+    def respond_to_missing?(name, include_private)
+      return true if key?(name)
+
+      super
     end
 
     private
@@ -182,5 +278,23 @@ module RedAmber
       html = IRuby::HTML.table(converted.to_h, maxrows: 8, maxcols: 15)
       "#{self.class} <#{size} x #{n_keys} vector#{pl(n_keys)}> #{html}"
     end
+
+    def name_unnamed_keys
+      return unless @table[:'']
+
+      # We can't use #keys because it causes mismatch of @table and @keys
+      keys = @table.schema.fields.map { |f| f.name.to_sym }
+      unnamed = (:unnamed1..).find { |e| !keys.include?(e) }
+      fields =
+        @table.schema.fields.map do |field|
+          if field.name.empty?
+            Arrow::Field.new(unnamed, field.data_type)
+          else
+            field
+          end
+        end
+      schema = Arrow::Schema.new(fields)
+      @table = Arrow::Table.new(schema, @table.columns)
+    end
   end
 end

data/lib/red_amber/data_frame_displayable.rb

@@ -5,15 +5,36 @@ require 'stringio'
 module RedAmber
   # mix-ins for the class DataFrame
   module DataFrameDisplayable
-    def to_s
+    INDEX_KEY = :index_key_for_format_table
+
+    def to_s(width: 80)
       return '' if empty?
 
-      format_table(width: 80)
+      format_table(width: width)
     end
 
-    # def describe() end
-
-    # def summary() end
+    # Show statistical summary by a new DatFrame.
+    #   Make stats for numeric columns only.
+    #   NaNs are ignored.
+    #   Counts also show non-NaN counts.
+    #
+    # @return [DataFrame] a new dataframe.
+    def summary
+      num_keys = keys.select { |key| self[key].numeric? }
+
+      DataFrame.new(
+        variables: num_keys,
+        count: num_keys.map { |k| self[k].count },
+        mean: num_keys.map { |k| self[k].mean },
+        std: num_keys.map { |k| self[k].std },
+        min: num_keys.map { |k| self[k].min },
+        '25%': num_keys.map { |k| self[k].quantile(0.25) },
+        median: num_keys.map { |k| self[k].median },
+        '75%': num_keys.map { |k| self[k].quantile(0.75) },
+        max: num_keys.map { |k| self[k].max }
+      )
+    end
+    alias_method :describe, :summary
 
     def inspect
       if ENV.fetch('RED_AMBER_OUTPUT_MODE', 'Table') == 'TDR'
@@ -131,15 +152,11 @@ module RedAmber
       a
     end
 
-    def format_table(width: 80)
-      head = 5
-      tail = 3
-      n_digit = 1
-
+    def format_table(width: 80, head: 5, tail: 3, n_digit: 2)
       original = self
-      indices = size > head + tail ? [*0...head, *(size - tail)...size] : [*0...size]
+      indices = size > head + tail ? [*0..head, *(size - tail)...size] : [*0...size]
       df = slice(indices).assign do
-        assigner = { '': indices.map { |i| (i + 1).to_s } }
+        assigner = { INDEX_KEY => indices.map { |i| (i + 1).to_s } }
         vectors.each_with_object(assigner) do |v, a|
           a[v.key] = v.to_a.map do |e|
             if e.nil?
@@ -155,13 +172,13 @@ module RedAmber
         end
       end
 
-      df = df.pick { [keys[-1], keys[0..-2]] }
-      df = size > head + tail ? df[0, 0, 0...head, 0, -tail..-1] : df[0, 0, 0..-1]
+      df = df.pick { [INDEX_KEY, keys - [INDEX_KEY]] }
+      df = size > head + tail ? df[0, 0, 0..head, -tail..-1] : df[0, 0, 0..-1]
       df = df.assign do
         vectors.each_with_object({}) do |v, assigner|
-          vec = v.replace(0, v.key.to_s)
-                 .replace(1, v.key == :'' ? '' : "<#{original[v.key].type}>")
-          assigner[v.key] = size > head + tail ? vec.replace(head + 2, ':') : vec
+          vec = v.replace(0, v.key == INDEX_KEY ? '' : v.key.to_s)
+                 .replace(1, v.key == INDEX_KEY ? '' : "<#{original[v.key].type}>")
+          assigner[v.key] = original.size > head + tail + 1 ? vec.replace(head + 2, ':') : vec
         end
       end
 
@@ -197,7 +214,7 @@ module RedAmber
     end
 
     def format_for_column(vector, original, width)
-      if vector.key != :'' && !original[vector.key].numeric?
+      if vector.key != INDEX_KEY && !original[vector.key].numeric?
         "%-#{width}s"
       else
         "%#{width}s"

data/lib/red_amber/data_frame_reshaping.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module RedAmber
+  # mix-ins for the class DataFrame
+  module DataFrameReshaping
+    # Transpose a wide DataFrame.
+    #
+    # @param key [Symbol] key of the index column
+    #   to transepose into keys.
+    #   If it is not specified, keys[0] is used.
+    # @param new_key [Symbol] key name of transposed index column.
+    #   If it is not specified, :N is used. If it already exists, :N1 or :N1.succ is used.
+    # @return [DataFrame] trnsposed DataFrame
+    def transpose(key: keys.first, name: :N)
+      raise DataFrameArgumentError, "Self does not include: #{key}" unless keys.include?(key)
+
+      # Find unused name
+      new_keys = self[key].to_a.map { |e| e.to_s.to_sym }
+      name = (:N1..).find { |k| !new_keys.include?(k) } if new_keys.include?(name)
+
+      hash = { name => (keys - [key]) }
+      i = keys.index(key)
+      each_row do |h|
+        k = h.values[i]
+        hash[k] = h.values - [k]
+      end
+      DataFrame.new(hash)
+    end
+
+    # Reshape wide DataFrame to a longer DataFrame.
+    #
+    # @param keep_keys [Array] keys to keep.
+    # @param name [Symbol, String] key of the column which is come **from values**.
+    # @param value [Symbol, String] key of the column which is come **from values**.
+    # @return [DataFrame] long DataFrame.
+    def to_long(*keep_keys, name: :N, value: :V)
+      not_included = keep_keys - keys
+      raise DataFrameArgumentError, "Not have keys #{not_included}" unless not_included.empty?
+
+      name = name.to_sym
+      raise DataFrameArgumentError, "Invalid key: #{name}" if keep_keys.include?(name)
+
+      value = value.to_sym
+      raise DataFrameArgumentError, "Invalid key: #{value}" if keep_keys.include?(value)
+
+      hash = Hash.new { |h, k| h[k] = [] }
+      l = keys.size - keep_keys.size
+      each_row do |row|
+        row.each do |k, v|
+          if keep_keys.include?(k)
+            hash[k].concat([v] * l)
+          else
+            hash[name] << k
+            hash[value] << v
+          end
+        end
+      end
+      DataFrame.new(hash)
+    end
+
+    # Reshape long DataFrame to a wide DataFrame.
+    #
+    # @param name [Symbol, String] key of the column which will be expanded **to key names**.
+    # @param value [Symbol, String] key of the column which will be expanded **to values**.
+    # @return [DataFrame] wide DataFrame.
+    def to_wide(name: :N, value: :V)
+      name = name.to_sym
+      raise DataFrameArgumentError, "Invalid key: #{name}" unless keys.include?(name)
+
+      value = value.to_sym
+      raise DataFrameArgumentError, "Invalid key: #{value}" unless keys.include?(value)
+
+      hash = Hash.new { |h, k| h[k] = {} }
+      keep_keys = keys - [name, value]
+      each_row do |row|
+        keeps, converts = row.partition { |k, _| keep_keys.include?(k) }
+        h = converts.to_h
+        hash[keeps.to_h][h[name].to_s.to_sym] = h[value]
+      end
+      ks = hash.first[0].keys + hash.first[1].keys
+      vs = hash.map { |k, v| k.values + v.values }.transpose
+      DataFrame.new(ks.zip(vs))
+    end
+  end
+end

data/lib/red_amber/data_frame_selectable.rb

@@ -3,8 +3,8 @@
 module RedAmber
   # mix-in for the class DataFrame
   module DataFrameSelectable
-    # select variables: [symbol] or [string]
-    # select observations: [array of index], [range]
+    # select columns: [symbol] or [string]
+    # select rows: [array of index], [range]
     def [](*args)
       args.flatten!
       raise DataFrameArgumentError, 'Empty dataframe' if empty?
@@ -22,17 +22,17 @@ module RedAmber
       raise DataFrameArgumentError, "Invalid argument: #{args}"
     end
 
-    # slice and select some observations to create sub DataFrame
+    # slice and select rows to create sub DataFrame
     def slice(*args, &block)
       slicer = args
       if block
         raise DataFrameArgumentError, 'Must not specify both arguments and block.' unless args.empty?
 
-        slicer = instance_eval(&block)
+        slicer = [instance_eval(&block)]
       end
-      slicer = [slicer].flatten
+      slicer.flatten!
 
-      raise DataFrameArgumentError, 'Empty dataframe' if empty?
+      raise DataFrameArgumentError, 'Self is an empty dataframe' if empty?
       return remove_all_values if slicer.empty? || slicer[0].nil?
 
       vector = parse_to_vector(slicer)
@@ -46,15 +46,59 @@ module RedAmber
       raise DataFrameArgumentError, "Invalid argument #{slicer}"
     end
 
-    # remove selected observations to create sub DataFrame
+    def slice_by(key, keep_key: false, &block)
+      raise DataFrameArgumentError, 'Self is an empty dataframe' if empty?
+      raise DataFrameArgumentError, 'No block given' unless block
+      raise DataFrameArgumentError, "#{key} is no a key of self" unless key?(key)
+      return self if key.nil?
+
+      slicer = instance_eval(&block)
+      return DataFrame.new unless slicer
+
+      if slicer.is_a?(Range)
+        from = slicer.begin
+        from =
+          if from.is_a?(String)
+            self[key].index(from)
+          elsif from.nil?
+            0
+          elsif from < 0
+            size + from
+          else
+            from
+          end
+        to = slicer.end
+        to =
+          if to.is_a?(String)
+            self[key].index(to)
+          elsif to.nil?
+            size - 1
+          elsif to < 0
+            size + to
+          else
+            to
+          end
+        slicer = (from..to).to_a
+      else
+        slicer = slicer.map { |x| x.is_a?(String) ? self[key].index(x) : x }
+      end
+
+      if keep_key
+        take(slicer)
+      else
+        take(slicer).drop(key)
+      end
+    end
+
+    # remove selected rows to create remainer DataFrame
     def remove(*args, &block)
       remover = args
       if block
         raise DataFrameArgumentError, 'Must not specify both arguments and block.' unless args.empty?
 
-        remover = instance_eval(&block)
+        remover = [instance_eval(&block)]
       end
-      remover = [remover].flatten
+      remover.flatten!
 
       raise DataFrameArgumentError, 'Empty dataframe' if empty?
       return self if remover.empty? || remover[0].nil?