red_amber 0.2.1 → 0.2.3

data/lib/red_amber/data_frame_displayable.rb

@@ -37,8 +37,12 @@ module RedAmber
     alias_method :describe, :summary
 
     def inspect
-      if ENV.fetch('RED_AMBER_OUTPUT_MODE', 'Table') == 'TDR'
+      mode = ENV.fetch('RED_AMBER_OUTPUT_MODE', 'Table')
+      case mode.upcase
+      when 'TDR'
         "#<#{shape_str(with_id: true)}>\n#{dataframe_info(3)}"
+      when 'MINIMUM'
+        shape_str
       else
         "#<#{shape_str(with_id: true)}>\n#{self}"
       end
@@ -55,6 +59,23 @@ module RedAmber
       "#{shape_str}\n#{dataframe_info(limit, tally_level: tally, max_element: elements)}"
     end
 
+    def to_iruby
+      require 'iruby'
+      return ['text/plain', '(empty DataFrame)'] if empty?
+
+      mode = ENV.fetch('RED_AMBER_OUTPUT_MODE', 'Table')
+      case mode.upcase
+      when 'PLAIN'
+        ['text/plain', inspect]
+      when 'MINIMUM'
+        ['text/plain', shape_str]
+      when 'TDR'
+        size <= 5 ? ['text/plain', tdr_str(tally: 0)] : ['text/plain', tdr_str]
+      else # 'TABLE'
+        ['text/html', html_table]
+      end
+    end
+
     private # =====
 
     def shape_str(with_id: false)
@@ -98,7 +119,7 @@ module RedAmber
             else
               [shorthand(vector, size, max_element)]
             end
-        sio.printf header_format, i + 1, key, type, data_tally.size, a.join(', ')
+        sio.printf header_format, i, key, type, data_tally.size, a.join(', ')
       end
       sio.string
     end
@@ -153,10 +174,12 @@ module RedAmber
     end
 
     def format_table(width: 80, head: 5, tail: 3, n_digit: 2)
+      return "  #{keys.join(' ')}\n  (Empty Vectors)\n" if size.zero?
+
       original = self
       indices = size > head + tail ? [*0..head, *(size - tail)...size] : [*0...size]
       df = slice(indices).assign do
-        assigner = { INDEX_KEY => indices.map { |i| (i + 1).to_s } }
+        assigner = { INDEX_KEY => indices.map(&:to_s) }
         vectors.each_with_object(assigner) do |v, a|
           a[v.key] = v.to_a.map do |e|
             if e.nil?
@@ -220,5 +243,37 @@ module RedAmber
         "%#{width}s"
       end
     end
+
+    def html_table
+      reduced = size > 8 ? self[0..4, -4..-1] : self
+
+      converted = reduced.assign do
+        vectors.select.with_object({}) do |vector, assigner|
+          assigner[vector.key] = vector.map do |element|
+            case element
+            in TrueClass
+              '<i>(true)</i>'
+            in FalseClass
+              '<i>(false)</i>'
+            in NilClass
+              '<i>(nil)</i>'
+            in ''
+              '""'
+            in String
+              element.sub(/^(\s+)$/, '"\1"') # blank spaces
+            in Float
+              format('%g', element)
+            in Integer
+              format('%d', element)
+            else
+              element
+            end
+          end
+        end
+      end
+
+      html = IRuby::HTML.table(converted.to_h, maxrows: 8, maxcols: 15)
+      "#{self.class} <#{size} x #{n_keys} vector#{pl(n_keys)}> #{html}"
+    end
   end
 end

data/lib/red_amber/data_frame_loadsave.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module RedAmber
+  # mix-ins for the class DataFrame
+  module DataFrameLoadSave
+    # Enable `self.load` as class method of DataFrame
+    def self.included(klass)
+      klass.extend ClassMethods
+    end
+
+    # Enable `self.load` as class method of DataFrame
+    module ClassMethods
+      # Load DataFrame via Arrow::Table.load
+      def load(path, options = {})
+        DataFrame.new(Arrow::Table.load(path, options))
+      end
+    end
+
+    # Save DataFrame
+    def save(output, options = {})
+      @table.save(output, options)
+    end
+
+    # Save and reload to cast automatically
+    #   Via tsv format file temporally as default
+    #
+    #   experimental feature
+    def auto_cast(format: :tsv)
+      return self if empty?
+
+      tempfile = Arrow::ResizableBuffer.new(1024)
+      save(tempfile, format: format)
+      DataFrame.load(tempfile, format: format)
+    end
+  end
+end

data/lib/red_amber/data_frame_reshaping.rb

@@ -9,16 +9,17 @@ module RedAmber
     #   to transepose into keys.
     #   If it is not specified, keys[0] is used.
     # @param new_key [Symbol] key name of transposed index column.
-    #   If it is not specified, :N is used. If it already exists, :N1 or :N1.succ is used.
+    #   If it is not specified, :NAME is used. If it already exists, :NAME1 or :NAME1.succ is used.
     # @return [DataFrame] trnsposed DataFrame
-    def transpose(key: keys.first, name: :N)
+    def transpose(key: keys.first, name: :NAME)
       raise DataFrameArgumentError, "Self does not include: #{key}" unless keys.include?(key)
 
       # Find unused name
       new_keys = self[key].to_a.map { |e| e.to_s.to_sym }
-      name = (:N1..).find { |k| !new_keys.include?(k) } if new_keys.include?(name)
+      name = (:NAME1..).find { |k| !new_keys.include?(k) } if new_keys.include?(name)
 
-      hash = { name => (keys - [key]) }
+      names = (keys - [key]).map { |x| x&.to_s }
+      hash = { name => names }
       i = keys.index(key)
       each_row do |h|
         k = h.values[i]
@@ -33,7 +34,7 @@ module RedAmber
     # @param name [Symbol, String] key of the column which is come **from values**.
     # @param value [Symbol, String] key of the column which is come **from values**.
     # @return [DataFrame] long DataFrame.
-    def to_long(*keep_keys, name: :N, value: :V)
+    def to_long(*keep_keys, name: :NAME, value: :VALUE)
       not_included = keep_keys - keys
       raise DataFrameArgumentError, "Not have keys #{not_included}" unless not_included.empty?
 
@@ -55,6 +56,7 @@ module RedAmber
           end
         end
       end
+      hash[name] = hash[name].map { |x| x&.to_s }
       DataFrame.new(hash)
     end
 
@@ -63,7 +65,7 @@ module RedAmber
     # @param name [Symbol, String] key of the column which will be expanded **to key names**.
     # @param value [Symbol, String] key of the column which will be expanded **to values**.
     # @return [DataFrame] wide DataFrame.
-    def to_wide(name: :N, value: :V)
+    def to_wide(name: :NAME, value: :VALUE)
       name = name.to_sym
       raise DataFrameArgumentError, "Invalid key: #{name}" unless keys.include?(name)
 

data/lib/red_amber/data_frame_selectable.rb

@@ -17,7 +17,7 @@ module RedAmber
         raise DataFrameArgumentError, "Size is not match in booleans: #{args}"
       end
       return take_by_array(vector) if vector.numeric?
-      return select_vars_by_keys(vector.to_a.map(&:to_sym)) if vector.string? || vector.type == :dictionary
+      return select_vars_by_keys(vector.to_a.map(&:to_sym)) if vector.string? || vector.dictionary?
 
       raise DataFrameArgumentError, "Invalid argument: #{args}"
     end
@@ -118,10 +118,10 @@ module RedAmber
         end
 
         normalized_indices = normalized_indices.floor.to_a.map(&:to_i) # round to integer array
-        return remove_all_values if normalized_indices == indices
+        return remove_all_values if normalized_indices == indices.to_a
         return self if normalized_indices.empty?
 
-        index_array = indices - normalized_indices
+        index_array = indices.to_a - normalized_indices
 
         datum = Arrow::Function.find(:take).execute([table, index_array])
         return DataFrame.new(datum.value)
@@ -168,14 +168,14 @@ module RedAmber
 
     # Undocumented
     # TODO: support for option {boundscheck: true}
-    def take(*indices)
-      indices.flatten!
-      return remove_all_values if indices.empty?
+    def take(*arg_indices)
+      arg_indices.flatten!
+      return remove_all_values if arg_indices.empty?
 
-      indices = indices[0] if indices.one? && !indices[0].is_a?(Numeric)
-      indices = Vector.new(indices) unless indices.is_a?(Vector)
+      arg_indices = arg_indices[0] if arg_indices.one? && !arg_indices[0].is_a?(Numeric)
+      arg_indices = Vector.new(arg_indices) unless arg_indices.is_a?(Vector)
 
-      take_by_array(indices)
+      take_by_array(arg_indices)
     end
 
     # Undocumented

data/lib/red_amber/data_frame_variable_operation.rb

@@ -15,16 +15,22 @@ module RedAmber
       return DataFrame.new if picker.empty? || picker == [nil]
 
       key_vector = Vector.new(keys)
-      picker_vector = parse_to_vector(picker)
-
-      picker = key_vector.filter(*picker_vector).to_a if picker_vector.boolean?
-      picker = key_vector.take(*picker_vector).to_a if picker_vector.numeric?
+      vec = parse_to_vector(picker, vsize: n_keys)
+
+      ary =
+        if vec.boolean?
+          key_vector.filter(*vec).to_a
+        elsif vec.numeric?
+          key_vector.take(*vec).to_a
+        elsif vec.string? || vec.dictionary?
+          vec.to_a
+        else
+          raise DataFrameArgumentError, "Invalid argument #{args}"
+        end
 
-      # DataFrame#[] creates a Vector with single key is specified.
+      # DataFrame#[] creates a Vector if single key is specified.
       # DataFrame#pick creates a DataFrame with single key.
-      return DataFrame.new(@table[picker]) if sym_or_str?(picker)
-
-      raise DataFrameArgumentError, "Invalid argument #{args}"
+      DataFrame.new(@table[ary])
     end
 
     # drop some variables to create remainer sub DataFrame
@@ -38,24 +44,24 @@ module RedAmber
       dropper.flatten!
 
       key_vector = Vector.new(keys)
-      dropper_vector = parse_to_vector(dropper)
-
-      picker =
-        if dropper_vector.boolean?
-          key_vector.filter(*dropper_vector.primitive_invert).each.map(&:to_sym)
-        elsif dropper_vector.numeric?
-          keys - key_vector.take(*dropper_vector).each.map(&:to_sym)
+      vec = parse_to_vector(dropper, vsize: n_keys)
+
+      ary =
+        if vec.boolean?
+          key_vector.filter(*vec.primitive_invert).each.map(&:to_sym) # Array
+        elsif vec.numeric?
+          keys - key_vector.take(*vec).each.map(&:to_sym) # Array
+        elsif vec.string? || vec.dictionary?
+          keys - vec.to_a.map { _1&.to_sym } # Array
         else
-          keys - dropper
+          raise DataFrameArgumentError, "Invalid argument #{args}"
         end
 
-      return DataFrame.new if picker.empty?
+      return DataFrame.new if ary.empty?
 
-      # DataFrame#[] creates a Vector with single key is specified.
+      # DataFrame#[] creates a Vector if single key is specified.
       # DataFrame#drop creates a DataFrame with single key.
-      return DataFrame.new(@table[picker]) if sym_or_str?(picker)
-
-      raise DataFrameArgumentError, "Invalid argument #{args}"
+      DataFrame.new(@table[ary])
     end
 
     # rename variables to create a new DataFrame

data/lib/red_amber/group.rb

@@ -3,35 +3,84 @@
 module RedAmber
   # group class
   class Group
+    include Enumerable # This feature is experimental
+
     # Creates a new Group object.
     #
     # @param dataframe [DataFrame] dataframe to be grouped.
     # @param group_keys [Array<>] keys for grouping.
     def initialize(dataframe, *group_keys)
       @dataframe = dataframe
-      @table = @dataframe.table
       @group_keys = group_keys.flatten
 
-      raise GroupArgumentError, 'group_keys is empty.' if @group_keys.empty?
+      raise GroupArgumentError, 'group_keys are empty.' if @group_keys.empty?
 
       d = @group_keys - @dataframe.keys
       raise GroupArgumentError, "#{d} is not a key of\n #{@dataframe}." unless d.empty?
 
-      @group = @table.group(*@group_keys)
+      @filters = @group_counts = @base_table = nil
+      @group = @dataframe.table.group(*@group_keys)
     end
 
+    attr_reader :dataframe, :group_keys
+
     functions = %i[count sum product mean min max stddev variance]
     functions.each do |function|
       define_method(function) do |*summary_keys|
-        by(function, summary_keys)
+        summary_keys = Array(summary_keys).flatten
+        d = summary_keys - @dataframe.keys
+        raise GroupArgumentError, "#{d} is not a key of\n #{@dataframe}." unless summary_keys.empty? || d.empty?
+
+        table = @group.aggregate(*build_aggregation_keys("hash_#{function}", summary_keys))
+        df = DataFrame.new(table)
+        df.pick(@group_keys, df.keys - @group_keys)
       end
     end
 
-    def inspect
-      tallys = @dataframe.pick(@group_keys).vectors.map.with_object({}) do |v, h|
-        h[v.key] = v.tally
+    alias_method :__count, :count
+    private :__count
+
+    def count(*summary_keys)
+      df = __count(summary_keys)
+      # if counts are the same (and do not include NaN or nil), aggregate count columns.
+      if df.pick(@group_keys.size..).to_h.values.uniq.size == 1
+        df.pick(0..@group_keys.size).rename { [keys[-1], :count] }
+      else
+        df
       end
-      "#<#{self.class}:#{format('0x%016x', object_id)}\n#{tallys}>"
+    end
+
+    def filters
+      @filters ||= begin
+        first, *others = @group_keys.map do |key|
+          vector = @dataframe[key]
+          vector.uniq.each.map { |u| u.nil? ? vector.is_nil : vector == u }
+        end
+
+        if others.empty?
+          first.select(&:any?)
+        else
+          first.product(*others).map { |a| a.reduce(&:&) }.select(&:any?)
+        end
+      end
+    end
+
+    def each
+      filters
+      return enum_for(:each) unless block_given?
+
+      @filters.each do |filter|
+        yield @dataframe[filter]
+      end
+      @filters.size
+    end
+
+    def group_count
+      DataFrame.new(add_columns_to_table(base_table, [:group_count], [group_counts]))
+    end
+
+    def inspect
+      "#<#{self.class} : #{format('0x%016x', object_id)}>\n#{group_count}"
     end
 
     def summarize(&block)
@@ -48,16 +97,50 @@ module RedAmber
 
     private
 
-    def by(func, summary_keys)
-      summary_keys = Array(summary_keys).flatten
-      d = summary_keys - @dataframe.keys
-      raise GroupArgumentError, "#{d} is not a key of\n #{@dataframe}." unless summary_keys.empty? || d.empty?
+    def build_aggregation_keys(function_name, summary_keys)
+      if summary_keys.empty?
+        [function_name]
+      else
+        summary_keys.map { |key| "#{function_name}(#{key})" }
+      end
+    end
+
+    # @group_counts.sum == @dataframe.size
+    def group_counts
+      @group_counts ||= filters.map(&:sum)
+    end
+
+    def base_table
+      @base_table ||= begin
+        indexes = filters.map { |filter| filter.index(true) }
+        @dataframe.table[@group_keys].take(indexes)
+      end
+    end
+
+    def add_columns_to_table(table, keys, data_arrays)
+      fields = table.schema.fields
+      arrays = table.columns.map(&:data)
+
+      keys.zip(data_arrays).each do |key, array|
+        data = Arrow::ChunkedArray.new([array])
+        fields << Arrow::Field.new(key, data.value_data_type)
+        arrays << data
+      end
+
+      Arrow::Table.new(Arrow::Schema.new(fields), arrays)
+    end
 
-      df = RedAmber::DataFrame.new(@group.send(func, *summary_keys))
-      df = df[@group_keys, df.keys - @group_keys]
-      # if counts are the same (no nil included), aggregate count columns.
-      df = df[df.keys[0..1]].rename(df.keys[1], :count) if func == :count && df.to_h.values[1..].uniq.size == 1
-      df
+    # Call Vector aggregating function and return an array of arrays:
+    #   [keys, data_arrays]
+    #   (Experimental feature)
+    def call_aggregating_function(func, summary_keys, _options)
+      summary_keys.each.with_object([[], []]) do |key, (keys, arrays)|
+        vector = @dataframe[key]
+        arrays << filters.map { |filter| vector.filter(filter).send(func) }
+        keys << "#{func}(#{key})".to_sym
+      rescue Arrow::Error::NotImplemented
+        # next
+      end
     end
   end
 end

data/lib/red_amber/helper.rb

@@ -9,52 +9,42 @@ module RedAmber
       num > 1 ? 's' : ''
     end
 
-    def out_of_range?(indeces)
-      indeces.max >= size || indeces.min < -size
-    end
-
-    def integers?(enum)
-      enum.all?(Integer)
-    end
-
-    def sym_or_str?(enum)
-      enum.all? { |e| e.is_a?(Symbol) || e.is_a?(String) }
-    end
-
     def booleans?(enum)
       enum.all? { |e| e.is_a?(TrueClass) || e.is_a?(FalseClass) || e.is_a?(NilClass) }
     end
 
-    def create_dataframe_from_vector(key, vector)
-      DataFrame.new(key => vector.data)
-    end
-
-    def parse_to_vector(args)
+    def parse_to_vector(args, vsize: size)
       a = args.reduce([]) do |accum, elem|
-        accum.concat(normalize_element(elem))
+        accum.concat(normalize_element(elem, vsize: vsize))
       end
       Vector.new(a)
     end
 
-    def normalize_element(elem)
+    def normalize_element(elem, vsize: size)
       case elem
-      when Numeric, String, Symbol, TrueClass, FalseClass, NilClass
-        [elem]
+      when NilClass
+        [nil]
       when Range
-        both_end = [elem.begin, elem.end]
-        both_end[1] -= 1 if elem.exclude_end? && elem.end.is_a?(Integer)
-
-        if both_end.any?(Integer) || both_end.all?(&:nil?)
-          if both_end.any? { |e| e&.>=(size) || e&.<(-size) }
-            raise DataFrameArgumentError, "Index out of range: #{elem} for 0..#{size - 1}"
+        bg = elem.begin
+        en = elem.end
+        if [bg, en].any?(Integer)
+          bg += vsize if bg&.negative?
+          en += vsize if en&.negative?
+          en -= 1 if en.is_a?(Integer) && elem.exclude_end?
+          if bg&.negative? || (en && en >= vsize)
+            raise DataFrameArgumentError, "Index out of range: #{elem} for 0..#{vsize - 1}"
           end
 
-          (0...size).to_a[elem]
+          Array(0...vsize)[elem]
+        elsif bg.nil? && en.nil?
+          Array(0...vsize)
         else
-          elem.to_a
+          Array(elem)
         end
+      when Enumerator
+        elem.to_a
       else
-        Array(elem)
+        Array[elem]
       end
     end
   end

data/lib/red_amber/vector.rb

@@ -11,31 +11,39 @@ module RedAmber
     include Helper
 
     def initialize(*array)
-      @key = nil # default is 'headless'
-      if array.empty? || array[0].nil?
+      @key = nil # default is 'headless' Vector
+      if array.empty? || array.first.nil?
         Vector.new([])
       else
         array.flatten!
-        case array[0]
-        when Vector
-          @data = array[0].data
-          return
-        when Arrow::Array, Arrow::ChunkedArray
-          @data = array[0]
-          return
-        when Range
-          @data = Arrow::Array.new(Array(array[0]))
-          return
-        end
-        begin
-          @data = Arrow::Array.new(Array(array))
-        rescue Error
-          raise VectorArgumentError, "Invalid argument: #{array}"
-        end
+        @data =
+          case array
+          in [Vector => v]
+            v.data
+          in [Arrow::Array => a]
+            a
+          in [Arrow::ChunkedArray => ca]
+            ca
+          in [arrow_array_like] if arrow_array_like.respond_to?(:to_arrow_array)
+            arrow_array_like.to_arrow_array
+          in [Range => r]
+            Arrow::Array.new(Array(r))
+          else
+            begin
+              Arrow::Array.new(Array(array))
+            rescue Error
+              raise VectorArgumentError, "Invalid argument: #{array}"
+            end
+          end
       end
     end
 
     attr_reader :data
+
+    def to_arrow_array
+      @data
+    end
+
     attr_accessor :key
 
     def to_s
@@ -43,19 +51,24 @@ module RedAmber
     end
 
     def inspect(limit: 80)
-      sio = StringIO.new << '['
-      to_a.each_with_object(sio).with_index do |(e, s), i|
-        next_str = "#{s.size > 1 ? ', ' : ''}#{e.inspect}"
-        if (s.size + next_str.size) < limit
-          s << next_str
-        else
-          s << ', ... ' if i < size
-          break
+      if ENV.fetch('RED_AMBER_OUTPUT_MODE', 'Table').casecmp('MINIMUM').zero?
+        # Better performance than `.upcase == 'MINIMUM'``
+        "#{self.class}(:#{type}, size=#{size})"
+      else
+        sio = StringIO.new << '['
+        to_a.each_with_object(sio).with_index do |(e, s), i|
+          next_str = "#{s.size > 1 ? ', ' : ''}#{e.inspect}"
+          if (s.size + next_str.size) < limit
+            s << next_str
+          else
+            s << ', ... ' if i < size
+            break
+          end
         end
-      end
-      sio << ']'
+        sio << ']'
 
-      format "#<#{self.class}(:#{type}, size=#{size}):0x%016x>\n%s\n", object_id, sio.string
+        format "#<#{self.class}(:#{type}, size=#{size}):0x%016x>\n%s\n", object_id, sio.string
+      end
     end
 
     def values
@@ -71,7 +84,7 @@ module RedAmber
     alias_method :indeces, :indices
 
     def to_ary
-      to_a
+      values
     end
 
     def size
@@ -110,6 +123,10 @@ module RedAmber
       type_class == Arrow::StringDataType
     end
 
+    def dictionary?
+      type_class == Arrow::DictionaryDataType
+    end
+
     def temporal?
       type_class < Arrow::TemporalDataType
     end
@@ -126,10 +143,19 @@ module RedAmber
       end
     end
 
+    def map(&block)
+      return enum_for(:map) unless block
+
+      Vector.new(to_a.map(&block))
+    end
+    alias_method :collect, :map
+
+    # undocumented
     def chunked?
       @data.is_a? Arrow::ChunkedArray
     end
 
+    # undocumented
     def n_chunks
       chunked? ? @data.n_chunks : 0
     end

data/lib/red_amber/vector_functions.rb

@@ -187,12 +187,6 @@ module RedAmber
     alias_method :ne, :not_equal
 
     def coerce(other)
-      case other
-      when Vector, Array, Arrow::Array
-        raise VectorArgumentError, "Size unmatch: #{size} != #{other.length}" unless size == other.length
-
-        [Vector.new(Array(other)), self]
-      end
       [Vector.new(Array(other) * size), self]
     end
 
@@ -271,8 +265,6 @@ module RedAmber
         find(function).execute([data, other.data], options)
       when Arrow::Array, Arrow::ChunkedArray, Arrow::Scalar, Array, Numeric, String, TrueClass, FalseClass
         find(function).execute([data, other], options)
-      else
-        raise VectorArgumentError, "Operand is not supported: #{other.class}"
       end
     end