daru_lite 0.1.1 → 0.1.2

data/lib/daru_lite/data_frame/missable.rb

@@ -0,0 +1,75 @@
+module DaruLite
+  class DataFrame
+    module Missable
+      extend Gem::Deprecate
+
+      # Rolling fillna
+      # replace all Float::NAN and NIL values with the preceeding or following value
+      #
+      # @param direction [Symbol] (:forward, :backward) whether replacement value is preceeding or following
+      #
+      # @example
+      #   df = DaruLite::DataFrame.new({
+      #    a: [1,    2,          3,   nil,        Float::NAN, nil, 1,   7],
+      #    b: [:a,  :b,          nil, Float::NAN, nil,        3,   5,   nil],
+      #    c: ['a',  Float::NAN, 3,   4,          3,          5,   nil, 7]
+      #   })
+      #
+      #   => #<DaruLite::DataFrame(8x3)>
+      #        a   b   c
+      #    0   1   a   a
+      #    1   2   b NaN
+      #    2   3 nil   3
+      #    3 nil NaN   4
+      #    4 NaN nil   3
+      #    5 nil   3   5
+      #    6   1   5 nil
+      #    7   7 nil   7
+      #
+      #   2.3.3 :068 > df.rolling_fillna(:forward)
+      #   => #<DaruLite::DataFrame(8x3)>
+      #        a   b   c
+      #    0   1   a   a
+      #    1   2   b   a
+      #    2   3   b   3
+      #    3   3   b   4
+      #    4   3   b   3
+      #    5   3   3   5
+      #    6   1   5   5
+      #    7   7   5   7
+      #
+      def rolling_fillna!(direction = :forward)
+        @data.each { |vec| vec.rolling_fillna!(direction) }
+        self
+      end
+
+      def rolling_fillna(direction = :forward)
+        dup.rolling_fillna!(direction)
+      end
+
+      # Return a vector with the number of missing values in each row.
+      #
+      # == Arguments
+      #
+      # * +missing_values+ - An Array of the values that should be
+      # treated as 'missing'. The default missing value is *nil*.
+      def missing_values_rows(missing_values = [nil])
+        number_of_missing = each_row.map do |row|
+          row.indexes(*missing_values).size
+        end
+
+        DaruLite::Vector.new number_of_missing, index: @index, name: "#{@name}_missing_rows"
+      end
+
+      # TODO: remove next version
+      alias vector_missing_values missing_values_rows
+
+      def has_missing_data?
+        @data.any? { |vec| vec.include_values?(*DaruLite::MISSING_VALUES) }
+      end
+      alias flawed? has_missing_data?
+      deprecate :has_missing_data?, :include_values?, 2016, 10
+      deprecate :flawed?, :include_values?, 2016, 10
+    end
+  end
+end

data/lib/daru_lite/data_frame/pivotable.rb

@@ -0,0 +1,108 @@
+module DaruLite
+  class DataFrame
+    module Pivotable
+      # Pivots a data frame on specified vectors and applies an aggregate function
+      # to quickly generate a summary.
+      #
+      # == Options
+      #
+      # +:index+ - Keys to group by on the pivot table row index. Pass vector names
+      # contained in an Array.
+      #
+      # +:vectors+ - Keys to group by on the pivot table column index. Pass vector
+      # names contained in an Array.
+      #
+      # +:agg+ - Function to aggregate the grouped values. Default to *:mean*. Can
+      # use any of the statistics functions applicable on Vectors that can be found in
+      # the DaruLite::Statistics::Vector module.
+      #
+      # +:values+ - Columns to aggregate. Will consider all numeric columns not
+      # specified in *:index* or *:vectors*. Optional.
+      #
+      # == Usage
+      #
+      #   df = DaruLite::DataFrame.new({
+      #     a: ['foo'  ,  'foo',  'foo',  'foo',  'foo',  'bar',  'bar',  'bar',  'bar'],
+      #     b: ['one'  ,  'one',  'one',  'two',  'two',  'one',  'one',  'two',  'two'],
+      #     c: ['small','large','large','small','small','large','small','large','small'],
+      #     d: [1,2,2,3,3,4,5,6,7],
+      #     e: [2,4,4,6,6,8,10,12,14]
+      #   })
+      #   df.pivot_table(index: [:a], vectors: [:b], agg: :sum, values: :e)
+      #
+      #   #=>
+      #   # #<DaruLite::DataFrame:88342020 @name = 08cdaf4e-b154-4186-9084-e76dd191b2c9 @size = 2>
+      #   #            [:e, :one] [:e, :two]
+      #   #     [:bar]         18         26
+      #   #     [:foo]         10         12
+      def pivot_table(opts = {})
+        raise ArgumentError, 'Specify grouping index' if Array(opts[:index]).empty?
+
+        index               = opts[:index]
+        vectors             = opts[:vectors] || []
+        aggregate_function  = opts[:agg] || :mean
+        values              = prepare_pivot_values index, vectors, opts
+        raise IndexError, 'No numeric vectors to aggregate' if values.empty?
+
+        grouped = group_by(index)
+        return grouped.send(aggregate_function) if vectors.empty?
+
+        super_hash = make_pivot_hash grouped, vectors, values, aggregate_function
+
+        pivot_dataframe super_hash
+      end
+
+      private
+
+      def prepare_pivot_values(index, vectors, opts)
+        case opts[:values]
+        when nil # values not specified at all.
+          (@vectors.to_a - (index | vectors)) & numeric_vector_names
+        when Array # multiple values specified.
+          opts[:values]
+        else # single value specified.
+          [opts[:values]]
+        end
+      end
+
+      def make_pivot_hash(grouped, vectors, values, aggregate_function)
+        grouped.groups.transform_values { |_| {} }.tap do |super_hash|
+          values.each do |value|
+            grouped.groups.each do |group_name, row_numbers|
+              row_numbers.each do |num|
+                arry = [value, *vectors.map { |v| self[v][num] }]
+                sub_hash = super_hash[group_name]
+                sub_hash[arry] ||= []
+
+                sub_hash[arry] << self[value][num]
+              end
+            end
+          end
+
+          setup_pivot_aggregates super_hash, aggregate_function
+        end
+      end
+
+      def setup_pivot_aggregates(super_hash, aggregate_function)
+        super_hash.each_value do |sub_hash|
+          sub_hash.each do |group_name, aggregates|
+            sub_hash[group_name] = DaruLite::Vector.new(aggregates).send(aggregate_function)
+          end
+        end
+      end
+
+      def pivot_dataframe(super_hash)
+        df_index   = DaruLite::MultiIndex.from_tuples super_hash.keys
+        df_vectors = DaruLite::MultiIndex.from_tuples super_hash.values.flat_map(&:keys).uniq
+
+        DaruLite::DataFrame.new({}, index: df_index, order: df_vectors).tap do |pivoted_dataframe|
+          super_hash.each do |row_index, sub_h|
+            sub_h.each do |vector_index, val|
+              pivoted_dataframe[vector_index][row_index] = val
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/daru_lite/data_frame/queryable.rb

@@ -0,0 +1,67 @@
+module DaruLite
+  class DataFrame
+    module Queryable
+      # Check if a vector is present
+      def has_vector?(vector)
+        @vectors.include? vector
+      end
+
+      # Check if any of given values occur in the data frame
+      # @param [Array] values to check for
+      # @return [true, false] true if any of the given values occur in the
+      #   dataframe, false otherwise
+      # @example
+      #   df = DaruLite::DataFrame.new({
+      #     a: [1,    2,          3,   nil,        Float::NAN, nil, 1,   7],
+      #     b: [:a,  :b,          nil, Float::NAN, nil,        3,   5,   8],
+      #     c: ['a',  Float::NAN, 3,   4,          3,          5,   nil, 7]
+      #   }, index: 11..18)
+      #   df.include_values? nil
+      #   # => true
+      def include_values?(*values)
+        @data.any? { |vec| vec.include_values?(*values) }
+      end
+
+      # Works like Array#any?.
+      #
+      # @param [Symbol] axis (:vector) The axis to iterate over. Can be :vector or
+      #   :row. A DaruLite::Vector object is yielded in the block.
+      # @example Using any?
+      #   df = DaruLite::DataFrame.new({a: [1,2,3,4,5], b: ['a', 'b', 'c', 'd', 'e']})
+      #   df.any?(:row) do |row|
+      #     row[:a] < 3 and row[:b] == 'b'
+      #   end #=> true
+      def any?(axis = :vector, &block)
+        if %i[vector column].include?(axis)
+          @data.any?(&block)
+        elsif axis == :row
+          each_row do |row|
+            return true if yield(row)
+          end
+          false
+        else
+          raise ArgumentError, "Unidentified axis #{axis}"
+        end
+      end
+
+      # Works like Array#all?
+      #
+      # @param [Symbol] axis (:vector) The axis to iterate over. Can be :vector or
+      #   :row. A DaruLite::Vector object is yielded in the block.
+      # @example Using all?
+      #   df = DaruLite::DataFrame.new({a: [1,2,3,4,5], b: ['a', 'b', 'c', 'd', 'e']})
+      #   df.all?(:row) do |row|
+      #     row[:a] < 10
+      #   end #=> true
+      def all?(axis = :vector, &block)
+        if %i[vector column].include?(axis)
+          @data.all?(&block)
+        elsif axis == :row
+          each_row.all?(&block)
+        else
+          raise ArgumentError, "Unidentified axis #{axis}"
+        end
+      end
+    end
+  end
+end

data/lib/daru_lite/data_frame/setable.rb

@@ -0,0 +1,109 @@
+module DaruLite
+  class DataFrame
+    module Setable
+      # Set rows by positions
+      # @param [Array<Integer>] positions positions of rows to set
+      # @param [Array, DaruLite::Vector] vector vector to be assigned
+      # @example
+      #   df = DaruLite::DataFrame.new({
+      #     a: [1, 2, 3],
+      #     b: ['a', 'b', 'c']
+      #   })
+      #   df.set_row_at [0, 1], ['x', 'x']
+      #   df
+      #   #=> #<DaruLite::DataFrame(3x2)>
+      #   #       a   b
+      #   #   0   x   x
+      #   #   1   x   x
+      #   #   2   3   c
+      def set_row_at(positions, vector)
+        validate_positions(*positions, nrows)
+        vector =
+          if vector.is_a? DaruLite::Vector
+            vector.reindex @vectors
+          else
+            DaruLite::Vector.new vector
+          end
+
+        raise SizeError, 'Vector length should match row length' if
+          vector.size != @vectors.size
+
+        @data.each_with_index do |vec, pos|
+          vec.set_at(positions, vector.at(pos))
+        end
+        @index = @data[0].index
+        set_size
+      end
+
+      # Set vectors by positions
+      # @param [Array<Integer>] positions positions of vectors to set
+      # @param [Array, DaruLite::Vector] vector vector to be assigned
+      # @example
+      #   df = DaruLite::DataFrame.new({
+      #     a: [1, 2, 3],
+      #     b: ['a', 'b', 'c']
+      #   })
+      #   df.set_at [0], ['x', 'y', 'z']
+      #   df
+      #   #=> #<DaruLite::DataFrame(3x2)>
+      #   #       a   b
+      #   #   0   x   a
+      #   #   1   y   b
+      #   #   2   z   c
+      def set_at(positions, vector)
+        if positions.last == :row
+          positions.pop
+          return set_row_at(positions, vector)
+        end
+
+        validate_positions(*positions, ncols)
+        vector =
+          if vector.is_a? DaruLite::Vector
+            vector.reindex @index
+          else
+            DaruLite::Vector.new vector
+          end
+
+        raise SizeError, 'Vector length should match index length' if
+          vector.size != @index.size
+
+        positions.each { |pos| @data[pos] = vector }
+      end
+
+      # Insert a new row/vector of the specified name or modify a previous row.
+      # Instead of using this method directly, use df.row[:a] = [1,2,3] to set/create
+      # a row ':a' to [1,2,3], or df.vector[:vec] = [1,2,3] for vectors.
+      #
+      # In case a DaruLite::Vector is specified after the equality the sign, the indexes
+      # of the vector will be matched against the row/vector indexes of the DataFrame
+      # before an insertion is performed. Unmatched indexes will be set to nil.
+      def []=(*args)
+        vector = args.pop
+        axis = extract_axis(args)
+        names = args
+
+        dispatch_to_axis axis, :insert_or_modify, names, vector
+      end
+
+      def add_row(row, index = nil)
+        self.row[*(index || @size)] = row
+      end
+
+      def add_vector(n, vector)
+        self[n] = vector
+      end
+
+      def insert_vector(n, name, source)
+        raise ArgumentError unless source.is_a? Array
+
+        vector = DaruLite::Vector.new(source, index: @index, name: @name)
+        @data << vector
+        @vectors = @vectors.add name
+        ordr = @vectors.dup.to_a
+        elmnt = ordr.pop
+        ordr.insert n, elmnt
+        self.order = ordr
+      end
+    end
+  end
+end

data/lib/daru_lite/data_frame/sortable.rb

@@ -0,0 +1,241 @@
+module DaruLite
+  class DataFrame
+    module Sortable
+      # Reorder the vectors in a dataframe
+      # @param [Array] order_array new order of the vectors
+      # @example
+      #   df = DaruLite::DataFrame({
+      #     a: [1, 2, 3],
+      #     b: [4, 5, 6]
+      #   }, order: [:a, :b])
+      #   df.order = [:b, :a]
+      #   df
+      #   # => #<DaruLite::DataFrame(3x2)>
+      #   #       b   a
+      #   #   0   4   1
+      #   #   1   5   2
+      #   #   2   6   3
+      def order=(order_array)
+        raise ArgumentError, 'Invalid order' unless order_array.tally == vectors.to_a.tally
+
+        initialize(to_h, order: order_array)
+      end
+
+      # Return the dataframe with rotate vectors positions, the vector at position count is now
+      # the first vector of the dataframe.
+      # If only one vector in the dataframe, the dataframe is return without any change.
+      # @param count => Integer, the vector at position count will be the first vector of the dataframe.
+      # @example
+      #   df = DaruLite::DataFrame({
+      #     a: [1, 2, 3],
+      #     b: [4, 5, 6],
+      #     total: [5, 7, 9],
+      #   })
+      #   df.rotate_vectors(-1)
+      #   df
+      #   # => #<DaruLite::DataFrame(3x3)>
+      #   #       total b   a
+      #   #   0   5     4   1
+      #   #   1   7     5   2
+      #   #   2   9     6   3
+      def rotate_vectors(count = -1)
+        return self unless vectors.many?
+
+        self.order = vectors.to_a.rotate(count)
+        self
+      end
+
+      # Sorts a dataframe (ascending/descending) in the given pripority sequence of
+      # vectors, with or without a block.
+      #
+      # @param vector_order [Array] The order of vector names in which the DataFrame
+      #   should be sorted.
+      # @param opts [Hash] opts The options to sort with.
+      # @option opts [TrueClass,FalseClass,Array] :ascending (true) Sort in ascending
+      #   or descending order. Specify Array corresponding to *order* for multiple
+      #   sort orders.
+      # @option opts [Hash] :by (lambda{|a| a }) Specify attributes of objects to
+      #   to be used for sorting, for each vector name in *order* as a hash of
+      #   vector name and lambda expressions. In case a lambda for a vector is not
+      #   specified, the default will be used.
+      # @option opts [TrueClass,FalseClass,Array] :handle_nils (false) Handle nils
+      #   automatically or not when a block is provided.
+      #   If set to True, nils will appear at top after sorting.
+      #
+      # @example Sort a dataframe with a vector sequence.
+      #
+      #
+      #   df = DaruLite::DataFrame.new({a: [1,2,1,2,3], b: [5,4,3,2,1]})
+      #
+      #   df.sort [:a, :b]
+      #   # =>
+      #   # <DaruLite::DataFrame:30604000 @name = d6a9294e-2c09-418f-b646-aa9244653444 @size = 5>
+      #   #                   a          b
+      #   #        2          1          3
+      #   #        0          1          5
+      #   #        3          2          2
+      #   #        1          2          4
+      #   #        4          3          1
+      #
+      # @example Sort a dataframe without a block. Here nils will be handled automatically.
+      #
+      #   df = DaruLite::DataFrame.new({a: [-3,nil,-1,nil,5], b: [4,3,2,1,4]})
+      #
+      #   df.sort([:a])
+      #   # =>
+      #   # <DaruLite::DataFrame:14810920 @name = c07fb5c7-2201-458d-b679-6a1f7ebfe49f @size = 5>
+      #   #                    a          b
+      #   #         1        nil          3
+      #   #         3        nil          1
+      #   #         0         -3          4
+      #   #         2         -1          2
+      #   #         4          5          4
+      #
+      # @example Sort a dataframe with a block with nils handled automatically.
+      #
+      #   df = DaruLite::DataFrame.new({a: [nil,-1,1,nil,-1,1], b: ['aaa','aa',nil,'baaa','x',nil] })
+      #
+      #   df.sort [:b], by: {b: lambda { |a| a.length } }
+      #   # NoMethodError: undefined method `length' for nil:NilClass
+      #   # from (pry):8:in `block in __pry__'
+      #
+      #   df.sort [:b], by: {b: lambda { |a| a.length } }, handle_nils: true
+      #
+      #   # =>
+      #   # <DaruLite::DataFrame:28469540 @name = 5f986508-556f-468b-be0c-88cc3534445c @size = 6>
+      #   #                    a          b
+      #   #         2          1        nil
+      #   #         5          1        nil
+      #   #         4         -1          x
+      #   #         1         -1         aa
+      #   #         0        nil        aaa
+      #   #         3        nil       baaa
+      #
+      # @example Sort a dataframe with a block with nils handled manually.
+      #
+      #   df = DaruLite::DataFrame.new({a: [nil,-1,1,nil,-1,1], b: ['aaa','aa',nil,'baaa','x',nil] })
+      #
+      #   # To print nils at the bottom one can use lambda { |a| (a.nil?)[1]:[0,a.length] }
+      #   df.sort [:b], by: {b: lambda { |a| (a.nil?)?[1]:[0,a.length] } }, handle_nils: true
+      #
+      #   # =>
+      #   #<DaruLite::DataFrame:22214180 @name = cd7703c7-1dca-4560-840b-5ea51a852ef9 @size = 6>
+      #   #                 a          b
+      #   #      4         -1          x
+      #   #      1         -1         aa
+      #   #      0        nil        aaa
+      #   #      3        nil       baaa
+      #   #      2          1        nil
+      #   #      5          1        nil
+
+      def sort!(vector_order, opts = {})
+        raise ArgumentError, 'Required atleast one vector name' if vector_order.empty?
+
+        # To enable sorting with categorical data,
+        # map categories to integers preserving their order
+        old = convert_categorical_vectors vector_order
+        block = sort_prepare_block vector_order, opts
+
+        order = @index.size.times.sort(&block)
+        new_index = @index.reorder order
+
+        # To reverse map mapping of categorical data to integers
+        restore_categorical_vectors old
+
+        @data.each do |vector|
+          vector.reorder! order
+        end
+
+        self.index = new_index
+
+        self
+      end
+
+      # Non-destructive version of #sort!
+      def sort(vector_order, opts = {})
+        dup.sort! vector_order, opts
+      end
+
+      private
+
+      def convert_categorical_vectors(names)
+        names.filter_map do |n|
+          next unless self[n].category?
+
+          old = [n, self[n]]
+          self[n] = DaruLite::Vector.new(self[n].to_ints)
+          old
+        end
+      end
+
+      def restore_categorical_vectors(old)
+        old.each { |name, vector| self[name] = vector }
+      end
+
+      def sort_build_row(vector_locs, by_blocks, ascending, handle_nils, r1, r2) # rubocop:disable  Metrics/ParameterLists
+        # Create an array to be used for comparison of two rows in sorting
+        vector_locs
+          .zip(by_blocks, ascending, handle_nils)
+          .map do |vector_loc, by, asc, handle_nil|
+          value = @data[vector_loc].data[asc ? r1 : r2]
+
+          if by
+            value = begin
+              by.call(value)
+            rescue StandardError
+              nil
+            end
+          end
+
+          sort_handle_nils value, asc, handle_nil || !by
+        end
+      end
+
+      def sort_handle_nils(value, asc, handle_nil)
+        if !handle_nil
+          value
+        elsif asc
+          [value.nil? ? 0 : 1, value]
+        else
+          [value.nil? ? 1 : 0, value]
+        end
+      end
+
+      def sort_coerce_boolean(opts, symbol, default, size)
+        val = opts[symbol]
+        case val
+        when true, false
+          Array.new(size, val)
+        when nil
+          Array.new(size, default)
+        when Array
+          raise ArgumentError, "Specify same number of vector names and #{symbol}" if
+            size != val.size
+
+          val
+        else
+          raise ArgumentError, "Can't coerce #{symbol} from #{val.class} to boolean option"
+        end
+      end
+
+      def sort_prepare_block(vector_order, opts)
+        ascending   = sort_coerce_boolean opts, :ascending, true, vector_order.size
+        handle_nils = sort_coerce_boolean opts, :handle_nils, false, vector_order.size
+
+        by_blocks = vector_order.map { |v| (opts[:by] || {})[v] }
+        vector_locs = vector_order.map { |v| @vectors[v] }
+
+        lambda do |index1, index2|
+          # Build left and right array to compare two rows
+          left  = sort_build_row vector_locs, by_blocks, ascending, handle_nils, index1, index2
+          right = sort_build_row vector_locs, by_blocks, ascending, handle_nils, index2, index1
+
+          # Resolve conflict by Index if all attributes are same
+          left  << index1
+          right << index2
+          left <=> right
+        end
+      end
+    end
+  end
+end