daru_lite 0.1 → 0.1.2

data/lib/daru_lite/data_frame/iterable.rb

@@ -0,0 +1,339 @@
+module DaruLite
+  class DataFrame
+    module Iterable
+      # Iterate over each index of the DataFrame.
+      def each_index(&block)
+        return to_enum(:each_index) unless block
+
+        @index.each(&block)
+
+        self
+      end
+
+      # Iterate over each vector
+      def each_vector(&block)
+        return to_enum(:each_vector) unless block
+
+        @data.each(&block)
+
+        self
+      end
+
+      alias each_column each_vector
+
+      # Iterate over each vector alongwith the name of the vector
+      def each_vector_with_index
+        return to_enum(:each_vector_with_index) unless block_given?
+
+        @vectors.each do |vector|
+          yield @data[@vectors[vector]], vector
+        end
+
+        self
+      end
+
+      alias each_column_with_index each_vector_with_index
+
+      # Iterate over each row
+      def each_row
+        return to_enum(:each_row) unless block_given?
+
+        @index.size.times do |pos|
+          yield row_at(pos)
+        end
+
+        self
+      end
+
+      def each_row_with_index
+        return to_enum(:each_row_with_index) unless block_given?
+
+        @index.each do |index|
+          yield access_row(index), index
+        end
+
+        self
+      end
+
+      # Iterate over each row or vector of the DataFrame. Specify axis
+      # by passing :vector or :row as the argument. Default to :vector.
+      #
+      # == Description
+      #
+      # `#each` works exactly like Array#each. The default mode for `each`
+      # is to iterate over the columns of the DataFrame. To iterate over
+      # rows you must pass the axis, i.e `:row` as an argument.
+      #
+      # == Arguments
+      #
+      # * +axis+ - The axis to iterate over. Can be :vector (or :column)
+      # or :row. Default to :vector.
+      def each(axis = :vector, &block)
+        dispatch_to_axis axis, :each, &block
+      end
+
+      # Iterate over a row or vector and return results in a DaruLite::Vector.
+      # Specify axis with :vector or :row. Default to :vector.
+      #
+      # == Description
+      #
+      # The #collect iterator works similar to #map, the only difference
+      # being that it returns a DaruLite::Vector comprising of the results of
+      # each block run. The resultant Vector has the same index as that
+      # of the axis over which collect has iterated. It also accepts the
+      # optional axis argument.
+      #
+      # == Arguments
+      #
+      # * +axis+ - The axis to iterate over. Can be :vector (or :column)
+      # or :row. Default to :vector.
+      def collect(axis = :vector, &block)
+        dispatch_to_axis_pl axis, :collect, &block
+      end
+
+      # Map over each vector or row of the data frame according to
+      # the argument specified. Will return an Array of the resulting
+      # elements. To map over each row/vector and get a DataFrame,
+      # see #recode.
+      #
+      # == Description
+      #
+      # The #map iterator works like Array#map. The value returned by
+      # each run of the block is added to an Array and the Array is
+      # returned. This method also accepts an axis argument, like #each.
+      # The default is :vector.
+      #
+      # == Arguments
+      #
+      # * +axis+ - The axis to map over. Can be :vector (or :column) or :row.
+      # Default to :vector.
+      def map(axis = :vector, &block)
+        dispatch_to_axis_pl axis, :map, &block
+      end
+
+      # Destructive map. Modifies the DataFrame. Each run of the block
+      # must return a DaruLite::Vector. You can specify the axis to map over
+      # as the argument. Default to :vector.
+      #
+      # == Arguments
+      #
+      # * +axis+ - The axis to map over. Can be :vector (or :column) or :row.
+      # Default to :vector.
+      def map!(axis = :vector, &block)
+        if %i[vector column].include?(axis)
+          map_vectors!(&block)
+        elsif axis == :row
+          map_rows!(&block)
+        end
+      end
+
+      # Maps over the DataFrame and returns a DataFrame. Each run of the
+      # block must return a DaruLite::Vector object. You can specify the axis
+      # to map over. Default to :vector.
+      #
+      # == Description
+      #
+      # Recode works similarly to #map, but an important difference between
+      # the two is that recode returns a modified DaruLite::DataFrame instead
+      # of an Array. For this reason, #recode expects that every run of the
+      # block to return a DaruLite::Vector.
+      #
+      # Just like map and each, recode also accepts an optional _axis_ argument.
+      #
+      # == Arguments
+      #
+      # * +axis+ - The axis to map over. Can be :vector (or :column) or :row.
+      # Default to :vector.
+      def recode(axis = :vector, &block)
+        dispatch_to_axis_pl axis, :recode, &block
+      end
+
+      # Replace specified values with given value
+      # @param [Array] old_values values to replace with new value
+      # @param [object] new_value new value to replace with
+      # @return [DaruLite::DataFrame] Data Frame itself with old values replace
+      #   with new value
+      # @example
+      #   df = DaruLite::DataFrame.new({
+      #     a: [1,    2,          3,   nil,        Float::NAN, nil, 1,   7],
+      #     b: [:a,  :b,          nil, Float::NAN, nil,        3,   5,   8],
+      #     c: ['a',  Float::NAN, 3,   4,          3,          5,   nil, 7]
+      #   }, index: 11..18)
+      #   df.replace_values nil, Float::NAN
+      #   # => #<DaruLite::DataFrame(8x3)>
+      #   #       a   b   c
+      #   #   11   1   a   a
+      #   #   12   2   b NaN
+      #   #   13   3 NaN   3
+      #   #   14 NaN NaN   4
+      #   #   15 NaN NaN   3
+      #   #   16 NaN   3   5
+      #   #   17   1   5 NaN
+      #   #   18   7   8   7
+      def replace_values(old_values, new_value)
+        @data.each { |vec| vec.replace_values old_values, new_value }
+        self
+      end
+
+      # Test each row with one or more tests.
+      # @param tests [Proc]  Each test is a Proc with the form
+      #                      *Proc.new {|row| row[:age] > 0}*
+      # The function returns an array with all errors.
+      #
+      # FIXME: description here is too sparse. As far as I can get,
+      # it should tell something about that each test is [descr, fields, block],
+      # and that first value may be column name to output. - zverok, 2016-05-18
+      def verify(*tests)
+        id = tests.first.is_a?(Symbol) ? tests.shift : @vectors.first
+
+        each_row_with_index.map do |row, i|
+          tests.reject { |*_, block| block.call(row) }
+               .map { |test| verify_error_message row, test, id, i }
+        end.flatten
+      end
+
+      def recode_vectors
+        block_given? or return to_enum(:recode_vectors)
+
+        dup.tap do |df|
+          df.each_vector_with_index do |v, i|
+            df[*i] = should_be_vector!(yield(v))
+          end
+        end
+      end
+
+      def recode_rows
+        block_given? or return to_enum(:recode_rows)
+
+        dup.tap do |df|
+          df.each_row_with_index do |r, i|
+            df.row[i] = should_be_vector!(yield(r))
+          end
+        end
+      end
+
+      # Map each vector and return an Array.
+      def map_vectors(&block)
+        return to_enum(:map_vectors) unless block
+
+        @data.map(&block)
+      end
+
+      # Destructive form of #map_vectors
+      def map_vectors!
+        return to_enum(:map_vectors!) unless block_given?
+
+        vectors.dup.each do |n|
+          self[n] = should_be_vector!(yield(self[n]))
+        end
+
+        self
+      end
+
+      # Map vectors alongwith the index.
+      def map_vectors_with_index(&block)
+        return to_enum(:map_vectors_with_index) unless block
+
+        each_vector_with_index.map(&block)
+      end
+
+      # Map each row
+      def map_rows(&block)
+        return to_enum(:map_rows) unless block
+
+        each_row.map(&block)
+      end
+
+      def map_rows_with_index(&block)
+        return to_enum(:map_rows_with_index) unless block
+
+        each_row_with_index.map(&block)
+      end
+
+      def map_rows!
+        return to_enum(:map_rows!) unless block_given?
+
+        index.dup.each do |i|
+          row[i] = should_be_vector!(yield(row[i]))
+        end
+
+        self
+      end
+
+      def apply_method(method, keys: nil, by_position: true)
+        df = keys ? get_sub_dataframe(keys, by_position: by_position) : self
+
+        case method
+        when Symbol then df.send(method)
+        when Proc   then method.call(df)
+        when Array
+          method.map(&:to_proc).map { |proc| proc.call(df) } # works with Array of both Symbol and/or Proc
+        else raise
+        end
+      end
+      alias apply_method_on_sub_df apply_method
+
+      # Retrieves a DaruLite::Vector, based on the result of calculation
+      # performed on each row.
+      def collect_rows(&block)
+        return to_enum(:collect_rows) unless block
+
+        DaruLite::Vector.new(each_row.map(&block), index: @index)
+      end
+
+      def collect_row_with_index(&block)
+        return to_enum(:collect_row_with_index) unless block
+
+        DaruLite::Vector.new(each_row_with_index.map(&block), index: @index)
+      end
+
+      # Retrives a DaruLite::Vector, based on the result of calculation
+      # performed on each vector.
+      def collect_vectors(&block)
+        return to_enum(:collect_vectors) unless block
+
+        DaruLite::Vector.new(each_vector.map(&block), index: @vectors)
+      end
+
+      def collect_vector_with_index(&block)
+        return to_enum(:collect_vector_with_index) unless block
+
+        DaruLite::Vector.new(each_vector_with_index.map(&block), index: @vectors)
+      end
+
+      # Generate a matrix, based on vector names of the DataFrame.
+      #
+      # @return {::Matrix}
+      # :nocov:
+      # FIXME: Even not trying to cover this: I can't get, how it is expected
+      # to work.... -- zverok
+      def collect_matrix
+        return to_enum(:collect_matrix) unless block_given?
+
+        vecs = vectors.to_a
+        rows = vecs.collect do |row|
+          vecs.collect do |col|
+            yield row, col
+          end
+        end
+
+        Matrix.rows(rows)
+      end
+      # :nocov:
+
+      private
+
+      def should_be_vector!(val)
+        return val if val.is_a?(DaruLite::Vector)
+
+        raise TypeError, "Every iteration must return DaruLite::Vector not #{val.class}"
+      end
+
+      def verify_error_message(row, test, id, i)
+        description, fields, = test
+        values = fields.empty? ? '' : " (#{fields.collect { |k| "#{k}=#{row[k]}" }.join(', ')})"
+        "#{i + 1} [#{row[id]}]: #{description}#{values}"
+      end
+    end
+  end
+end

data/lib/daru_lite/data_frame/joinable.rb

@@ -0,0 +1,152 @@
+module DaruLite
+  class DataFrame
+    module Joinable
+      # Concatenate another DataFrame along corresponding columns.
+      # If columns do not exist in both dataframes, they are filled with nils
+      def concat(other_df)
+        vectors = (@vectors.to_a + other_df.vectors.to_a).uniq
+
+        data = vectors.map do |v|
+          get_vector_anyways(v).dup.concat(other_df.get_vector_anyways(v))
+        end
+
+        DaruLite::DataFrame.new(data, order: vectors)
+      end
+
+      # Concatenates another DataFrame as #concat.
+      # Additionally it tries to preserve the index. If the indices contain
+      # common elements, #union will overwrite the according rows in the
+      # first dataframe.
+      def union(other_df)
+        index = (@index.to_a + other_df.index.to_a).uniq
+        df = row[*(@index.to_a - other_df.index.to_a)]
+
+        df = df.concat(other_df)
+        df.index = DaruLite::Index.new(index)
+        df
+      end
+
+      # Merge vectors from two DataFrames. In case of name collision,
+      # the vectors names are changed to x_1, x_2 ....
+      #
+      # @return {DaruLite::DataFrame}
+      def merge(other_df)
+        unless nrows == other_df.nrows
+          raise ArgumentError,
+                "Number of rows must be equal in this: #{nrows} and other: #{other_df.nrows}"
+        end
+
+        new_fields = (@vectors.to_a + other_df.vectors.to_a)
+        new_fields = ArrayHelper.recode_repeated(new_fields)
+        DataFrame.new({}, order: new_fields).tap do |df_new|
+          (0...nrows).each do |i|
+            df_new.add_row row[i].to_a + other_df.row[i].to_a
+          end
+          df_new.index = @index if @index == other_df.index
+          df_new.update
+        end
+      end
+
+      # Join 2 DataFrames with SQL style joins. Currently supports inner, left
+      # outer, right outer and full outer joins.
+      #
+      # @param [DaruLite::DataFrame] other_df Another DataFrame on which the join is
+      #   to be performed.
+      # @param [Hash] opts Options Hash
+      # @option :how [Symbol] Can be one of :inner, :left, :right or :outer.
+      # @option :on [Array] The columns on which the join is to be performed.
+      #   Column names specified here must be common to both DataFrames.
+      # @option :indicator [Symbol] The name of a vector to add to the resultant
+      #   dataframe that indicates whether the record was in the left (:left_only),
+      #   right (:right_only), or both (:both) joining dataframes.
+      # @return [DaruLite::DataFrame]
+      # @example Inner Join
+      #   left = DaruLite::DataFrame.new({
+      #     :id   => [1,2,3,4],
+      #     :name => ['Pirate', 'Monkey', 'Ninja', 'Spaghetti']
+      #   })
+      #   right = DaruLite::DataFrame.new({
+      #     :id => [1,2,3,4],
+      #     :name => ['Rutabaga', 'Pirate', 'Darth Vader', 'Ninja']
+      #   })
+      #   left.join(right, how: :inner, on: [:name])
+      #   #=>
+      #   ##<DaruLite::DataFrame:82416700 @name = 74c0811b-76c6-4c42-ac93-e6458e82afb0 @size = 2>
+      #   #                 id_1       name       id_2
+      #   #         0          1     Pirate          2
+      #   #         1          3      Ninja          4
+      def join(other_df, opts = {})
+        DaruLite::Core::Merge.join(self, other_df, opts)
+      end
+
+      # Creates a new dataset for one to many relations
+      # on a dataset, based on pattern of field names.
+      #
+      # for example, you have a survey for number of children
+      # with this structure:
+      #   id, name, child_name_1, child_age_1, child_name_2, child_age_2
+      # with
+      #   ds.one_to_many([:id], "child_%v_%n"
+      # the field of first parameters will be copied verbatim
+      # to new dataset, and fields which responds to second
+      # pattern will be added one case for each different %n.
+      #
+      # @example
+      #   cases=[
+      #     ['1','george','red',10,'blue',20,nil,nil],
+      #     ['2','fred','green',15,'orange',30,'white',20],
+      #     ['3','alfred',nil,nil,nil,nil,nil,nil]
+      #   ]
+      #   ds=DaruLite::DataFrame.rows(cases, order:
+      #     [:id, :name,
+      #      :car_color1, :car_value1,
+      #      :car_color2, :car_value2,
+      #      :car_color3, :car_value3])
+      #   ds.one_to_many([:id],'car_%v%n').to_matrix
+      #   #=> Matrix[
+      #   #   ["red", "1", 10],
+      #   #   ["blue", "1", 20],
+      #   #   ["green", "2", 15],
+      #   #   ["orange", "2", 30],
+      #   #   ["white", "2", 20]
+      #   #   ]
+      def one_to_many(parent_fields, pattern)
+        vars, numbers = one_to_many_components(pattern)
+
+        DataFrame.new([], order: [*parent_fields, '_col_id', *vars]).tap do |ds|
+          each_row do |row|
+            verbatim = parent_fields.map { |f| [f, row[f]] }.to_h
+            numbers.each do |n|
+              generated = one_to_many_row row, n, vars, pattern
+              next if generated.values.all?(&:nil?)
+
+              ds.add_row(verbatim.merge(generated).merge('_col_id' => n))
+            end
+          end
+          ds.update
+        end
+      end
+
+      private
+
+      def one_to_many_components(pattern)
+        re = Regexp.new pattern.gsub('%v', '(.+?)').gsub('%n', '(\\d+?)')
+
+        vars, numbers =
+          @vectors
+          .map { |v| v.scan(re) }
+          .reject(&:empty?).flatten(1).transpose
+
+        [vars.uniq, numbers.map(&:to_i).sort.uniq]
+      end
+
+      def one_to_many_row(row, number, vars, pattern)
+        vars
+          .to_h do |v|
+            name = pattern.sub('%v', v).sub('%n', number.to_s)
+            [v, row[name]]
+          end
+      end
+    end
+  end
+end

data/lib/daru_lite/data_frame/missable.rb

@@ -0,0 +1,75 @@
+module DaruLite
+  class DataFrame
+    module Missable
+      extend Gem::Deprecate
+
+      # Rolling fillna
+      # replace all Float::NAN and NIL values with the preceeding or following value
+      #
+      # @param direction [Symbol] (:forward, :backward) whether replacement value is preceeding or following
+      #
+      # @example
+      #   df = DaruLite::DataFrame.new({
+      #    a: [1,    2,          3,   nil,        Float::NAN, nil, 1,   7],
+      #    b: [:a,  :b,          nil, Float::NAN, nil,        3,   5,   nil],
+      #    c: ['a',  Float::NAN, 3,   4,          3,          5,   nil, 7]
+      #   })
+      #
+      #   => #<DaruLite::DataFrame(8x3)>
+      #        a   b   c
+      #    0   1   a   a
+      #    1   2   b NaN
+      #    2   3 nil   3
+      #    3 nil NaN   4
+      #    4 NaN nil   3
+      #    5 nil   3   5
+      #    6   1   5 nil
+      #    7   7 nil   7
+      #
+      #   2.3.3 :068 > df.rolling_fillna(:forward)
+      #   => #<DaruLite::DataFrame(8x3)>
+      #        a   b   c
+      #    0   1   a   a
+      #    1   2   b   a
+      #    2   3   b   3
+      #    3   3   b   4
+      #    4   3   b   3
+      #    5   3   3   5
+      #    6   1   5   5
+      #    7   7   5   7
+      #
+      def rolling_fillna!(direction = :forward)
+        @data.each { |vec| vec.rolling_fillna!(direction) }
+        self
+      end
+
+      def rolling_fillna(direction = :forward)
+        dup.rolling_fillna!(direction)
+      end
+
+      # Return a vector with the number of missing values in each row.
+      #
+      # == Arguments
+      #
+      # * +missing_values+ - An Array of the values that should be
+      # treated as 'missing'. The default missing value is *nil*.
+      def missing_values_rows(missing_values = [nil])
+        number_of_missing = each_row.map do |row|
+          row.indexes(*missing_values).size
+        end
+
+        DaruLite::Vector.new number_of_missing, index: @index, name: "#{@name}_missing_rows"
+      end
+
+      # TODO: remove next version
+      alias vector_missing_values missing_values_rows
+
+      def has_missing_data?
+        @data.any? { |vec| vec.include_values?(*DaruLite::MISSING_VALUES) }
+      end
+      alias flawed? has_missing_data?
+      deprecate :has_missing_data?, :include_values?, 2016, 10
+      deprecate :flawed?, :include_values?, 2016, 10
+    end
+  end
+end

data/lib/daru_lite/data_frame/pivotable.rb

@@ -0,0 +1,108 @@
+module DaruLite
+  class DataFrame
+    module Pivotable
+      # Pivots a data frame on specified vectors and applies an aggregate function
+      # to quickly generate a summary.
+      #
+      # == Options
+      #
+      # +:index+ - Keys to group by on the pivot table row index. Pass vector names
+      # contained in an Array.
+      #
+      # +:vectors+ - Keys to group by on the pivot table column index. Pass vector
+      # names contained in an Array.
+      #
+      # +:agg+ - Function to aggregate the grouped values. Default to *:mean*. Can
+      # use any of the statistics functions applicable on Vectors that can be found in
+      # the DaruLite::Statistics::Vector module.
+      #
+      # +:values+ - Columns to aggregate. Will consider all numeric columns not
+      # specified in *:index* or *:vectors*. Optional.
+      #
+      # == Usage
+      #
+      #   df = DaruLite::DataFrame.new({
+      #     a: ['foo'  ,  'foo',  'foo',  'foo',  'foo',  'bar',  'bar',  'bar',  'bar'],
+      #     b: ['one'  ,  'one',  'one',  'two',  'two',  'one',  'one',  'two',  'two'],
+      #     c: ['small','large','large','small','small','large','small','large','small'],
+      #     d: [1,2,2,3,3,4,5,6,7],
+      #     e: [2,4,4,6,6,8,10,12,14]
+      #   })
+      #   df.pivot_table(index: [:a], vectors: [:b], agg: :sum, values: :e)
+      #
+      #   #=>
+      #   # #<DaruLite::DataFrame:88342020 @name = 08cdaf4e-b154-4186-9084-e76dd191b2c9 @size = 2>
+      #   #            [:e, :one] [:e, :two]
+      #   #     [:bar]         18         26
+      #   #     [:foo]         10         12
+      def pivot_table(opts = {})
+        raise ArgumentError, 'Specify grouping index' if Array(opts[:index]).empty?
+
+        index               = opts[:index]
+        vectors             = opts[:vectors] || []
+        aggregate_function  = opts[:agg] || :mean
+        values              = prepare_pivot_values index, vectors, opts
+        raise IndexError, 'No numeric vectors to aggregate' if values.empty?
+
+        grouped = group_by(index)
+        return grouped.send(aggregate_function) if vectors.empty?
+
+        super_hash = make_pivot_hash grouped, vectors, values, aggregate_function
+
+        pivot_dataframe super_hash
+      end
+
+      private
+
+      def prepare_pivot_values(index, vectors, opts)
+        case opts[:values]
+        when nil # values not specified at all.
+          (@vectors.to_a - (index | vectors)) & numeric_vector_names
+        when Array # multiple values specified.
+          opts[:values]
+        else # single value specified.
+          [opts[:values]]
+        end
+      end
+
+      def make_pivot_hash(grouped, vectors, values, aggregate_function)
+        grouped.groups.transform_values { |_| {} }.tap do |super_hash|
+          values.each do |value|
+            grouped.groups.each do |group_name, row_numbers|
+              row_numbers.each do |num|
+                arry = [value, *vectors.map { |v| self[v][num] }]
+                sub_hash = super_hash[group_name]
+                sub_hash[arry] ||= []
+
+                sub_hash[arry] << self[value][num]
+              end
+            end
+          end
+
+          setup_pivot_aggregates super_hash, aggregate_function
+        end
+      end
+
+      def setup_pivot_aggregates(super_hash, aggregate_function)
+        super_hash.each_value do |sub_hash|
+          sub_hash.each do |group_name, aggregates|
+            sub_hash[group_name] = DaruLite::Vector.new(aggregates).send(aggregate_function)
+          end
+        end
+      end
+
+      def pivot_dataframe(super_hash)
+        df_index   = DaruLite::MultiIndex.from_tuples super_hash.keys
+        df_vectors = DaruLite::MultiIndex.from_tuples super_hash.values.flat_map(&:keys).uniq
+
+        DaruLite::DataFrame.new({}, index: df_index, order: df_vectors).tap do |pivoted_dataframe|
+          super_hash.each do |row_index, sub_h|
+            sub_h.each do |vector_index, val|
+              pivoted_dataframe[vector_index][row_index] = val
+            end
+          end
+        end
+      end
+    end
+  end
+end