daru_lite 0.1 → 0.1.2

data/lib/daru_lite/data_frame/calculatable.rb

@@ -0,0 +1,140 @@
+module DaruLite
+  class DataFrame
+    module Calculatable
+      # Sum all numeric/specified vectors in the DataFrame.
+      #
+      # Returns a new vector that's a containing a sum of all numeric
+      # or specified vectors of the DataFrame. By default, if the vector
+      # contains a nil, the sum is nil.
+      # With :skipnil argument set to true, nil values are assumed to be
+      # 0 (zero) and the sum vector is returned.
+      #
+      # @param args [Array] List of vectors to sum. Default is nil in which case
+      #   all numeric vectors are summed.
+      #
+      # @option opts [Boolean] :skipnil Consider nils as 0. Default is false.
+      #
+      # @return Vector with sum of all vectors specified in the argument.
+      #   If vecs parameter is empty, sum all numeric vector.
+      #
+      # @example
+      #    df = DaruLite::DataFrame.new({
+      #       a: [1, 2, nil],
+      #       b: [2, 1, 3],
+      #       c: [1, 1, 1]
+      #     })
+      #    => #<DaruLite::DataFrame(3x3)>
+      #           a   b   c
+      #       0   1   2   1
+      #       1   2   1   1
+      #       2 nil   3   1
+      #    df.vector_sum [:a, :c]
+      #    => #<DaruLite::Vector(3)>
+      #       0   2
+      #       1   3
+      #       2 nil
+      #    df.vector_sum
+      #    => #<DaruLite::Vector(3)>
+      #       0   4
+      #       1   4
+      #       2 nil
+      #    df.vector_sum skipnil: true
+      #    => #<DaruLite::Vector(3)>
+      #           c
+      #       0   4
+      #       1   4
+      #       2   4
+      #
+      def vector_sum(*args)
+        defaults = { vecs: nil, skipnil: false }
+        options = args.last.is_a?(::Hash) ? args.pop : {}
+        options = defaults.merge(options)
+        vecs = args[0] || options[:vecs]
+        skipnil = args[1] || options[:skipnil]
+
+        vecs ||= numeric_vectors
+        sum = DaruLite::Vector.new [0] * @size, index: @index, name: @name, dtype: @dtype
+        vecs.inject(sum) { |memo, n| self[n].add(memo, skipnil: skipnil) }
+      end
+
+      # Calculate mean of the rows of the dataframe.
+      #
+      # == Arguments
+      #
+      # * +max_missing+ - The maximum number of elements in the row that can be
+      # zero for the mean calculation to happen. Default to 0.
+      def vector_mean(max_missing = 0)
+        # FIXME: in vector_sum we preserve created vector dtype, but
+        # here we are not. Is this by design or ...? - zverok, 2016-05-18
+        mean_vec = DaruLite::Vector.new [0] * @size, index: @index, name: "mean_#{@name}"
+
+        each_row_with_index.with_object(mean_vec) do |(row, i), memo|
+          memo[i] = row.indexes(*DaruLite::MISSING_VALUES).size > max_missing ? nil : row.mean
+        end
+      end
+
+      # Returns a vector, based on a string with a calculation based
+      # on vector.
+      #
+      # The calculation will be eval'ed, so you can put any variable
+      # or expression valid on ruby.
+      #
+      # For example:
+      #   a = DaruLite::Vector.new [1,2]
+      #   b = DaruLite::Vector.new [3,4]
+      #   ds = DaruLite::DataFrame.new({:a => a,:b => b})
+      #   ds.compute("a+b")
+      #   => Vector [4,6]
+      def compute(text, &block)
+        return instance_eval(&block) if block
+
+        instance_eval(text)
+      end
+
+      # DSL for yielding each row and returning a DaruLite::Vector based on the
+      # value each run of the block returns.
+      #
+      # == Usage
+      #
+      #   a1 = DaruLite::Vector.new([1, 2, 3, 4, 5, 6, 7])
+      #   a2 = DaruLite::Vector.new([10, 20, 30, 40, 50, 60, 70])
+      #   a3 = DaruLite::Vector.new([100, 200, 300, 400, 500, 600, 700])
+      #   ds = DaruLite::DataFrame.new({ :a => a1, :b => a2, :c => a3 })
+      #   total = ds.vector_by_calculation { a + b + c }
+      #   # <DaruLite::Vector:82314050 @name = nil @size = 7 >
+      #   #   nil
+      #   # 0 111
+      #   # 1 222
+      #   # 2 333
+      #   # 3 444
+      #   # 4 555
+      #   # 5 666
+      #   # 6 777
+      def vector_by_calculation(&block)
+        a = each_row.map { |r| r.instance_eval(&block) }
+
+        DaruLite::Vector.new a, index: @index
+      end
+
+      def vector_count_characters(vecs = nil)
+        vecs ||= @vectors.to_a
+
+        collect_rows do |row|
+          vecs.sum { |v| row[v].to_s.size }
+        end
+      end
+
+      # Generate a summary of this DataFrame based on individual vectors in the DataFrame
+      # @return [String] String containing the summary of the DataFrame
+      def summary
+        summary = "= #{name}"
+        summary << "\n  Number of rows: #{nrows}"
+        @vectors.each do |v|
+          summary << "\n  Element:[#{v}]\n"
+          summary << self[v].summary(1)
+        end
+        summary
+      end
+    end
+  end
+end

data/lib/daru_lite/data_frame/convertible.rb

@@ -0,0 +1,107 @@
+module DaruLite
+  class DataFrame
+    module Convertible
+      # Create a sql, basen on a given Dataset
+      #
+      # == Arguments
+      #
+      # * table - String specifying name of the table that will created in SQL.
+      # * charset - Character set. Default is "UTF8".
+      #
+      # @example
+      #
+      #  ds = DaruLite::DataFrame.new({
+      #   :id   => DaruLite::Vector.new([1,2,3,4,5]),
+      #   :name => DaruLite::Vector.new(%w{Alex Peter Susan Mary John})
+      #  })
+      #  ds.create_sql('names')
+      #   #=>"CREATE TABLE names (id INTEGER,\n name VARCHAR (255)) CHARACTER SET=UTF8;"
+      #
+      def create_sql(table, charset = 'UTF8')
+        sql    = "CREATE TABLE #{table} ("
+        fields = vectors.to_a.collect do |f|
+          v = self[f]
+          "#{f} #{v.db_type}"
+        end
+
+        sql + fields.join(",\n ") + ") CHARACTER SET=#{charset};"
+      end
+
+      # Returns the dataframe.  This can be convenient when the user does not
+      # know whether the object is a vector or a dataframe.
+      # @return [self] the dataframe
+      def to_df
+        self
+      end
+
+      # Convert all vectors of type *:numeric* into a Matrix.
+      def to_matrix
+        Matrix.columns each_vector.select(&:numeric?).map(&:to_a)
+      end
+
+      # Converts the DataFrame into an array of hashes where key is vector name
+      # and value is the corresponding element. The 0th index of the array contains
+      # the array of hashes while the 1th index contains the indexes of each row
+      # of the dataframe. Each element in the index array corresponds to its row
+      # in the array of hashes, which has the same index.
+      def to_a
+        [each_row.map(&:to_h), @index.to_a]
+      end
+
+      # Convert to json. If no_index is false then the index will NOT be included
+      # in the JSON thus created.
+      def to_json(no_index = true)
+        if no_index
+          to_a[0].to_json
+        else
+          to_a.to_json
+        end
+      end
+
+      # Converts DataFrame to a hash (explicit) with keys as vector names and values as
+      # the corresponding vectors.
+      def to_h
+        @vectors
+          .each_with_index
+          .map { |vec_name, idx| [vec_name, @data[idx]] }.to_h
+      end
+
+      # Convert to html for IRuby.
+      def to_html(threshold = DaruLite.max_rows)
+        table_thead = to_html_thead
+        table_tbody = to_html_tbody(threshold)
+        path = if index.is_a?(MultiIndex)
+                 File.expand_path('../iruby/templates/dataframe_mi.html.erb', __dir__)
+               else
+                 File.expand_path('../iruby/templates/dataframe.html.erb', __dir__)
+               end
+        ERB.new(File.read(path).strip).result(binding)
+      end
+
+      def to_html_thead
+        table_thead_path =
+          if index.is_a?(MultiIndex)
+            File.expand_path('../iruby/templates/dataframe_mi_thead.html.erb', __dir__)
+          else
+            File.expand_path('../iruby/templates/dataframe_thead.html.erb', __dir__)
+          end
+        ERB.new(File.read(table_thead_path).strip).result(binding)
+      end
+
+      def to_html_tbody(threshold = DaruLite.max_rows)
+        threshold ||= @size
+        table_tbody_path =
+          if index.is_a?(MultiIndex)
+            File.expand_path('../iruby/templates/dataframe_mi_tbody.html.erb', __dir__)
+          else
+            File.expand_path('../iruby/templates/dataframe_tbody.html.erb', __dir__)
+          end
+        ERB.new(File.read(table_tbody_path).strip).result(binding)
+      end
+
+      def to_s
+        "#<#{self.class}#{": #{@name}" if @name}(#{nrows}x#{ncols})>"
+      end
+    end
+  end
+end

data/lib/daru_lite/data_frame/duplicatable.rb

@@ -0,0 +1,64 @@
+module DaruLite
+  class DataFrame
+    module Duplicatable
+      extend Gem::Deprecate
+
+      # Duplicate the DataFrame entirely.
+      #
+      # == Arguments
+      #
+      # * +vectors_to_dup+ - An Array specifying the names of Vectors to
+      # be duplicated. Will duplicate the entire DataFrame if not specified.
+      def dup(vectors_to_dup = nil)
+        vectors_to_dup ||= @vectors.to_a
+
+        src = vectors_to_dup.map { |vec| @data[@vectors.pos(vec)].dup }
+        new_order = DaruLite::Index.new(vectors_to_dup)
+
+        DaruLite::DataFrame.new src, order: new_order, index: @index.dup, name: @name, clone: true
+      end
+
+      # Only clone the structure of the DataFrame.
+      def clone_structure
+        DaruLite::DataFrame.new([], order: @vectors.dup, index: @index.dup, name: @name)
+      end
+
+      # Returns a 'view' of the DataFrame, i.e the object ID's of vectors are
+      # preserved.
+      #
+      # == Arguments
+      #
+      # +vectors_to_clone+ - Names of vectors to clone. Optional. Will return
+      # a view of the whole data frame otherwise.
+      def clone(*vectors_to_clone)
+        vectors_to_clone.flatten! if ArrayHelper.array_of?(vectors_to_clone, Array)
+        vectors_to_clone = @vectors.to_a if vectors_to_clone.empty?
+
+        h = vectors_to_clone.map { |vec| [vec, self[vec]] }.to_h
+        DaruLite::DataFrame.new(h, clone: false, order: vectors_to_clone, name: @name)
+      end
+
+      # Returns a 'shallow' copy of DataFrame if missing data is not present,
+      # or a full copy of only valid data if missing data is present.
+      def clone_only_valid
+        if include_values?(*DaruLite::MISSING_VALUES)
+          reject_values(*DaruLite::MISSING_VALUES)
+        else
+          clone
+        end
+      end
+
+      # Creates a new duplicate dataframe containing only rows
+      # without a single missing value.
+      def dup_only_valid(vecs = nil)
+        rows_with_nil = @data.map { |vec| vec.indexes(*DaruLite::MISSING_VALUES) }
+                             .inject(&:concat)
+                             .uniq
+
+        row_indexes = @index.to_a
+        (vecs.nil? ? self : dup(vecs)).row[*(row_indexes - rows_with_nil)]
+      end
+      deprecate :dup_only_valid, :reject_values, 2016, 10
+    end
+  end
+end

data/lib/daru_lite/data_frame/fetchable.rb

@@ -0,0 +1,301 @@
+module DaruLite
+  class DataFrame
+    module Fetchable
+      # Access row or vector. Specify name of row/vector followed by axis(:row, :vector).
+      # Defaults to *:vector*. Use of this method is not recommended for accessing
+      # rows. Use df.row[:a] for accessing row with index ':a'.
+      def [](*names)
+        axis = extract_axis(names, :vector)
+        dispatch_to_axis axis, :access, *names
+      end
+
+      # Retrive rows by positions
+      # @param [Array<Integer>] positions of rows to retrive
+      # @return [DaruLite::Vector, DaruLite::DataFrame] vector for single position and dataframe for multiple positions
+      # @example
+      #   df = DaruLite::DataFrame.new({
+      #     a: [1, 2, 3],
+      #     b: ['a', 'b', 'c']
+      #   })
+      #   df.row_at 1, 2
+      #   # => #<DaruLite::DataFrame(2x2)>
+      #   #       a   b
+      #   #   1   2   b
+      #   #   2   3   c
+      def row_at(*positions)
+        original_positions = positions
+        positions = coerce_positions(*positions, nrows)
+        validate_positions(*positions, nrows)
+
+        if positions.is_a? Integer
+          row = get_rows_for([positions])
+          DaruLite::Vector.new(row, index: @vectors, name: @index.at(positions))
+        else
+          new_rows = get_rows_for(original_positions)
+          DaruLite::DataFrame.new(
+            new_rows,
+            index: @index.at(*original_positions),
+            order: @vectors,
+            name: @name
+          )
+        end
+      end
+
+      # Retrive vectors by positions
+      # @param [Array<Integer>] positions of vectors to retrive
+      # @return [DaruLite::Vector, DaruLite::DataFrame] vector for single position and dataframe for multiple positions
+      # @example
+      #   df = DaruLite::DataFrame.new({
+      #     a: [1, 2, 3],
+      #     b: ['a', 'b', 'c']
+      #   })
+      #   df.at 0
+      #   # => #<DaruLite::Vector(3)>
+      #   #       a
+      #   #   0   1
+      #   #   1   2
+      #   #   2   3
+      def at(*positions)
+        if AXES.include? positions.last
+          axis = positions.pop
+          return row_at(*positions) if axis == :row
+        end
+
+        original_positions = positions
+        positions = coerce_positions(*positions, ncols)
+        validate_positions(*positions, ncols)
+
+        if positions.is_a? Integer
+          @data[positions].dup
+        else
+          DaruLite::DataFrame.new positions.map { |pos| @data[pos].dup },
+                                  index: @index,
+                                  order: @vectors.at(*original_positions),
+                                  name: @name
+        end
+      end
+
+      # The first ten elements of the DataFrame
+      #
+      # @param [Fixnum] quantity (10) The number of elements to display from the top.
+      def head(quantity = 10)
+        row.at 0..(quantity - 1)
+      end
+      alias first head
+
+      # The last ten elements of the DataFrame
+      #
+      # @param [Fixnum] quantity (10) The number of elements to display from the bottom.
+      def tail(quantity = 10)
+        start = [-quantity, -size].max
+        row.at start..-1
+      end
+      alias last tail
+
+      # Extract a dataframe given row indexes or positions
+      # @param keys [Array] can be positions (if by_position is true) or indexes (if by_position if false)
+      # @return [DaruLite::Dataframe]
+      def get_sub_dataframe(keys, by_position: true)
+        return DaruLite::DataFrame.new({}) if keys == []
+
+        keys = @index.pos(*keys) unless by_position
+
+        sub_df = row_at(*keys)
+        sub_df = sub_df.to_df.transpose if sub_df.is_a?(DaruLite::Vector)
+
+        sub_df
+      end
+
+      def get_vector_anyways(v)
+        @vectors.include?(v) ? self[v].to_a : Array.new(size)
+      end
+
+      # @param indexes [Array] index(s) at which row tuples are retrieved
+      # @return [Array] returns array of row tuples at given index(s)
+      # @example Using DaruLite::Index
+      #   df = DaruLite::DataFrame.new({
+      #     a: [1, 2, 3],
+      #     b: ['a', 'a', 'b']
+      #   })
+      #
+      #   df.access_row_tuples_by_indexs(1,2)
+      #   # => [[2, "a"], [3, "b"]]
+      #
+      #   df.index = DaruLite::Index.new([:one,:two,:three])
+      #   df.access_row_tuples_by_indexs(:one,:three)
+      #   # => [[1, "a"], [3, "b"]]
+      #
+      # @example Using DaruLite::MultiIndex
+      #   mi_idx = DaruLite::MultiIndex.from_tuples [
+      #     [:a,:one,:bar],
+      #     [:a,:one,:baz],
+      #     [:b,:two,:bar],
+      #     [:a,:two,:baz],
+      #   ]
+      #   df_mi = DaruLite::DataFrame.new({
+      #     a: 1..4,
+      #     b: 'a'..'d'
+      #   }, index: mi_idx )
+      #
+      #   df_mi.access_row_tuples_by_indexs(:b, :two, :bar)
+      #   # => [[3, "c"]]
+      #   df_mi.access_row_tuples_by_indexs(:a)
+      #   # => [[1, "a"], [2, "b"], [4, "d"]]
+      def access_row_tuples_by_indexs(*indexes)
+        return get_sub_dataframe(indexes, by_position: false).map_rows(&:to_a) if
+        @index.is_a?(DaruLite::MultiIndex)
+
+        positions = @index.pos(*indexes)
+        if positions.is_a? Numeric
+          row = get_rows_for([positions])
+          row.first.is_a?(Array) ? row : [row]
+        else
+          new_rows = get_rows_for(indexes, by_position: false)
+          indexes.map { |index| new_rows.map { |r| r[index] } }
+        end
+      end
+
+      # Split the dataframe into many dataframes based on category vector
+      # @param [object] cat_name name of category vector to split the dataframe
+      # @return [Array] array of dataframes split by category with category vector
+      #   used to split not included
+      # @example
+      #   df = DaruLite::DataFrame.new({
+      #     a: [1, 2, 3],
+      #     b: ['a', 'a', 'b']
+      #   })
+      #   df.to_category :b
+      #   df.split_by_category :b
+      #   # => [#<DaruLite::DataFrame: a (2x1)>
+      #   #       a
+      #   #   0   1
+      #   #   1   2,
+      #   # #<DaruLite::DataFrame: b (1x1)>
+      #   #       a
+      #   #   2   3]
+      def split_by_category(cat_name)
+        cat_dv = self[cat_name]
+        raise ArgumentError, "#{cat_name} is not a category vector" unless
+          cat_dv.category?
+
+        cat_dv.categories.map do |cat|
+          where(cat_dv.eq cat)
+            .rename(cat)
+            .delete_vector cat_name
+        end
+      end
+
+      # Return the indexes of all the numeric vectors. Will include vectors with nils
+      # alongwith numbers.
+      def numeric_vectors
+        # FIXME: Why _with_index ?..
+        each_vector_with_index
+          .select { |vec, _i| vec.numeric? }
+          .map(&:last)
+      end
+
+      def numeric_vector_names
+        @vectors.select { |v| self[v].numeric? }
+      end
+
+      # Return a DataFrame of only the numerical Vectors. If clone: false
+      # is specified as option, only a *view* of the Vectors will be
+      # returned. Defaults to clone: true.
+      def only_numerics(opts = {})
+        cln = opts[:clone] != false
+        arry = numeric_vectors.map { |v| self[v] }
+
+        order = Index.new(numeric_vectors)
+        DaruLite::DataFrame.new(arry, clone: cln, order: order, index: @index)
+      end
+
+      private
+
+      def access_vector(*names)
+        if names.first.is_a?(Range)
+          dup(@vectors.subset(names.first))
+        elsif @vectors.is_a?(MultiIndex)
+          access_vector_multi_index(*names)
+        else
+          access_vector_single_index(*names)
+        end
+      end
+
+      def access_vector_multi_index(*names)
+        pos = @vectors[names]
+
+        return @data[pos] if pos.is_a?(Integer)
+
+        new_vectors = pos.map { |tuple| @data[@vectors[tuple]] }
+
+        pos = pos.drop_left_level(names.size) if names.size < @vectors.width
+
+        DaruLite::DataFrame.new(new_vectors, index: @index, order: pos)
+      end
+
+      def access_vector_single_index(*names)
+        if names.count < 2
+          begin
+            pos = @vectors.is_a?(DaruLite::DateTimeIndex) ? @vectors[names.first] : @vectors.pos(names.first)
+          rescue IndexError
+            raise IndexError, "Specified vector #{names.first} does not exist"
+          end
+          return @data[pos] if pos.is_a?(Numeric)
+
+          names = pos
+        end
+
+        new_vectors = names.map { |name| [name, @data[@vectors.pos(name)]] }.to_h
+
+        order = names.is_a?(Array) ? DaruLite::Index.new(names) : names
+        DaruLite::DataFrame.new(new_vectors, order: order, index: @index, name: @name)
+      end
+
+      def access_row(*indexes)
+        positions = @index.pos(*indexes)
+
+        if positions.is_a? Numeric
+          row = get_rows_for([positions])
+          DaruLite::Vector.new row, index: @vectors, name: indexes.first
+        else
+          new_rows = get_rows_for(indexes, by_position: false)
+          DaruLite::DataFrame.new new_rows, index: @index.subset(*indexes), order: @vectors
+        end
+      end
+
+      # @param keys [Array] can be an array of positions (if by_position is true) or indexes (if by_position if false)
+      # because of coercion by DaruLite::Vector#at and DaruLite::Vector#[], can return either an Array of
+      #   values (representing a row) or an array of Vectors (that can be seen as rows)
+      def get_rows_for(keys, by_position: true)
+        raise unless keys.is_a?(Array)
+
+        if by_position
+          pos = keys
+          @data.map { |vector| vector.at(*pos) }
+        else
+          # TODO: for now (2018-07-27), it is different than using
+          #    get_rows_for(@index.pos(*keys))
+          #    because DaruLite::Vector#at and DaruLite::Vector#[] don't handle DaruLite::MultiIndex the same way
+          indexes = keys
+          @data.map { |vec| vec[*indexes] }
+        end
+      end
+
+      # coerce ranges, integers and array in appropriate ways
+      def coerce_positions(*positions, size)
+        if positions.size == 1
+          case positions.first
+          when Integer
+            positions.first
+          when Range
+            size.times.to_a[positions.first]
+          else
+            raise ArgumentError, 'Unknown position type.'
+          end
+        else
+          positions
+        end
+      end
+    end
+  end
+end