daru 0.1.3.1 → 0.1.4

data/lib/daru/plotting/nyaplot/vector.rb

@@ -0,0 +1,46 @@
+module Daru
+  module Plotting
+    module Vector
+      module NyaplotLibrary
+        # Plots a Vector with Nyaplot on IRuby using the given options. Yields the
+        # plot object (Nyaplot::Plot) and the diagram object (Nyaplot::Diagram)
+        # to the block, which can be used for setting various options as per the
+        # Nyaplot API.
+        #
+        # == Options
+        #   type (:scatter, :bar, :histogram), title, x_label, y_label, color(true/false)
+        #
+        # == Usage
+        #   vector = Daru::Vector.new [10,20,30,40], [:one, :two, :three, :four]
+        #   vector.plot(type: :bar) do |plot|
+        #     plot.title "My first plot"
+        #     plot.width 1200
+        #   end
+        def plot opts={}
+          options = {
+            type: :scatter
+          }.merge(opts)
+
+          x_axis  = options[:type] == :scatter ? Array.new(size) { |i| i } : @index.to_a
+          plot    = Nyaplot::Plot.new
+          diagram = create_diagram plot, options[:type], x_axis
+
+          yield plot, diagram if block_given?
+
+          plot.show
+        end
+
+        private
+
+        def create_diagram plot, type, x_axis
+          case type
+          when :box, :histogram
+            plot.add(type, @data.to_a)
+          else
+            plot.add(type, x_axis, @data.to_a)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/daru/vector.rb

@@ -1,16 +1,98 @@
 require 'daru/maths/arithmetic/vector.rb'
 require 'daru/maths/statistics/vector.rb'
-require 'daru/plotting/vector.rb'
+require 'daru/plotting/gruff.rb'
+require 'daru/plotting/nyaplot.rb'
 require 'daru/accessors/array_wrapper.rb'
 require 'daru/accessors/nmatrix_wrapper.rb'
 require 'daru/accessors/gsl_wrapper.rb'
+require 'daru/category.rb'
 
 module Daru
-  class Vector
+  class Vector # rubocop:disable Metrics/ClassLength
     include Enumerable
     include Daru::Maths::Arithmetic::Vector
     include Daru::Maths::Statistics::Vector
-    include Daru::Plotting::Vector if Daru.has_nyaplot?
+    extend Gem::Deprecate
+
+    class << self
+      # Create a new vector by specifying the size and an optional value
+      # and block to generate values.
+      #
+      # == Description
+      #
+      # The *new_with_size* class method lets you create a Daru::Vector
+      # by specifying the size as the argument. The optional block, if
+      # supplied, is run once for populating each element in the Vector.
+      #
+      # The result of each run of the block is the value that is ultimately
+      # assigned to that position in the Vector.
+      #
+      # == Options
+      # :value
+      # All the rest like .new
+      def new_with_size n, opts={}, &block
+        value = opts.delete :value
+        block ||= ->(_) { value }
+        Daru::Vector.new Array.new(n, &block), opts
+      end
+
+      # Create a vector using (almost) any object
+      # * Array: flattened
+      # * Range: transformed using to_a
+      # * Daru::Vector
+      # * Numeric and string values
+      #
+      # == Description
+      #
+      # The `Vector.[]` class method creates a vector from almost any
+      # object that has a `#to_a` method defined on it. It is similar
+      # to R's `c` method.
+      #
+      # == Usage
+      #
+      #   a = Daru::Vector[1,2,3,4,6..10]
+      #   #=>
+      #   # <Daru::Vector:99448510 @name = nil @size = 9 >
+      #   #   nil
+      #   # 0   1
+      #   # 1   2
+      #   # 2   3
+      #   # 3   4
+      #   # 4   6
+      #   # 5   7
+      #   # 6   8
+      #   # 7   9
+      #   # 8  10
+      def [](*indexes)
+        values = indexes.map do |a|
+          a.respond_to?(:to_a) ? a.to_a : a
+        end.flatten
+        Daru::Vector.new(values)
+      end
+
+      def _load(data) # :nodoc:
+        h = Marshal.load(data)
+        Daru::Vector.new(h[:data],
+          index: h[:index],
+          name: h[:name],
+          dtype: h[:dtype], missing_values: h[:missing_values])
+      end
+
+      def coerce(data, options={})
+        case data
+        when Daru::Vector
+          data
+        when Array, Hash
+          new(data, options)
+        else
+          raise ArgumentError, "Can't coerce #{data.class} to #{self}"
+        end
+      end
+    end
+
+    def size
+      @data.size
+    end
 
     def each(&block)
       return to_enum(:each) unless block_given?
@@ -26,17 +108,17 @@ module Daru
       self
     end
 
-    def each_with_index
+    def each_with_index &block
       return to_enum(:each_with_index) unless block_given?
 
-      @index.each { |i| yield(self[i], i) }
+      @data.to_a.zip(@index.to_a).each(&block)
+
       self
     end
 
     def map!(&block)
       return to_enum(:map!) unless block_given?
       @data.map!(&block)
-      update
       self
     end
 
@@ -44,8 +126,6 @@ module Daru
     attr_reader :name
     # The row index. Can be either Daru::Index or Daru::MultiIndex.
     attr_reader :index
-    # The total number of elements of the vector.
-    attr_reader :size
     # The underlying dtype of the Vector. Can be either :array, :nmatrix or :gsl.
     attr_reader :dtype
     # If the dtype is :nmatrix, this attribute represents the data type of the
@@ -54,13 +134,16 @@ module Daru
     attr_reader :nm_dtype
     # An Array or the positions in the vector that are being treated as 'missing'.
     attr_reader :missing_positions
+    deprecate :missing_positions, :indexes, 2016, 10
     # Store a hash of labels for values. Supplementary only. Recommend using index
     # for proper usage.
     attr_accessor :labels
     # Store vector data in an array
     attr_reader :data
-    # Attach arbitrary metadata to vector (usu a hash)
-    attr_accessor :metadata
+    # Ploting library being used for this vector
+    attr_reader :plotting_library
+    # TODO: Make private.
+    attr_reader :nil_positions, :nan_positions
 
     # Create a Vector object.
     #
@@ -93,102 +176,27 @@ module Daru
     #   vecarr = Daru::Vector.new [1,2,3,4], index: [:a, :e, :i, :o]
     #   vechsh = Daru::Vector.new({a: 1, e: 2, i: 3, o: 4})
     def initialize source, opts={}
-      index = nil
-      if source.is_a?(Hash)
-        index  = source.keys
-        source = source.values
+      if opts[:type] == :category
+        # Initialize category type vector
+        extend Daru::Category
+        initialize_category source, opts
       else
-        index  = opts[:index]
-        source ||= []
+        # Initialize non-category type vector
+        initialize_vector source, opts
       end
-      name = opts[:name]
-      set_name name
-
-      @metadata = opts[:metadata] || {}
-
-      @data  = cast_vector_to(opts[:dtype] || :array, source, opts[:nm_dtype])
-      @index = try_create_index(index || @data.size)
-
-      if @index.size > @data.size
-        cast(dtype: :array) # NM with nils seg faults
-        (@index.size - @data.size).times { @data << nil }
-      elsif @index.size < @data.size
-        raise IndexError, "Expected index size >= vector size. Index size : #{@index.size}, vector size : #{@data.size}"
-      end
-
-      @possibly_changed_type = true
-      set_missing_values opts[:missing_values]
-      set_missing_positions
-      set_size
     end
 
-    # Create a new vector by specifying the size and an optional value
-    # and block to generate values.
-    #
-    # == Description
-    #
-    # The *new_with_size* class method lets you create a Daru::Vector
-    # by specifying the size as the argument. The optional block, if
-    # supplied, is run once for populating each element in the Vector.
-    #
-    # The result of each run of the block is the value that is ultimately
-    # assigned to that position in the Vector.
-    #
-    # == Options
-    # :value
-    # All the rest like .new
-    def self.new_with_size n, opts={}, &block
-      value = opts[:value]
-      opts.delete :value
-      if block
-        Daru::Vector.new Array.new(n) { |i| block.call(i) }, opts
+    def plotting_library= lib
+      case lib
+      when :gruff, :nyaplot
+        @plotting_library = lib
+        extend Module.const_get(
+          "Daru::Plotting::Vector::#{lib.to_s.capitalize}Library"
+        ) if Daru.send("has_#{lib}?".to_sym)
       else
-        Daru::Vector.new Array.new(n) { value }, opts
-      end
-    end
-
-    # Create a vector using (almost) any object
-    # * Array: flattened
-    # * Range: transformed using to_a
-    # * Daru::Vector
-    # * Numeric and string values
-    #
-    # == Description
-    #
-    # The `Vector.[]` class method creates a vector from almost any
-    # object that has a `#to_a` method defined on it. It is similar
-    # to R's `c` method.
-    #
-    # == Usage
-    #
-    #   a = Daru::Vector[1,2,3,4,6..10]
-    #   #=>
-    #   # <Daru::Vector:99448510 @name = nil @size = 9 >
-    #   #   nil
-    #   # 0   1
-    #   # 1   2
-    #   # 2   3
-    #   # 3   4
-    #   # 4   6
-    #   # 5   7
-    #   # 6   8
-    #   # 7   9
-    #   # 8  10
-    def self.[](*args)
-      values = []
-      args.each do |a|
-        case a
-        when Array
-          values.concat a.flatten
-        when Daru::Vector
-          values.concat a.to_a
-        when Range
-          values.concat a.to_a
-        else
-          values << a
-        end
+        raise ArguementError, "Plotting library #{lib} not supported. "\
+          'Supported libraries are :nyaplot and :gruff'
       end
-      Daru::Vector.new(values)
     end
 
     # Get one or more elements with specified index or a range.
@@ -203,19 +211,63 @@ module Daru
     #   # For vectors employing hierarchial multi index
     #
     def [](*input_indexes)
-      # Get a proper index object
-      indexes = @index[*input_indexes]
+      # Get array of positions indexes
+      positions = @index.pos(*input_indexes)
 
       # If one object is asked return it
-      return @data[indexes] if indexes.is_a? Numeric
+      return @data[positions] if positions.is_a? Numeric
 
-      # Form a new Vector using indexes and return it
+      # Form a new Vector using positional indexes
       Daru::Vector.new(
-        indexes.map { |loc| @data[@index[loc]] },
-        name: @name, metadata: @metadata.dup, index: indexes.conform(input_indexes), dtype: @dtype
+        positions.map { |loc| @data[loc] },
+        name: @name,
+        index: @index.subset(*input_indexes), dtype: @dtype
       )
     end
 
+    # Returns vector of values given positional values
+    # @param [Array<object>] *positions positional values
+    # @return [object] vector
+    # @example
+    #   dv = Daru::Vector.new 'a'..'e'
+    #   dv.at 0, 1, 2
+    #   # => #<Daru::Vector(3)>
+    #   #   0   a
+    #   #   1   b
+    #   #   2   c
+    def at *positions
+      # to be used to form index
+      original_positions = positions
+      positions = coerce_positions(*positions)
+      validate_positions(*positions)
+
+      if positions.is_a? Integer
+        @data[positions]
+      else
+        values = positions.map { |pos| @data[pos] }
+        Daru::Vector.new values, index: @index.at(*original_positions), dtype: dtype
+      end
+    end
+
+    # Change value at given positions
+    # @param [Array<object>] *positions positional values
+    # @param [object] val value to assign
+    # @example
+    #   dv = Daru::Vector.new 'a'..'e'
+    #   dv.set_at [0, 1], 'x'
+    #   dv
+    #   # => #<Daru::Vector(5)>
+    #   #   0   x
+    #   #   1   x
+    #   #   2   c
+    #   #   3   d
+    #   #   4   e
+    def set_at positions, val
+      validate_positions(*positions)
+      positions.map { |pos| @data[pos] = val }
+      update_position_cache
+    end
+
     # Just like in Hashes, you can specify the index label of the Daru::Vector
     # and assign an element an that place in the Daru::Vector.
     #
@@ -229,57 +281,14 @@ module Daru
     #   #  a 999
     #   #  b   2
     #   #  c   3
-    def []=(*location, value)
-      cast(dtype: :array) if value.nil? && dtype != :array
-
-      @possibly_changed_type = true if @type == :object  && (value.nil? ||
-        value.is_a?(Numeric))
-      @possibly_changed_type = true if @type == :numeric && (!value.is_a?(Numeric) &&
-        !value.nil?)
-
-      pos = @index[*location]
-
-      if pos.is_a?(Numeric)
-        @data[pos] = value
-      else
-        begin
-          pos.each { |tuple| self[tuple] = value }
-        rescue NoMethodError
-          raise IndexError, "Specified index #{pos.inspect} does not exist."
-        end
-      end
-
-      set_size
-      set_missing_positions unless Daru.lazy_update
-    end
+    def []=(*indexes, val)
+      cast(dtype: :array) if val.nil? && dtype != :array
 
-    # The values to be treated as 'missing'. *nil* is the default missing
-    # type. To set missing values see the missing_values= method.
-    def missing_values
-      @missing_values.keys
-    end
+      guard_type_check(val)
 
-    # Assign an Array to treat certain values as 'missing'.
-    #
-    # == Usage
-    #
-    #   v = Daru::Vector.new [1,2,3,4,5]
-    #   v.missing_values = [3]
-    #   v.update
-    #   v.missing_positions
-    #   #=> [2]
-    def missing_values= values
-      set_missing_values values
-      set_missing_positions unless Daru.lazy_update
-    end
+      modify_vector(indexes, val)
 
-    # Method for updating the metadata (i.e. missing value positions) of the
-    # after assingment/deletion etc. are complete. This is provided so that
-    # time is not wasted in creating the metadata for the vector each time
-    # assignment/deletion of elements is done. Updating data this way is called
-    # lazy loading. To set or unset lazy loading, see the .lazy_update= method.
-    def update
-      Daru.lazy_update and set_missing_positions
+      update_position_cache
     end
 
     # Two vectors are equal if the have the exact same index values corresponding
@@ -287,7 +296,7 @@ module Daru
     def == other
       case other
       when Daru::Vector
-        @index == other.index && @size == other.size &&
+        @index == other.index && size == other.size &&
           @index.all? { |index| self[index] == other[index] }
       else
         super
@@ -405,8 +414,8 @@ module Daru
     #   # 11   5
     #   # 13   5
     #   # 15   1
-    def where bool_arry
-      Daru::Core::Query.vector_where @data.to_a, @index.to_a, bool_arry, dtype
+    def where bool_array
+      Daru::Core::Query.vector_where self, bool_array
     end
 
     def head q=10
@@ -414,18 +423,41 @@ module Daru
     end
 
     def tail q=10
-      self[(@size - q)..(@size-1)]
+      start = [size - q, 0].max
+      self[start..(size-1)]
     end
 
     def empty?
       @index.empty?
     end
 
+    def numeric?
+      type == :numeric
+    end
+
+    def object?
+      type == :object
+    end
+
     # Reports whether missing data is present in the Vector.
     def has_missing_data?
-      !missing_positions.empty?
+      !indexes(*Daru::MISSING_VALUES).empty?
     end
     alias :flawed? :has_missing_data?
+    deprecate :has_missing_data?, :include_values?, 2016, 10
+    deprecate :flawed?, :include_values?, 2016, 10
+
+    # Check if any one of mentioned values occur in the vector
+    # @param [Array] *values values to check for
+    # @return [true, false] returns true if any one of specified values
+    #   occur in the vector
+    # @example
+    #   dv = Daru::Vector.new [1, 2, 3, 4, nil]
+    #   dv.include_values? nil, Float::NAN
+    #   # => true
+    def include_values?(*values)
+      values.any? { |v| include_with_nan? @data, v }
+    end
 
     # Append an element to the vector by specifying the element and index
     def concat element, index
@@ -434,8 +466,7 @@ module Daru
       @index |= [index]
       @data[@index[index]] = element
 
-      set_size
-      set_missing_positions unless Daru.lazy_update
+      update_position_cache
     end
     alias :push :concat
     alias :<< :concat
@@ -463,8 +494,7 @@ module Daru
       @data.delete_at @index[index]
       @index = Daru::Index.new(@index.to_a - [index])
 
-      set_size
-      set_missing_positions unless Daru.lazy_update
+      update_position_cache
     end
 
     # The type of data contained in the vector. Can be :object or :numeric. If
@@ -489,6 +519,16 @@ module Daru
       @type
     end
 
+    # Tells if vector is categorical or not.
+    # @return [true, false] true if vector is of type category, false otherwise
+    # @example
+    #   dv = Daru::Vector.new [1, 2, 3], type: :category
+    #   dv.category?
+    #   # => true
+    def category?
+      type == :category
+    end
+
     # Get index of element
     def index_of element
       case dtype
@@ -500,11 +540,9 @@ module Daru
     # Keep only unique elements of the vector alongwith their indexes.
     def uniq
       uniq_vector = @data.uniq
-      new_index   = uniq_vector.each_with_object([]) do |element, acc|
-        acc << index_of(element)
-      end
+      new_index   = uniq_vector.map { |element| index_of(element) }
 
-      Daru::Vector.new uniq_vector, name: @name, metadata: @metadata.dup, index: new_index, dtype: @dtype
+      Daru::Vector.new uniq_vector, name: @name, index: new_index, dtype: @dtype
     end
 
     def any? &block
@@ -531,47 +569,46 @@ module Daru
     #   v = Daru::Vector.new ["My first guitar", "jazz", "guitar"]
     #   # Say you want to sort these strings by length.
     #   v.sort(ascending: false) { |a,b| a.length <=> b.length }
-    def sort opts={}
-      opts = {
-        ascending: true
-      }.merge(opts)
-
-      vector_index = @data.each_with_index
-      vector_index =
-        if block_given?
-          vector_index.sort { |a,b| yield(a[0], b[0]) }
-        else
-          vector_index.sort { |(av, ai), (bv, bi)|
-            if !av.nil? && !bv.nil?
-              av <=> bv
-            elsif av.nil? && bv.nil?
-              ai <=> bi
-            elsif av.nil?
-              opts[:ascending] ? -1 : 1
-            else
-              opts[:ascending] ? 1 : -1
-            end
-          }
-        end
-      vector_index.reverse! unless opts[:ascending]
+    def sort opts={}, &block
+      opts = {ascending: true}.merge(opts)
+
+      vector_index = resort_index(@data.each_with_index, opts, &block)
       vector, index = vector_index.transpose
-      old_index = @index.to_a
-      index = index.map { |i| old_index[i] }
 
-      Daru::Vector.new(vector, index: index, name: @name, metadata: @metadata.dup, dtype: @dtype)
+      index = @index.reorder index
+
+      Daru::Vector.new(vector, index: index, name: @name, dtype: @dtype)
+    end
+
+    DEFAULT_SORTER = lambda { |(lv, li), (rv, ri)|
+      case
+      when lv.nil? && rv.nil?
+        li <=> ri
+      when lv.nil?
+        -1
+      when rv.nil?
+        1
+      else
+        lv <=> rv
+      end
+    }
+
+    def resort_index vector_index, opts
+      if block_given?
+        vector_index.sort { |(lv, _li), (rv, _ri)| yield(lv, rv) }
+      else
+        vector_index.sort(&DEFAULT_SORTER)
+      end
+        .tap { |res| res.reverse! unless opts[:ascending] }
     end
 
     # Just sort the data and get an Array in return using Enumerable#sort.
     # Non-destructive.
+    # :nocov:
     def sorted_data &block
       @data.to_a.sort(&block)
     end
-
-    # Returns *true* if the value passed is actually exists or is not marked as
-    # a *missing value*.
-    def exists? value
-      !@missing_values.key?(self[index_of(value)])
-    end
+    # :nocov:
 
     # Like map, but returns a Daru::Vector with the returned values.
     def recode dt=nil, &block
@@ -593,19 +630,12 @@ module Daru
     def delete_if
       return to_enum(:delete_if) unless block_given?
 
-      keep_e = []
-      keep_i = []
-      each_with_index do |n, i|
-        unless yield(n)
-          keep_e << n
-          keep_i << i
-        end
-      end
+      keep_e, keep_i = each_with_index.select { |n, _i| !yield(n) }.transpose
 
       @data = cast_vector_to @dtype, keep_e
       @index = Daru::Index.new(keep_i)
-      set_missing_positions unless Daru.lazy_update
-      set_size
+
+      update_position_cache
 
       self
     end
@@ -614,32 +644,16 @@ module Daru
     def keep_if
       return to_enum(:keep_if) unless block_given?
 
-      keep_e = []
-      keep_i = []
-      each_with_index do |n, i|
-        if yield(n)
-          keep_e << n
-          keep_i << i
-        end
-      end
-
-      @data = cast_vector_to @dtype, keep_e
-      @index = Daru::Index.new(keep_i)
-      set_missing_positions unless Daru.lazy_update
-      set_size
-
-      self
+      delete_if { |val| !yield(val) }
     end
 
     # Reports all values that doesn't comply with a condition.
     # Returns a hash with the index of data and the invalid data.
     def verify
-      h = {}
-      (0...size).each do |i|
-        h[i] = @data[i] unless yield(@data[i])
-      end
-
-      h
+      (0...size)
+        .map { |i| [i, @data[i]] }
+        .reject { |_i, val| yield(val) }
+        .to_h
     end
 
     # Return an Array with the data splitted by a separator.
@@ -674,29 +688,19 @@ module Daru
     #
     def split_by_separator sep=','
       split_data = splitted sep
-      factors = split_data.flatten.uniq.compact
-
-      out = factors.map { |x| [x, []] }.to_h
-
-      split_data.each do |r|
-        if r.nil?
-          factors.each do |f|
-            out[f].push(nil)
-          end
-        else
-          factors.each do |f|
-            out[f].push(r.include?(f) ? 1 : 0)
-          end
-        end
-      end
-
-      out.map { |k, v| [k, Daru::Vector.new(v)] }.to_h
+      split_data
+        .flatten.uniq.compact.map do |key|
+        [
+          key,
+          Daru::Vector.new(split_data.map { |v| split_value(key, v) })
+        ]
+      end.to_h
     end
 
     def split_by_separator_freq(sep=',')
-      split_by_separator(sep).map do |k, v|
-        [k, v.inject { |s,x| s+x.to_i }]
-      end.to_h
+      split_by_separator(sep).map { |k, v|
+        [k, v.map(&:to_i).inject(:+)]
+      }.to_h
     end
 
     def reset_index!
@@ -718,23 +722,15 @@ module Daru
     #   #  1  false
     #   #  2  false
     #   #  3  true
+    #
     def is_nil?
-      nil_truth_vector = clone_structure
-      @index.each do |idx|
-        nil_truth_vector[idx] = self[idx].nil? ? true : false
-      end
-
-      nil_truth_vector
+      # FIXME: EXTREMELY bad name for method not returning boolean - zverok, 2016-05-18
+      recode(&:nil?)
     end
 
     # Opposite of #is_nil?
     def not_nil?
-      nil_truth_vector = clone_structure
-      @index.each do |idx|
-        nil_truth_vector[idx] = self[idx].nil? ? false : true
-      end
-
-      nil_truth_vector
+      recode { |v| !v.nil? }
     end
 
     # Replace all nils in the vector with the value passed as an argument. Destructive.
@@ -744,7 +740,7 @@ module Daru
     #
     # * +replacement+ - The value which should replace all nils
     def replace_nils! replacement
-      missing_positions.each do |idx|
+      indexes(*Daru::MISSING_VALUES).each do |idx|
         self[idx] = replacement
       end
 
@@ -765,13 +761,13 @@ module Daru
     #   ts.lag   # => [nil, 0.69, 0.23, 0.44, ...]
     #   ts.lag(2) # => [nil, nil, 0.69, 0.23, ...]
     def lag k=1
-      return dup if k == 0
+      return dup if k.zero?
 
       dat = @data.to_a.dup
       (dat.size - 1).downto(k) { |i| dat[i] = dat[i - k] }
       (0...k).each { |i| dat[i] = nil }
 
-      Daru::Vector.new(dat, index: @index, name: @name, metadata: @metadata.dup)
+      Daru::Vector.new(dat, index: @index, name: @name)
     end
 
     def detach_index
@@ -788,7 +784,19 @@ module Daru
 
     # number of non-missing elements
     def n_valid
-      @size - missing_positions.size
+      size - indexes(*Daru::MISSING_VALUES).size
+    end
+    deprecate :n_valid, :count_values, 2016, 10
+
+    # Count the number of values specified
+    # @param [Array] *values values to count for
+    # @return [Integer] the number of times the values mentioned occurs
+    # @example
+    #   dv = Daru::Vector.new [1, 2, 1, 2, 3, 4, nil, nil]
+    #   dv.count_values nil
+    #   # => 2
+    def count_values(*values)
+      positions(*values).size
     end
 
     # Returns *true* if an index exists
@@ -796,6 +804,11 @@ module Daru
       @index.include? index
     end
 
+    # @return [Daru::DataFrame] the vector as a single-vector dataframe
+    def to_df
+      Daru::DataFrame.new({@name => @data}, name: @name, index: @index)
+    end
+
     # Convert Vector to a horizontal or vertical Ruby Matrix.
     #
     # == Arguments
@@ -811,11 +824,39 @@ module Daru
       end
     end
 
+    # Convert vector to nmatrix object
+    # @param [Symbol] axis :horizontal or :vertical
+    # @return [NMatrix] NMatrix object containing all values of the vector
+    # @example
+    #   dv = Daru::Vector.new [1, 2, 3]
+    #   dv.to_nmatrix
+    #   # =>
+    #   # [
+    #   #   [1, 2, 3] ]
+    def to_nmatrix axis=:horizontal
+      raise ArgumentError, 'Can not convert to nmatrix'\
+        'because the vector is numeric' unless numeric? && !include?(nil)
+
+      case axis
+      when :horizontal
+        NMatrix.new [1, size], to_a
+      when :vertical
+        NMatrix.new [size, 1], to_a
+      else
+        raise ArgumentError, 'Invalid axis specified. '\
+          'Valid axis are :horizontal and :vertical'
+      end
+    end
+
     # If dtype != gsl, will convert data to GSL::Vector with to_a. Otherwise returns
     # the stored GSL::Vector object.
     def to_gsl
       raise NoMethodError, 'Install gsl-nmatrix for access to this functionality.' unless Daru.has_gsl?
-      dtype == :gsl ? @data.data : GSL::Vector.alloc(only_valid(:array).to_a)
+      if dtype == :gsl
+        @data.data
+      else
+        GSL::Vector.alloc(reject_values(*Daru::MISSING_VALUES).to_a)
+      end
     end
 
     # Convert to hash (explicit). Hash keys are indexes and values are the correspoding elements
@@ -835,30 +876,12 @@ module Daru
 
     # Convert to html for iruby
     def to_html threshold=30
-      name = @name || 'nil'
-      html = '<table>' \
-        '<tr>' \
-          '<th colspan="2">' \
-            "Daru::Vector:#{object_id} " + " size: #{size}" \
-          '</th>' \
-        '</tr>'
-      html += '<tr><th> </th><th>' + name.to_s + '</th></tr>'
-      @index.each_with_index do |index, num|
-        html += '<tr><td>' + index.to_s + '</td>' + '<td>' + self[index].to_s + '</td></tr>'
-
-        next if num <= threshold
-        html += '<tr><td>...</td><td>...</td></tr>'
-
-        last_index = @index.to_a.last
-        html += '<tr>' \
-                  '<td>' + last_index.to_s       + '</td>' \
-                  '<td>' + self[last_index].to_s + '</td>' \
-                '</tr>'
-        break
-      end
-      html += '</table>'
-
-      html
+      path = if index.is_a?(MultiIndex)
+               File.expand_path('../iruby/templates/vector_mi.html.erb', __FILE__)
+             else
+               File.expand_path('../iruby/templates/vector.html.erb', __FILE__)
+             end
+      ERB.new(File.read(path).strip).result(binding)
     end
 
     def to_s
@@ -870,10 +893,11 @@ module Daru
       ReportBuilder.new(no_title: true).add(self).send(method)
     end
 
-    def report_building b
+    # :nocov:
+    def report_building b # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
       b.section(name: name) do |s|
         s.text "n :#{size}"
-        s.text "n valid:#{n_valid}"
+        s.text "n valid:#{count_values(*Daru::MISSING_VALUES)}"
         if @type == :object
           s.text  "factors: #{factors.to_a.join(',')}"
           s.text  "mode: #{mode}"
@@ -881,7 +905,7 @@ module Daru
           s.table(name: 'Distribution') do |t|
             frequencies.sort_by(&:to_s).each do |k,v|
               key = @index.include?(k) ? @index[k] : k
-              t.row [key, v, ('%0.2f%%' % (v.quo(n_valid)*100))]
+              t.row [key, v, ('%0.2f%%' % (v.quo(count_values(*Daru::MISSING_VALUES))*100))]
             end
           end
         end
@@ -898,47 +922,71 @@ module Daru
         end
       end
     end
+    # :nocov:
 
     # Over rides original inspect for pretty printing in irb
     def inspect spacing=20, threshold=15
-      longest =
-        [
-          @name.to_s.size,
-          (@index.to_a.map(&:to_s).map(&:size).max || 0),
-          (@data.map(&:to_s).map(&:size).max || 0),
-          3 # 'nil'.size
-        ].max
-
-      content   = ''
-      longest   = spacing if longest > spacing
-      name      = @name || 'nil'
-      metadata  = @metadata || 'nil'
-      formatter = "\n%#{longest}.#{longest}s %#{longest}.#{longest}s"
-      content  += "\n#<#{self.class}:#{object_id} @name = #{name} @metadata = #{metadata} @size = #{size} >"
-
-      content += formatter % ['', name]
-      @index.each_with_index do |index, num|
-        content += formatter % [index.to_s, (self[*index] || 'nil').to_s]
-        if num > threshold
-          content += formatter % ['...', '...']
-          break
-        end
+      row_headers = index.is_a?(MultiIndex) ? index.sparse_tuples : index.to_a
+
+      "#<#{self.class}(#{size})#{':cataegory' if category?}>\n" +
+        Formatters::Table.format(
+          to_a.lazy.map { |v| [v] },
+          headers: @name && [@name],
+          row_headers: row_headers,
+          threshold: threshold,
+          spacing: spacing
+        )
+    end
+
+    # Sets new index for vector. Preserves index->value correspondence.
+    # Sets nil for new index keys absent from original index.
+    # @note Unlike #reorder! which takes positions as input it takes
+    #   index as an input to reorder the vector
+    # @param [Daru::Index, Daru::MultiIndex] new_index new index to order with
+    # @return [Daru::Vector] vector reindexed with new index
+    def reindex! new_index
+      values = []
+      each_with_index do |val, i|
+        values[new_index[i]] = val if new_index.include?(i)
       end
-      content += "\n"
+      values.fill(nil, values.size, new_index.size - values.size)
+
+      @data = cast_vector_to @dtype, values
+      @index = new_index
 
-      content
+      update_position_cache
+
+      self
+    end
+
+    # Reorder the vector with given positions
+    # @note Unlike #reindex! which takes index as input, it takes
+    #   positions as an input to reorder the vector
+    # @param [Array] order the order to reorder the vector with
+    # @return reordered vector
+    # @example
+    #   dv = Daru::Vector.new [3, 2, 1], index: ['c', 'b', 'a']
+    #   dv.reorder! [2, 1, 0]
+    #   # => #<Daru::Vector(3)>
+    #   #   a   1
+    #   #   b   2
+    #   #   c   3
+    def reorder! order
+      @index = @index.reorder order
+      @data = order.map { |i| @data[i] }
+      update_position_cache
+      self
+    end
+
+    # Non-destructive version of #reorder!
+    def reorder order
+      dup.reorder! order
     end
 
     # Create a new vector with a different index, and preserve the indexing of
     # current elements.
     def reindex new_index
-      vector = Daru::Vector.new([], index: new_index, name: @name, metadata: @metadata.dup)
-
-      new_index.each do |idx|
-        vector[idx] = @index.include?(idx) ? self[idx] : nil
-      end
-
-      vector
+      dup.reindex!(new_index)
     end
 
     def index= idx
@@ -956,17 +1004,16 @@ module Daru
     #
     # @param new_name [Symbol] The new name.
     def rename new_name
-      if new_name.is_a?(Numeric)
-        @name = new_name
-        return
-      end
-
       @name = new_name
+      self
     end
 
-    # Duplicate elements and indexes
+    alias_method :name=, :rename
+
+    # Duplicated a vector
+    # @return [Daru::Vector] duplicated vector
     def dup
-      Daru::Vector.new @data.dup, name: @name, metadata: @metadata.dup, index: @index.dup
+      Daru::Vector.new @data.dup, name: @name, index: @index.dup
     end
 
     # == Bootstrap
@@ -1019,8 +1066,8 @@ module Daru
     #
     # == Reference:
     # * Sawyer, S. (2005). Resampling Data: Using a Statistical Jacknife.
-    def jackknife(estimators, k=1)
-      raise "n should be divisible by k:#{k}" unless size % k==0
+    def jackknife(estimators, k=1) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      raise "n should be divisible by k:#{k}" unless (size % k).zero?
 
       nb = (size / k).to_i
       h_est, es, ps = prepare_bootstrap(estimators)
@@ -1057,62 +1104,123 @@ module Daru
     # vector, setting this to false will return the same vector.
     # Otherwise, a duplicate will be returned irrespective of
     # presence of missing data.
-    def only_valid as_a=:vector, duplicate=true
-      return dup if !has_missing_data? && as_a == :vector && duplicate
-      return self if !has_missing_data? && as_a == :vector && !duplicate
-      return to_a if !has_missing_data? && as_a != :vector
-
-      new_index = @index.to_a - missing_positions
-      new_vector = new_index.map do |idx|
-        self[idx]
-      end
 
-      return new_vector if as_a != :vector
+    def only_valid as_a=:vector, _duplicate=true
+      # FIXME: Now duplicate is just ignored.
+      #   There are no spec that fail on this case, so I'll leave it
+      #   this way for now - zverok, 2016-05-07
 
-      Daru::Vector.new new_vector, index: new_index, name: @name, metadata: @metadata.dup, dtype: dtype
+      new_index = @index.to_a - indexes(*Daru::MISSING_VALUES)
+      new_vector = new_index.map { |idx| self[idx] }
+
+      if as_a == :vector
+        Daru::Vector.new new_vector, index: new_index, name: @name, dtype: dtype
+      else
+        new_vector
+      end
+    end
+    deprecate :only_valid, :reject_values, 2016, 10
+
+    # Return a vector with specified values removed
+    # @param [Array] *values values to reject from resultant vector
+    # @return [Daru::Vector] vector with specified values removed
+    # @example
+    #   dv = Daru::Vector.new [1, 2, nil, Float::NAN]
+    #   dv.reject_values nil, Float::NAN
+    #   # => #<Daru::Vector(2)>
+    #   #   0   1
+    #   #   1   2
+    def reject_values(*values)
+      resultant_pos = size.times.to_a - positions(*values)
+      dv = at(*resultant_pos)
+      # Handle the case when number of positions is 1
+      # and hence #at doesn't return a vector
+      if dv.is_a?(Daru::Vector)
+        dv
+      else
+        pos = resultant_pos.first
+        at(pos..pos)
+      end
+    end
+
+    # Return indexes of values specified
+    # @param [Array] *values values to find indexes for
+    # @return [Array] array of indexes of values specified
+    # @example
+    #   dv = Daru::Vector.new [1, 2, nil, Float::NAN], index: 11..14
+    #   dv.indexes nil, Float::NAN
+    #   # => [13, 14]
+    def indexes(*values)
+      index.to_a.values_at(*positions(*values))
+    end
+
+    # Replaces specified values with a new value
+    # @param [Array] old_values array of values to replace
+    # @param [object] new_value new value to replace with
+    # @note It performs the replace in place.
+    # @return [Daru::Vector] Same vector itself with values
+    #   replaced with new value
+    # @example
+    #   dv = Daru::Vector.new [1, 2, :a, :b]
+    #   dv.replace_values [:a, :b], nil
+    #   dv
+    #   # =>
+    #   # #<Daru::Vector:19903200 @name = nil @metadata = {} @size = 4 >
+    #   #     nil
+    #   #   0   1
+    #   #   1   2
+    #   #   2 nil
+    #   #   3 nil
+    def replace_values(old_values, new_value)
+      old_values = [old_values] unless old_values.is_a? Array
+      size.times do |pos|
+        set_at([pos], new_value) if include_with_nan? old_values, at(pos)
+      end
+      self
     end
 
     # Returns a Vector containing only missing data (preserves indexes).
     def only_missing as_a=:vector
       if as_a == :vector
-        self[*missing_positions]
+        self[*indexes(*Daru::MISSING_VALUES)]
       elsif as_a == :array
-        self[*missing_positions].to_a
+        self[*indexes(*Daru::MISSING_VALUES)].to_a
       end
     end
+    deprecate :only_missing, nil, 2016, 10
 
     # Returns a Vector with only numerical data. Missing data is included
     # but non-Numeric objects are excluded. Preserves index.
     def only_numerics
-      numeric_indexes = []
-
-      each_with_index do |v, i|
-        numeric_indexes << i if v.is_a?(Numeric) || @missing_values.key?(v)
-      end
+      numeric_indexes =
+        each_with_index
+        .select { |v, _i| v.is_a?(Numeric) || v.nil? }
+        .map(&:last)
 
       self[*numeric_indexes]
     end
 
+    DATE_REGEXP = /^(\d{2}-\d{2}-\d{4}|\d{4}-\d{2}-\d{2})$/
+
     # Returns the database type for the vector, according to its content
     def db_type
       # first, detect any character not number
-      if @data.find { |v| v.to_s=~/\d{2,2}-\d{2,2}-\d{4,4}/ } ||
-         @data.find { |v| v.to_s=~/\d{4,4}-\d{2,2}-\d{2,2}/ }
-
-        return 'DATE'
-      elsif @data.find { |v| v.to_s=~/[^0-9e.-]/ }
-        return 'VARCHAR (255)'
-      elsif @data.find { |v| v.to_s=~/\./ }
-        return 'DOUBLE'
+      case
+      when @data.any? { |v| v.to_s =~ DATE_REGEXP }
+        'DATE'
+      when @data.any? { |v| v.to_s =~ /[^0-9e.-]/ }
+        'VARCHAR (255)'
+      when @data.any? { |v| v.to_s =~ /\./ }
+        'DOUBLE'
       else
-        return 'INTEGER'
+        'INTEGER'
       end
     end
 
     # Copies the structure of the vector (i.e the index, size, etc.) and fills all
     # all values with nils.
     def clone_structure
-      Daru::Vector.new(([nil]*@size), name: @name, metadata: @metadata.dup, index: @index.dup)
+      Daru::Vector.new(([nil]*size), name: @name, index: @index.dup)
     end
 
     # Save the vector to a file
@@ -1129,38 +1237,156 @@ module Daru
         data:           @data.to_a,
         dtype:          @dtype,
         name:           @name,
-        metadata:       @metadata,
-        index:          @index,
-        missing_values: @missing_values
+        index:          @index
       )
     end
 
-    def self._load(data) # :nodoc:
-      h = Marshal.load(data)
-      Daru::Vector.new(h[:data],
-        index: h[:index],
-        name: h[:name], metadata: h[:metadata],
-        dtype: h[:dtype], missing_values: h[:missing_values])
-    end
-
+    # :nocov:
     def daru_vector(*)
       self
     end
+    # :nocov:
 
     alias :dv :daru_vector
 
+    # Converts a non category type vector to category type vector.
+    # @param [Hash] opts options to convert to category
+    # @option opts [true, false] :ordered Specify if vector is ordered or not.
+    #   If it is ordered, it can be sorted and min, max like functions would work
+    # @option opts [Array] :categories set categories in the specified order
+    # @return [Daru::Vector] vector with type category
+    def to_category opts={}
+      dv = Daru::Vector.new to_a, type: :category, name: @name, index: @index
+      dv.ordered = opts[:ordered] || false
+      dv.categories = opts[:categories] if opts[:categories]
+      dv
+    end
+
     def method_missing(name, *args, &block)
+      # FIXME: it is shamefully fragile. Should be either made stronger
+      # (string/symbol dychotomy, informative errors) or removed totally. - zverok
       if name =~ /(.+)\=/
-        self[name] = args[0]
+        self[$1.to_sym] = args[0]
       elsif has_index?(name)
         self[name]
       else
-        super(name, *args, &block)
+        super
+      end
+    end
+
+    def respond_to_missing?(name, include_private=false)
+      name.to_s.end_with?('=') || has_index?(name) || super
+    end
+
+    # Partition a numeric variable into categories.
+    # @param [Array<Numeric>] partitions an array whose consecutive elements
+    #   provide intervals for categories
+    # @param [Hash] opts options to cut the partition
+    # @option opts [:left, :right] :close_at specifies whether the interval closes at
+    #   the right side of left side
+    # @option opts [Array] :labels names of the categories
+    # @return [Daru::Vector] numeric variable converted to categorical variable
+    # @example
+    #   heights = Daru::Vector.new [30, 35, 32, 50, 42, 51]
+    #   height_cat = heights.cut [30, 40, 50, 60], labels=['low', 'medium', 'high']
+    #   # => #<Daru::Vector(6)>
+    #   #       0    low
+    #   #       1    low
+    #   #       2    low
+    #   #       3   high
+    #   #       4 medium
+    #   #       5   high
+    def cut partitions, opts={}
+      close_at, labels = opts[:close_at] || :right, opts[:labels]
+      partitions = partitions.to_a
+      values = to_a.map { |val| cut_find_category partitions, val, close_at }
+      cats = cut_categories(partitions, close_at)
+
+      dv = Daru::Vector.new values,
+        index: @index,
+        type: :category,
+        categories: cats
+
+      # Rename categories if new labels provided
+      if labels
+        dv.rename_categories Hash[cats.zip(labels)]
+      else
+        dv
+      end
+    end
+
+    def positions(*values)
+      case values
+      when [nil]
+        nil_positions
+      when [Float::NAN]
+        nan_positions
+      when [nil, Float::NAN], [Float::NAN, nil]
+        nil_positions + nan_positions
+      else
+        size.times.select { |i| include_with_nan? values, @data[i] }
       end
     end
 
     private
 
+    def nil_positions
+      @nil_positions ||
+        @nil_positions = size.times.select { |i| @data[i].nil? }
+    end
+
+    def nan_positions
+      @nan_positions ||
+        @nan_positions = size.times.select do |i|
+          @data[i].respond_to?(:nan?) && @data[i].nan?
+        end
+    end
+
+    def initialize_vector source, opts
+      index, source = parse_source(source, opts)
+      set_name opts[:name]
+
+      @data  = cast_vector_to(opts[:dtype] || :array, source, opts[:nm_dtype])
+      @index = Index.coerce(index || @data.size)
+
+      guard_sizes!
+
+      @possibly_changed_type = true
+      # Include plotting functionality
+      self.plotting_library = Daru.plotting_library
+    end
+
+    def parse_source source, opts
+      if source.is_a?(Hash)
+        [source.keys, source.values]
+      else
+        [opts[:index], source || []]
+      end
+    end
+
+    def guard_sizes!
+      if @index.size > @data.size
+        cast(dtype: :array) # NM with nils seg faults
+        @data.fill(nil, @data.size...@index.size)
+      elsif @index.size < @data.size
+        raise IndexError, "Expected index size >= vector size. Index size : #{@index.size}, vector size : #{@data.size}"
+      end
+    end
+
+    def guard_type_check value
+      @possibly_changed_type = true \
+        if object? && (value.nil? || value.is_a?(Numeric)) ||
+           numeric? && !value.is_a?(Numeric) && !value.nil?
+    end
+
+    def split_value key, v
+      case
+      when v.nil?           then nil
+      when v.include?(key)  then 1
+      else                       0
+      end
+    end
+
     # For an array or hash of estimators methods, returns
     # an array with three elements
     # 1.- A hash with estimators names as keys and lambdas as values
@@ -1180,18 +1406,6 @@ module Daru
       [h_est, h_est.keys, bss]
     end
 
-    def keep? a, b, order
-      eval = yield(a, b)
-      if order == :ascending
-        return true  if eval == -1
-        return false if eval == 1
-      elsif order == :descending
-        return false if eval == -1
-        return true  if eval == 1
-      end
-      false
-    end
-
     # Note: To maintain sanity, this _MUST_ be the _ONLY_ place in daru where the
     # @dtype variable is set and the underlying data type of vector changed.
     def cast_vector_to dtype, source=nil, nm_dtype=nil
@@ -1203,25 +1417,13 @@ module Daru
         when :nmatrix then Daru::Accessors::NMatrixWrapper.new(source, self, nm_dtype)
         when :gsl then Daru::Accessors::GSLWrapper.new(source, self)
         when :mdarray then raise NotImplementedError, 'MDArray not yet supported.'
-        else raise "Unknown dtype #{dtype}"
+        else raise ArgumentError, "Unknown dtype #{dtype}"
         end
 
       @dtype = dtype || :array
       new_vector
     end
 
-    def index_for index
-      if @index.include?(index)
-        @index[index]
-      elsif index.is_a?(Numeric)
-        index
-      end
-    end
-
-    def set_size
-      @size = @data.size
-    end
-
     def set_name name # rubocop:disable Style/AccessorMethodName
       @name =
         if name.is_a?(Numeric)  then name
@@ -1232,38 +1434,109 @@ module Daru
         end
     end
 
-    def set_missing_positions
-      @missing_positions = []
-      @index.each do |e|
-        @missing_positions << e if @missing_values.key?(self[e])
+    # Raises IndexError when one of the positions is an invalid position
+    def validate_positions *positions
+      positions = [positions] if positions.is_a? Integer
+      positions.each do |pos|
+        raise IndexError, "#{pos} is not a valid position." if pos >= size
       end
     end
 
-    def try_create_index potential_index
-      if potential_index.is_a?(Daru::MultiIndex) || potential_index.is_a?(Daru::Index)
-        potential_index
+    # coerce ranges, integers and array in appropriate ways
+    def coerce_positions *positions
+      if positions.size == 1
+        case positions.first
+        when Integer
+          positions.first
+        when Range
+          size.times.to_a[positions.first]
+        else
+          raise ArgumentError, 'Unkown position type.'
+        end
       else
-        Daru::Index.new(potential_index)
+        positions
       end
     end
 
-    def element_from_numeric_index location
-      pos = index_for location
-      pos ? @data[pos] : nil
+    # Helper method for []=.
+    # Assigs existing index to another value
+    def modify_vector(indexes, val)
+      positions = @index.pos(*indexes)
+
+      if positions.is_a? Numeric
+        @data[positions] = val
+      else
+        positions.each { |pos| @data[pos] = val }
+      end
     end
 
-    # Setup missing_values. The missing_values instance variable is set
-    # as a Hash for faster lookup times.
-    def set_missing_values values_arry # rubocop:disable Style/AccessorMethodName
-      @missing_values = {}
-      @missing_values[nil] = 0
-      if values_arry
-        values_arry.each do |e|
-          # If dtype is :gsl then missing values have to be converted to float
-          e = e.to_f if dtype == :gsl && e.is_a?(Numeric)
-          @missing_values[e] = 0
+    # Helper method for []=.
+    # Add a new index and assign it value
+    def insert_vector(indexes, val)
+      new_index = @index.add(*indexes)
+      # May be create +=
+      (new_index.size - @index.size).times { @data << val }
+      @index = new_index
+    end
+
+    # Works similar to #[]= but also insert the vector in case index is not valid
+    # It is there only to be accessed by Daru::DataFrame and not meant for user.
+    def set indexes, val
+      cast(dtype: :array) if val.nil? && dtype != :array
+      guard_type_check(val)
+
+      if @index.valid?(*indexes)
+        modify_vector(indexes, val)
+      else
+        insert_vector(indexes, val)
+      end
+
+      update_position_cache
+    end
+
+    def cut_find_category partitions, val, close_at
+      case close_at
+      when :right
+        right_index = partitions.index { |i| i > val }
+        raise ArgumentError, 'Invalid partition' if right_index.nil?
+        left_index = right_index - 1
+        "#{partitions[left_index]}-#{partitions[right_index]-1}"
+      when :left
+        right_index = partitions.index { |i| i >= val }
+        raise ArgumentError, 'Invalid partition' if right_index.nil?
+        left_index = right_index - 1
+        "#{partitions[left_index]+1}-#{partitions[right_index]}"
+      else
+        raise ArgumentError, "Invalid parameter #{close_at} to close_at."
+      end
+    end
+
+    def cut_categories partitions, close_at
+      case close_at
+      when :right
+        Array.new(partitions.size-1) do |left_index|
+          "#{partitions[left_index]}-#{partitions[left_index+1]-1}"
+        end
+      when :left
+        Array.new(partitions.size-1) do |left_index|
+          "#{partitions[left_index]+1}-#{partitions[left_index+1]}"
         end
       end
     end
+
+    def include_with_nan? array, value
+      # Returns true if value is included in array.
+      # Similar to include? but also works if value is Float::NAN
+      if value.respond_to?(:nan?) && value.nan?
+        array.any? { |i| i.respond_to?(:nan?) && i.nan? }
+      else
+        array.include? value
+      end
+    end
+
+    def update_position_cache
+      @nil_positions = nil
+      @nan_positions = nil
+    end
   end
 end