daru 0.0.5 → 0.1.0

data/Rakefile

@@ -1,4 +1,5 @@
 require 'rspec/core/rake_task'
+require 'bundler/gem_tasks'
 
 RSpec::Core::RakeTask.new(:spec)
 

data/daru.gemspec

@@ -1,9 +1,9 @@
 # coding: utf-8
 $:.unshift File.expand_path("../lib", __FILE__)
 
-require 'version.rb'
+require 'daru/version.rb'
 
-DESCRIPTION = <<MSG
+Daru::DESCRIPTION = <<MSG
 Daru (Data Analysis in RUby) is a library for analysis, manipulation and visualization
 of data.
 
@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ['Sameer Deshmukh']
   spec.email         = ['sameer.deshmukh93@gmail.com']
   spec.summary       = %q{Data Analysis in RUby}
-  spec.description   = DESCRIPTION
+  spec.description   = Daru::DESCRIPTION
   spec.homepage      = "http://github.com/v0dro/daru"
   spec.license       = 'BSD-2'
 
@@ -27,12 +27,16 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency 'bundler'
+  spec.add_runtime_dependency 'reportbuilder', '~> 1.4'
+  spec.add_runtime_dependency 'spreadsheet', '~> 1.0.3'
+
+  spec.add_development_dependency 'bundler', '~> 1.10'
   spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rserve-client', '~> 0.3'
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'awesome_print'
-  spec.add_development_dependency 'nyaplot'
-  if RUBY_ENGINE != 'jruby'
-    spec.add_development_dependency 'nmatrix', '~> 0.1.0'
-  end
+  spec.add_development_dependency 'nyaplot', '~> 0.1.5'
+  spec.add_development_dependency 'nmatrix', '~> 0.1.0'
+  spec.add_development_dependency 'distribution', '~> 0.7'
+  spec.add_development_dependency 'gsl-nmatrix', '~>1.17'
 end

data/lib/daru.rb

@@ -2,10 +2,45 @@ def jruby?
   RUBY_ENGINE == 'jruby'
 end
 
-require 'csv'
+module Daru
+  SPLIT_TOKEN = ','
+  class << self
+    @@lazy_update = false
+    
+    # A variable which will set whether Vector metadata is updated immediately or lazily.
+    # Call the #update method every time a values are set or removed in order to update
+    # metadata like positions of missing values.
+    attr_accessor :lazy_update
+    
+    def create_has_library(library)
+      define_singleton_method("has_#{library}?") do
+        cv = "@@#{library}"
+        unless class_variable_defined? cv
+          begin
+            require library.to_s
+            class_variable_set(cv, true)
+          rescue LoadError
+            class_variable_set(cv, false)
+          end
+        end
+        class_variable_get(cv)
+      end
+    end
+  end
+
+  create_has_library :gsl
+  create_has_library :nmatrix
+  create_has_library :nyaplot
+end
+
+autoload :Spreadsheet, 'spreadsheet'
+autoload :CSV, 'csv'
+
 require 'matrix'
 require 'securerandom'
+require 'reportbuilder'
 
+require 'daru/version.rb'
 require 'daru/index.rb'
 require 'daru/multi_index.rb'
 require 'daru/vector.rb'

data/lib/daru/accessors/array_wrapper.rb

@@ -3,13 +3,18 @@ module Daru
     # Internal class for wrapping ruby array
     class ArrayWrapper
       include Enumerable
+      extend Forwardable
 
+      def_delegators :@data, :slice!
+      
       def each(&block)
         @data.each(&block)
+        self
       end
 
       def map!(&block)
         @data.map!(&block)
+        self
       end
 
       attr_accessor :size
@@ -22,8 +27,8 @@ module Daru
         set_size
       end
 
-      def [] index
-        @data[index]
+      def [] *index
+        @data[*index]
       end
 
       def []= index, value
@@ -62,7 +67,7 @@ module Daru
       end
 
       def mean
-        sum.quo(@size - @context.nil_positions.size).to_f
+        sum.quo(@size - @context.missing_positions.size).to_f
       end
 
       def product

data/lib/daru/accessors/gsl_wrapper.rb

@@ -0,0 +1,113 @@
+module Daru
+  module Accessors
+    module GSLStatistics
+      def vector_standardized_compute(m,sd)
+        Daru::Vector.new @data.collect { |x| (x.to_f - m).quo(sd) }, dtype: :gsl,
+          index: @context.index, name: @context.name
+      end
+      
+      def vector_centered_compute(m)
+        Daru::Vector.new @data.collect {|x| (x.to_f - m)}, dtype: :gsl,
+          index: @context.index, name: @context.name
+      end
+
+      def sample_with_replacement(sample=1)
+        r = GSL::Rng.alloc(GSL::Rng::MT19937,rand(10000))
+        Daru::Vector.new(r.sample(@data, sample).to_a, dtype: :gsl, 
+          index: @context.index, name: @context.name)
+      end
+      
+      def sample_without_replacement(sample=1)
+        r = GSL::Rng.alloc(GSL::Rng::MT19937,rand(10000))
+        r.choose(@data, sample).to_a
+      end
+
+      def median
+        GSL::Stats::median_from_sorted_data(@data.sort)
+      end
+      
+      def variance_sample(m)
+        @data.variance_m
+      end
+      
+      def standard_deviation_sample(m)
+        @data.sd(m)
+      end
+
+      def variance_population(m)
+        @data.variance_with_fixed_mean(m)
+      end
+
+      def standard_deviation_population m
+        @data.sd_with_fixed_mean(m)
+      end
+
+      def skew
+        @data.skew
+      end
+
+      def kurtosis
+        @data.kurtosis
+      end
+    end
+
+    class GSLWrapper
+      include Enumerable
+      extend Forwardable
+      include Daru::Accessors::GSLStatistics
+
+      def_delegators :@data, :[], :size, :to_a, :each, :mean, 
+        :sum, :prod, :max, :min
+
+      alias :product :prod
+
+      attr_reader :data
+
+      def each(&block)
+        @data.each(&block)
+        self  
+      end
+
+      def map!(&block)
+        @data.map!(&block)
+        self
+      end
+
+      def initialize data, context
+        @data = ::GSL::Vector.alloc(data)
+        @context = context
+      end
+
+      def []= index, element
+        if index == size
+          push element
+        else
+          @data[index] = element
+        end
+      end
+
+      def delete_at index
+        @data.delete_at index
+      end
+
+      def index key
+        @data.to_a.index key
+      end
+
+      def push value
+        @data = @data.concat value
+        self
+      end
+      alias :<< :push
+      alias :concat :push
+
+      def dup
+        GSLWrapper.new(@data.to_a, @context)
+      end
+
+      def == other
+        @data == other.data
+      end
+    end
+  end
+end if Daru.has_gsl?

data/lib/daru/accessors/nmatrix_wrapper.rb

@@ -1,9 +1,3 @@
-begin
-  require 'nmatrix' unless jruby?
-rescue LoadError => e
-  puts "Please install the nmatrix gem for fast and efficient data storage."
-end
-
 module Daru
   module Accessors
     # Internal class for wrapping NMatrix
@@ -12,23 +6,18 @@ module Daru
 
       def each(&block)
         @data[0...@size].each(&block)
-      end
-
-      def map(&block)
-        @data[0...@size].map(&block)
+        self
       end
 
       def map!(&block)
         @data = NMatrix.new [@size*2], map(&block).to_a, dtype: nm_dtype
+        self
       end
 
       def inject(*args, &block)
         @data[0...@size].inject(*args, &block)
       end
 
-      alias_method :recode, :map
-      alias_method :recode!, :map!
-
       attr_reader :size, :data, :nm_dtype
       
       def initialize vector, context, nm_dtype=:int32
@@ -39,8 +28,8 @@ module Daru
         # init with twice the storage for reducing the need to resize
       end
 
-      def [] index
-        return @data[index] if index < @size
+      def [] *index
+        return @data[*index] if index[0] < @size
         nil
       end
  
@@ -79,7 +68,7 @@ module Daru
       end
  
       def dup
-        NMatrixWrapper.new @data.to_a, @context, @nm_dtype
+        NMatrixWrapper.new @data[0...@size].to_a, @context, @nm_dtype
       end
 
       def resize size = @size*2
@@ -109,4 +98,4 @@ module Daru
       end
     end
   end
-end
+end if Daru.has_nmatrix?

data/lib/daru/core/group_by.rb

@@ -171,7 +171,6 @@ module Daru
         else
           arry.map { |e| e.is_a?(Numeric) ? e : e.to_sym }
         end
-
         symbolized_arry
       end
 

data/lib/daru/dataframe.rb

@@ -12,17 +12,82 @@ module Daru
 
     include Daru::Maths::Arithmetic::DataFrame
     include Daru::Maths::Statistics::DataFrame
-    include Daru::Plotting::DataFrame
+    include Daru::Plotting::DataFrame if Daru.has_nyaplot?
 
     class << self
-      # Load data from a CSV file. 
-      #   Arguments - path, options, block(optional)
+      # Load data from a CSV file. Specify an optional block to grab the CSV 
+      # object and pre-condition it (for example use the `convert` or 
+      # `header_convert` methods).
       # 
-      # Accepts a block for pre-conditioning of CSV data if any.
+      # == Arguments
+      # 
+      # * path - Path of the file to load specified as a String.
+      # 
+      # == Options
+      # 
+      # Accepts the same options as the Daru::DataFrame constructor and CSV.open()
+      # and uses those to eventually construct the resulting DataFrame.
+      #
+      # == Verbose Description
+      #
+      # You can specify all the options to the `.from_csv` function that you 
+      # do to the Ruby `CSV.read()` function, since this is what is used internally.
+      #
+      # For example, if the columns in your CSV file are separated by something 
+      # other that commas, you can use the `:col_sep` option. If you want to 
+      # convert numeric values to numbers and not keep them as strings, you can 
+      # use the `:converters` option and set it to `:numeric`.
+      #
+      # The `.from_csv` function uses the following defaults for reading CSV files 
+      # (that are passed into the `CSV.read()` function):
+      #
+      #   {
+      #     :col_sep           => ',',
+      #     :converters        => :numeric
+      #   }
       def from_csv path, opts={}, &block
         Daru::IO.from_csv path, opts, &block      
       end
 
+      # Read data from an Excel file into a DataFrame.
+      # 
+      # == Arguments
+      # 
+      # * path - Path of the file to be read.
+      # 
+      # == Options
+      # 
+      # *:worksheet_id - ID of the worksheet that is to be read.
+      def from_excel path, opts={}, &block      
+        Daru::IO.from_excel path, opts, &block
+      end
+
+      # Read a database query and returns a Dataset
+      #
+      # USE:
+      #
+      #  dbh = DBI.connect("DBI:Mysql:database:localhost", "user", "password")
+      #  Daru::DataFrame.from_sql(dbh, "SELECT * FROM test")
+      def from_sql dbh, query
+        Daru::IO.from_sql dbh, query
+      end
+
+      # Read the database from a plaintext file. For this method to work,
+      # the data should be present in a plain text file in columns. See
+      # spec/fixtures/bank2.dat for an example.
+      # 
+      # == Arguments
+      # 
+      # * path - Path of the file to be read.
+      # * fields - Vector names of the resulting database.
+      # 
+      # == Usage
+      # 
+      #   df = Daru::DataFrame.from_plaintext 'spec/fixtures/bank2.dat', [:v1,:v2,:v3,:v4,:v5,:v6]
+      def from_plaintext path, fields
+        Daru::IO.from_plaintext path, fields
+      end
+
       # Create DataFrame by specifying rows as an Array of Arrays or Array of
       # Daru::Vector objects.
       def rows source, opts={}
@@ -52,6 +117,58 @@ module Daru
 
         df
       end
+
+      # Generates a new dataset, using three vectors
+      # - Rows
+      # - Columns
+      # - Values
+      #
+      # For example, you have these values
+      #
+      #   x   y   v
+      #   a   a   0
+      #   a   b   1
+      #   b   a   1
+      #   b   b   0
+      #
+      # You obtain
+      #   id  a   b
+      #    a  0   1
+      #    b  1   0
+      #
+      # Useful to process outputs from databases
+      def crosstab_by_assignation rows, columns, values
+        raise "Three vectors should be equal size" if 
+          rows.size != columns.size or rows.size!=values.size
+
+        cols_values = columns.factors
+        cols_n      = cols_values.size
+
+        h_rows = rows.factors.inject({}) do |a,v| 
+          a[v] = cols_values.inject({}) do |a1,v1| 
+            a1[v1]=nil 
+            a1
+          end
+          a
+        end
+
+        values.each_index do |i|
+          h_rows[rows[i]][columns[i]] = values[i]
+        end
+        df = Daru::DataFrame.new({}, order: [:_id] + cols_values.to_a)
+
+        rows.factors.each do |row|
+          n_row = Array.new(cols_n+1)
+          n_row[0] = row
+          cols_values.each_index do |i|
+            n_row[i+1] = h_rows[row][cols_values[i]]
+          end
+
+          df.add_row(n_row)
+        end
+        df.update
+        df
+      end
     end
 
     # The vectors (columns) index of the DataFrame
@@ -67,8 +184,29 @@ module Daru
     attr_reader :size
 
     # DataFrame basically consists of an Array of Vector objects.
-    #   These objects are indexed by row and column by vectors and index Index objects.
-    #   Arguments - source, vectors, index, name.
+    # These objects are indexed by row and column by vectors and index Index objects.
+    #
+    # == Arguments
+    # 
+    # * source - Source from the DataFrame is to be initialized. Can be a Hash
+    # of names and vectors (array or Daru::Vector), an array of arrays or
+    # array of Daru::Vectors.
+    # 
+    # == Options
+    # 
+    # +:order+ - An *Array*/*Daru::Index*/*Daru::MultiIndex* containing the order in 
+    # which Vectors should appear in the DataFrame.
+    # 
+    # +:index+ - An *Array*/*Daru::Index*/*Daru::MultiIndex* containing the order
+    # in which rows of the DataFrame will be named.
+    # 
+    # +:name+  - A name for the DataFrame.
+    #
+    # +:clone+ - Specify as *true* or *false*. When set to false, and Vector
+    # objects are passed for the source, the Vector objects will not duplicated
+    # when creating the DataFrame. Will have no effect if Array is passed in 
+    # the source, or if the passed Daru::Vectors have different indexes. 
+    # Default to *true*.
     # 
     # == Usage
     #   df = Daru::DataFrame.new({a: [1,2,3,4], b: [6,7,8,9]}, order: [:b, :a], 
@@ -84,9 +222,12 @@ module Daru
     def initialize source, opts={}
       vectors = opts[:order]
       index   = opts[:index]
-      @name   = (opts[:name] || SecureRandom.uuid).to_sym
+      clone   = opts[:clone] == false ? false : true
       @data   = []
 
+      temp_name = opts[:name]
+      @name   = temp_name.is_a?(Numeric) ? temp_name : (temp_name || SecureRandom.uuid).to_sym
+
       if source.empty?
         @vectors = create_index vectors
         @index   = create_index index
@@ -109,7 +250,7 @@ module Daru
             vectors.each_with_index do |name, idx|
               hsh[name] = source[idx]
             end
-            initialize(hsh, index: index, order: vectors, name: @name)
+            initialize(hsh, index: index, order: vectors, name: @name, clone: clone)
           else # array of hashes
             if vectors.nil?
               @vectors = Daru::Index.new source[0].keys.map(&:to_sym)
@@ -143,13 +284,19 @@ module Daru
               all_indexes.flatten!.uniq!.sort!
 
               @index = Daru::Index.new all_indexes
+              clone = true
             end
-            @vectors.each do |vector|
-              @data << Daru::Vector.new([], name: vector, index: @index)
 
-              @index.each do |idx|
-                @data[@vectors[vector]][idx] = source[vector][idx]
+            if clone
+              @vectors.each do |vector|
+                @data << Daru::Vector.new([], name: vector, index: @index)
+
+                @index.each do |idx|
+                  @data[@vectors[vector]][idx] = source[vector][idx]
+                end
               end
+            else
+              @data.concat source.values
             end
           else
             @index = create_index(index || source.values[0].size)
@@ -163,6 +310,7 @@ module Daru
 
       set_size
       validate
+      update
     end
 
     # Access row or vector. Specify name of row/vector followed by axis(:row, :vector).
@@ -176,6 +324,7 @@ module Daru
       else
         axis = :vector
       end
+      names.map! { |e| e.respond_to?(:to_sym) ? e.to_sym : e }
 
       if axis == :vector
         access_vector *names
@@ -194,11 +343,14 @@ module Daru
     # of the vector will be matched against the row/vector indexes of the DataFrame
     # before an insertion is performed. Unmatched indexes will be set to nil.
     def []=(*args)
-      name   = args[0]
-      axis   = args[1]
+      axis = args.include?(:row) ? :row : :vector
+      args.delete :vector
+      args.delete :row
+
+      name = args[0..-2]
       vector = args[-1]
+      name.map! { |e| e.respond_to?(:to_sym) ? e.to_sym : e }
 
-      axis = (!axis.is_a?(Symbol) and (axis != :vector or axis != :row)) ? :vector : axis
       if axis == :vector
         insert_or_modify_vector name, vector
       elsif axis == :row        
@@ -222,6 +374,14 @@ module Daru
       vector[name]
     end
 
+    def add_row row, index=nil
+      self.row[index || @size] = row
+    end
+
+    def add_vector n, vector
+      self[n] = vector
+    end
+
     # Access a row or set/create a row. Refer #[] and #[]= docs for details.
     # 
     # == Usage
@@ -232,13 +392,77 @@ module Daru
     end
 
     # Duplicate the DataFrame entirely.
-    def dup
-      src = {}
-      @vectors.each do |vector|
-        src[vector] = @data[@vectors[vector]].dup
+    # 
+    # == Arguments
+    # 
+    # * +vectors_to_dup+ - An Array specifying the names of Vectors to 
+    # be duplicated. Will duplicate the entire DataFrame if not specified.
+    def dup vectors_to_dup=nil
+      vectors_to_dup = @vectors unless vectors_to_dup
+
+      new_order =
+      if vectors.is_a?(MultiIndex)
+        src = []
+        vectors_to_dup.each do |vec|
+          src << @data[@vectors[vec]].dup
+        end
+
+        Daru::MultiIndex.new(vectors_to_dup)
+      else
+        src = {}
+        vectors_to_dup.each do |vector|
+          src[vector] = @data[@vectors[vector]].dup
+        end
+
+        Daru::Index.new(vectors_to_dup)
+      end
+
+      Daru::DataFrame.new src, order: new_order, index: @index.dup, name: @name, clone: true
+    end
+
+    # Only clone the structure of the DataFrame.
+    def clone_structure
+      Daru::DataFrame.new([], order: @vectors.dup, index: @index.dup, name: @name)
+    end
+
+    # Returns a 'view' of the DataFrame, i.e the object ID's of vectors are
+    # preserved.
+    # 
+    # == Arguments
+    # 
+    # +vectors_to_clone+ - Names of vectors to clone. Optional. Will return
+    # a view of the whole data frame otherwise.
+    def clone *vectors_to_clone
+      vectors_to_clone.flatten! unless vectors_to_clone.all? { |a| !a.is_a?(Array) }
+      return super if vectors_to_clone.empty?
+
+      h = vectors_to_clone.inject({}) do |hsh, vec|
+        hsh[vec] = self[vec]
+        hsh
       end
+      Daru::DataFrame.new(h, clone: false)
+    end
 
-      Daru::DataFrame.new src, order: @vectors.dup, index: @index.dup, name: @name
+    # Returns a 'shallow' copy of DataFrame if missing data is not present, 
+    # or a full copy of only valid data if missing data is present.
+    def clone_only_valid
+      if has_missing_data?
+        dup_only_valid
+      else
+        clone
+      end
+    end
+
+    # Creates a new duplicate dataframe containing only rows 
+    # without a single missing value.
+    def dup_only_valid vecs=nil
+      rows_with_nil = @data.inject([]) do |memo, vector|
+        memo.concat vector.missing_positions
+        memo
+      end.uniq
+
+      row_indexes = @index.to_a
+      (vecs.nil? ? self : dup(vecs)).row[*(row_indexes - rows_with_nil)]
     end
 
     # Iterate over each vector
@@ -286,21 +510,205 @@ module Daru
       self
     end
 
-    # Map each vector. Returns a DataFrame whose vectors are modified according
-    # to the value returned by the block. As is the case with Enumerable#map,
-    # the object returned by each block must be a Daru::Vector for the dataframe
-    # to remain relevant.
+    # Iterate over each row or vector of the DataFrame. Specify axis
+    # by passing :vector or :row as the argument. Default to :vector.
+    #
+    # == Description
+    #
+    # `#each` works exactly like Array#each. The default mode for `each` 
+    # is to iterate over the columns of the DataFrame. To iterate over 
+    # rows you must pass the axis, i.e `:row` as an argument.
+    # 
+    # == Arguments
+    # 
+    # * +axis+ - The axis to iterate over. Can be :vector (or :column)
+    # or :row. Default to :vector.
+    def each axis=:vector, &block
+      if axis == :vector or axis == :column
+        each_vector(&block)
+      elsif axis == :row
+        each_row(&block)
+      else
+        raise ArgumentError, "Unknown axis #{axis}"
+      end
+    end
+
+    # Iterate over a row or vector and return results in a Daru::Vector.
+    # Specify axis with :vector or :row. Default to :vector.
+    #
+    # == Description
+    #
+    # The #collect iterator works similar to #map, the only difference 
+    # being that it returns a Daru::Vector comprising of the results of 
+    # each block run. The resultant Vector has the same index as that 
+    # of the axis over which collect has iterated. It also accepts the 
+    # optional axis argument.
+    #
+    # == Arguments
+    # 
+    # * +axis+ - The axis to iterate over. Can be :vector (or :column)
+    # or :row. Default to :vector.
+    def collect axis=:vector, &block
+      if axis == :vector or axis == :column
+        collect_vectors(&block)
+      elsif axis == :row
+        collect_rows(&block)
+      else
+        raise ArgumentError, "Unknown axis #{axis}"
+      end
+    end
+
+    # Map over each vector or row of the data frame according to
+    # the argument specified. Will return an Array of the resulting
+    # elements. To map over each row/vector and get a DataFrame,
+    # see #recode.
+    # 
+    # == Description
+    # 
+    # The #map iterator works like Array#map. The value returned by 
+    # each run of the block is added to an Array and the Array is 
+    # returned. This method also accepts an axis argument, like #each. 
+    # The default is :vector.
+    # 
+    # == Arguments
+    # 
+    # * +axis+ - The axis to map over. Can be :vector (or :column) or :row.
+    # Default to :vector.
+    def map axis=:vector, &block
+      if axis == :vector or axis == :column
+        map_vectors(&block)
+      elsif axis == :row
+        map_rows(&block)
+      else
+        raise ArgumentError, "Unknown axis #{axis}"
+      end
+    end
+
+    # Destructive map. Modifies the DataFrame. Each run of the block
+    # must return a Daru::Vector. You can specify the axis to map over
+    # as the argument. Default to :vector.
+    # 
+    # == Arguments
+    # 
+    # * +axis+ - The axis to map over. Can be :vector (or :column) or :row.
+    # Default to :vector.
+    def map! axis=:vector, &block
+      if axis == :vector or axis == :column
+        map_vectors!(&block)
+      elsif axis == :row
+        map_rows!(&block)
+      end
+    end
+
+    # Maps over the DataFrame and returns a DataFrame. Each run of the
+    # block must return a Daru::Vector object. You can specify the axis
+    # to map over. Default to :vector.
+    #
+    # == Description
+    #
+    # Recode works similarly to #map, but an important difference between 
+    # the two is that recode returns a modified Daru::DataFrame instead 
+    # of an Array. For this reason, #recodeexpects that every run of the 
+    # block to return a Daru::Vector.
+    #
+    # Just like map and each, recode also accepts an optional _axis_ argument.
+    # 
+    # == Arguments
+    # 
+    # * +axis+ - The axis to map over. Can be :vector (or :column) or :row.
+    # Default to :vector.
+    def recode axis=:vector, &block
+      if axis == :vector or axis == :column
+        recode_vectors(&block)
+      elsif axis == :row
+        recode_rows(&block)
+      end
+    end
+
+    # Retain vectors or rows if the block returns a truthy value.
+    # 
+    # == Description
+    # 
+    # For filtering out certain rows/vectors based on their values, 
+    # use the #filter method. By default it iterates over vectors and 
+    # keeps those vectors for which the block returns true. It accepts 
+    # an optional axis argument which lets you specify whether you want 
+    # to iterate over vectors or rows.
+    # 
+    # == Arguments
+    # 
+    # * +axis+ - The axis to map over. Can be :vector (or :column) or :row.
+    # Default to :vector.
+    # 
+    # == Usage
+    # 
+    #   # Filter vectors
+    #
+    #   df.filter do |vector|
+    #     vector.type == :numeric and vector.median < 50
+    #   end
+    #
+    #   # Filter rows
+    #
+    #   df.filter(:row) do |row|
+    #     row[:a] + row[:d] < 100
+    #   end
+    def filter axis=:vector, &block
+      if axis == :vector or axis == :column
+        filter_vectors(&block)
+      elsif axis == :row
+        filter_rows(&block)
+      end
+    end
+
+    def recode_vectors &block
+      block_given? or return to_enum(:recode_vectors) 
+
+      df = self.dup
+      df.each_vector_with_index do |v, i|
+        ret = yield v
+        ret.is_a?(Daru::Vector) or raise TypeError, "Every iteration must return Daru::Vector not #{ret.class}"
+        df[*i] = ret
+      end
+
+      df
+    end
+
+    def recode_rows &block
+      block_given? or return to_enum(:recode_rows)
+
+      df = self.dup
+      df.each_row_with_index do |r, i|
+        ret = yield r
+        ret.is_a?(Daru::Vector) or raise TypeError, "Every iteration must return Daru::Vector not #{ret.class}"
+        df.row[i] = ret
+      end
+
+      df
+    end
+
+    # Map each vector and return an Array.
     def map_vectors(&block)
       return to_enum(:map_vectors) unless block_given?
 
-      self.dup.map_vectors!(&block)
+      arry = []
+      @data.each do |vec|
+        arry << yield(vec)
+      end
+
+      arry
     end
 
     # Destructive form of #map_vectors
     def map_vectors!(&block)
       return to_enum(:map_vectors!) unless block_given?
 
-      @data.map!(&block)
+      vectors.dup.each do |n|
+        v = yield self[n]
+        v.is_a?(Daru::Vector) or raise TypeError, "Must return a Daru::Vector not #{v.class}"
+        self[n] = v
+      end
+
       self
     end
 
@@ -308,37 +716,114 @@ module Daru
     def map_vectors_with_index(&block)
       return to_enum(:map_vectors_with_index) unless block_given?
 
-      df = self.dup
-      df.each_vector_with_index do |vector, name|
-        df[name, :vector] = yield(vector, name)
+      dt = []
+      each_vector_with_index do |vector, name|
+        dt << yield(vector, name)
       end
 
-      df
+      dt
     end
 
     # Map each row
     def map_rows(&block)
       return to_enum(:map_rows) unless block_given?
 
-      df = self.dup
-      df.each_row_with_index do |row, index|
-        df[index, :row] = yield(row)
+      dt = []
+      each_row do |row|
+        dt << yield(row)
       end
 
-      df
+      dt
     end
 
     def map_rows_with_index(&block)
       return to_enum(:map_rows_with_index) unless block_given?
 
-      df = self.dup
-      df.each_row_with_index do |row, index|
-        df[index, :row] = yield(row, index)
+      dt = []
+      each_row_with_index do |row, index|
+        dt << yield(row, index)
       end
 
-      df
+      dt
     end
 
+    def map_rows!(&block)
+      return to_enum(:map_rows!) unless block_given?
+
+      index.dup.each do |i|
+        r = yield self.row[i]
+        r.is_a?(Daru::Vector) or raise TypeError, "Returned object must be Daru::Vector not #{r.class}"
+        self.row[i] = r
+      end
+
+      self
+    end
+
+    # Retrieves a Daru::Vector, based on the result of calculation 
+    # performed on each row.
+    def collect_rows &block
+      return to_enum(:collect_rows) unless block_given?
+
+      data = []
+      each_row do |row|
+        data.push yield(row)
+      end
+
+      Daru::Vector.new(data, index: @index)
+    end
+
+    def collect_row_with_index &block
+      return to_enum(:collect_row_with_index) unless block_given?
+
+      data = []
+      each_row_with_index do |row, i|
+        data.push yield(row, i)
+      end
+
+      Daru::Vector.new(data, index: @index)
+    end
+
+    # Retrives a Daru::Vector, based on the result of calculation
+    # performed on each vector.
+    def collect_vectors &block
+      return to_enum(:collect_vectors) unless block_given?
+
+      data = []
+      each_vector do |vec|
+        data.push yield(vec)
+      end
+
+      Daru::Vector.new(data, index: @vectors)
+    end
+
+    def collect_vector_with_index &block
+      return to_enum(:collect_vector_with_index) unless block_given?
+
+      data = []
+      each_vector_with_index do |vec, i|
+        data.push yield(vec, i)
+      end
+
+      Daru::Vector.new(data, index: @vectors)
+    end
+
+    # Generate a matrix, based on vector names of the DataFrame.
+    #
+    # @return {::Matrix}
+    def collect_matrix
+      return to_enum(:collect_matrix) unless block_given?
+
+      vecs = vectors.to_a
+      rows = vecs.collect { |row|
+        vecs.collect { |col|
+          yield row,col
+        }
+      }
+
+      Matrix.rows(rows)
+    end
+
+
     # Delete a vector
     def delete_vector vector
       if @vectors.include? vector
@@ -367,6 +852,20 @@ module Daru
       set_size
     end
 
+    # Creates a DataFrame with the random data, of n size.
+    # If n not given, uses original number of rows.
+    #
+    # @return {Daru::DataFrame}
+    def bootstrap(n=nil)
+      n ||= nrows
+      ds_boot = Daru::DataFrame.new({}, order: @vectors)
+      n.times do
+        ds_boot.add_row(row[rand(n)])
+      end
+      ds_boot.update
+      ds_boot
+    end
+
     def keep_row_if &block
       deletion = []
 
@@ -388,6 +887,16 @@ module Daru
       end
     end
 
+    # creates a new vector with the data of a given field which the block returns true
+    def filter_vector vec
+      d = []
+      each_row do |row|
+        d.push(row[vec]) if yield row
+      end
+
+      Daru::Vector.new(d)
+    end
+
     # Iterates over each row and retains it in a new DataFrame if the block returns
     # true for that row.
     def filter_rows &block
@@ -419,18 +928,160 @@ module Daru
       df
     end
 
+    # Test each row with one or more tests. Each test is a Proc with the form
+    # *Proc.new {|row| row[:age] > 0}*
+    # 
+    # The function returns an array with all errors.
+    def verify(*tests)
+      if(tests[0].is_a? Symbol)
+        id = tests[0]
+        tests.shift
+      else
+        id = @vectors.first
+      end
+
+      vr = []
+      i  = 0
+      each(:row) do |row|
+        i += 1
+        tests.each do |test|
+          if !test[2].call(row)
+            values = ""
+            if test[1].size>0
+              values = " (" + test[1].collect{ |k| "#{k}=#{row[k]}" }.join(", ") + ")"
+            end
+            vr.push("#{i} [#{row[id]}]: #{test[0]}#{values}")
+          end
+        end
+      end
+      vr
+    end
+
+    # DSL for yielding each row and returning a Daru::Vector based on the
+    # value each run of the block returns.
+    # 
+    # == Usage
+    # 
+    #   a1 = Daru::Vector.new([1, 2, 3, 4, 5, 6, 7])
+    #   a2 = Daru::Vector.new([10, 20, 30, 40, 50, 60, 70])
+    #   a3 = Daru::Vector.new([100, 200, 300, 400, 500, 600, 700])
+    #   ds = Daru::DataFrame.new({ :a => a1, :b => a2, :c => a3 })
+    #   total = ds.vector_by_calculation { a + b + c }
+    #   # <Daru::Vector:82314050 @name = nil @size = 7 >
+    #   #   nil
+    #   # 0 111
+    #   # 1 222
+    #   # 2 333
+    #   # 3 444
+    #   # 4 555
+    #   # 5 666
+    #   # 6 777
+    def vector_by_calculation &block
+      a = []
+      each_row do |r|
+        a.push r.instance_eval(&block)
+      end
+
+      Daru::Vector.new a, index: @index
+    end
+
+    # Returns a vector, based on a string with a calculation based
+    # on vector.
+    # 
+    # The calculation will be eval'ed, so you can put any variable
+    # or expression valid on ruby.
+    # 
+    # For example:
+    #   a = Daru::Vector.new [1,2]
+    #   b = Daru::Vector.new [3,4]
+    #   ds = Daru::DataFrame.new({:a => a,:b => b})
+    #   ds.compute("a+b")
+    #   => Vector [4,6]
+    def compute text, &block
+      return instance_eval(&block) if block_given?
+      instance_eval(text) 
+    end
+
+    # Return a vector with the number of missing values in each row.
+    # 
+    # == Arguments
+    # 
+    # * +missing_values+ - An Array of the values that should be 
+    # treated as 'missing'. The default missing value is *nil*.
+    def missing_values_rows missing_values=[nil]
+      number_of_missing = []
+      each_row do |row|
+        row.missing_values = missing_values
+        number_of_missing << row.missing_positions.size
+      end
+
+      Daru::Vector.new number_of_missing, index: @index, name: "#{@name}_missing_rows".to_sym
+    end
+
+    # TODO: remove next version
+    alias :vector_missing_values :missing_values_rows
+
+    def has_missing_data?
+      !!@data.any? { |v| v.has_missing_data? }
+    end
+
+    alias :flawed? :has_missing_data?
+
+    # Return a nested hash using vector names as keys and an array constructed of 
+    # hashes with other values. If block provided, is used to provide the
+    # values, with parameters +row+ of dataset, +current+ last hash on 
+    # hierarchy and +name+ of the key to include
+    def nest *tree_keys, &block
+      tree_keys = tree_keys[0] if tree_keys[0].is_a? Array
+      out = {}
+
+      each_row do |row|
+        current = out
+        # Create tree
+        tree_keys[0, tree_keys.size-1].each do |f|
+          root = row[f]
+          current[root] ||= {}
+          current = current[root]
+        end
+        name = row[tree_keys.last]
+        if !block
+          current[name] ||= []
+          current[name].push(row.to_hash.delete_if { |key,value| tree_keys.include? key})
+        else
+          current[name] = block.call(row, current,name)
+        end
+      end
+
+      out
+    end
+
+    def vector_count_characters vecs=nil
+      vecs ||= @vectors.to_a
+
+      collect_row_with_index do |row, i|
+        vecs.inject(0) do |memo, vec|
+          memo + (row[vec].nil? ? 0 : row[vec].to_s.size)
+        end
+      end
+    end
+
+    def add_vectors_by_split(name,join='-',sep=Daru::SPLIT_TOKEN)
+      split = self[name].split_by_separator(sep)
+      split.each { |k,v| self[(name.to_s + join + k.to_s).to_sym] = v }
+    end
+
     # Return the number of rows and columns of the DataFrame in an Array.
     def shape
       [@index.size, @vectors.size]
     end
 
     # The number of rows
-    def rows
+    def nrows
       shape[0]
     end
 
     # The number of vectors
-    def cols
+    def ncols
       shape[1]
     end
 
@@ -439,11 +1090,37 @@ module Daru
       !!@vectors[*vector]
     end
 
+    def any? axis=:vector, &block
+      if axis == :vector or axis == :column
+        @data.any?(&block)
+      elsif axis == :row
+        each_row do |row|
+          return true if yield(row)
+        end
+        return false
+      else
+        raise ArgumentError, "Unidentified axis #{axis}"
+      end
+    end
+
+    def all? axis=:vector, &block
+      if axis == :vector or axis == :column
+        @data.all?(&block)
+      elsif axis == :row
+        each_row do |row|
+          return false unless yield(row)
+        end
+        return true
+      else
+        raise ArgumentError, "Unidentified axis #{axis}"
+      end
+    end
+
     # The first ten elements of the DataFrame
     #
     # @param [Fixnum] quantity (10) The number of elements to display from the top.
     def head quantity=10
-      self[0..quantity, :row]
+      self[0..(quantity-1), :row]
     end
 
     # The last ten elements of the DataFrame
@@ -453,7 +1130,59 @@ module Daru
       self[(@size - quantity)..(@size-1), :row]
     end
 
-    # Group elements by vector to perform operations on them.
+    # Returns a vector with sum of all vectors specified in the argument. 
+    # Tf vecs parameter is empty, sum all numeric vector.
+    def vector_sum vecs=nil
+      vecs ||= numeric_vectors
+      sum = Daru::Vector.new [0]*@size, index: @index, name: @name, dtype: @dtype
+
+      vecs.each do |n|
+        sum += self[n]
+      end
+
+      sum
+    end
+
+    # Calculate mean of the rows of the dataframe.
+    # 
+    # == Arguments
+    # 
+    # * +max_missing+ - The maximum number of elements in the row that can be
+    # zero for the mean calculation to happen. Default to 0.
+    def vector_mean max_missing=0
+      mean_vec = Daru::Vector.new [0]*@size, index: @index, name: "mean_#{@name}"
+
+      each_row_with_index do |row, i|
+        mean_vec[i] = row.missing_positions.size > max_missing ? nil : row.mean
+      end
+
+      mean_vec
+    end
+
+    # Group elements by vector to perform operations on them. Returns a 
+    # Daru::Core::GroupBy object.See the Daru::Core::GroupBy docs for a detailed
+    # list of possible operations.
+    # 
+    # == Arguments
+    # 
+    # * vectors - An Array contatining names of vectors to group by.
+    # 
+    # == Usage
+    # 
+    #   df = Daru::DataFrame.new({
+    #     a: %w{foo bar foo bar   foo bar foo foo},
+    #     b: %w{one one two three two two one three},
+    #     c:   [1  ,2  ,3  ,1    ,3  ,6  ,3  ,8],
+    #     d:   [11 ,22 ,33 ,44   ,55 ,66 ,77 ,88]
+    #   })
+    #   df.group_by([:a,:b,:c]).groups
+    #   #=> {["bar", "one", 2]=>[1],
+    #   # ["bar", "three", 1]=>[3],
+    #   # ["bar", "two", 6]=>[5],
+    #   # ["foo", "one", 1]=>[0],
+    #   # ["foo", "one", 3]=>[6],
+    #   # ["foo", "three", 8]=>[7],
+    #   # ["foo", "two", 3]=>[2, 4]}
     def group_by vectors
       vectors = [vectors] if vectors.is_a?(Symbol)
       vectors.each { |v| raise(ArgumentError, "Vector #{v} does not exist") unless
@@ -462,6 +1191,13 @@ module Daru
       Daru::Core::GroupBy.new(self, vectors)
     end
 
+    def reindex_vectors! new_vectors
+      raise ArgumentError, "Number of vectors passed into function (#{new_vectors.size}) should equal that present in the DataFrame (#{@vectors.size})" if 
+        @vectors.size != new_vectors.size
+
+      @vectors = Daru::Index.new new_vectors.map(&:to_sym), new_vectors.map { |e| @vectors[e] }
+    end
+
     # Change the index of the DataFrame and its underlying vectors. Destructive.
     # 
     # @param [Symbol, Array] new_index Specify an Array if 
@@ -481,19 +1217,58 @@ module Daru
       self.dup.reindex! new_index
     end
 
-    # Return the names of all the numeric vectors. Will include vectors with nils
+    # Return the indexes of all the numeric vectors. Will include vectors with nils
     # alongwith numbers.
     def numeric_vectors
       numerics = []
 
-      each_vector do |vec|
+      each_vector_with_index do |vec, i|
+        numerics << i if(vec.type == :numeric)
+      end
+      numerics
+    end
+
+    def numeric_vector_names
+      numerics = []
+
+      each_vector do |vec, i|
         numerics << vec.name if(vec.type == :numeric)
       end
       numerics
     end
 
+    # Return a DataFrame of only the numerical Vectors. If clone: false
+    # is specified as option, only a *view* of the Vectors will be
+    # returned. Defaults to clone: true.
+    def only_numerics opts={}
+      cln = opts[:clone] == false ? false : true
+      nv = numeric_vectors
+      arry = nv.inject([]) do |arr, v|
+        arr << self[v]
+        arr
+      end
+
+      order = @vectors.is_a?(MultiIndex) ? MultiIndex.new(nv) : Index.new(nv)
+      Daru::DataFrame.new(arry, clone: cln, order: order, index: @index)
+    end
+
+    # Generate a summary of this DataFrame with ReportBuilder.
+    def summary(method = :to_text)
+      ReportBuilder.new(no_title: true).add(self).send(method)
+    end
+
+    def report_building(b) # :nodoc: #
+      b.section(:name=>@name) do |g|
+        g.text "Number of rows: #{nrows}"
+        @vectors.each do |v|
+          g.text "Element:[#{v}]"
+          g.parse_element(self[v])
+        end
+      end
+    end
+
     # Sorts a dataframe (ascending/descending)according to the given sequence of 
-    #   vectors, using the attributes provided in the blocks. Works for 2 LEVELS ONLY.
+    # vectors, using the attributes provided in the blocks.
     # 
     # @param order [Array] The order of vector names in which the DataFrame
     #   should be sorted.
@@ -583,7 +1358,7 @@ module Daru
       elsif opts[:values].is_a?(Array)
         opts[:values]
       else # nil
-        (@vectors.to_a - (index | vectors)) & numeric_vectors
+        (@vectors.to_a - (index | vectors)) & numeric_vector_names
       end
       
       raise IndexError, "No numeric vectors to aggregate" if values.empty?
@@ -634,6 +1409,195 @@ module Daru
       end
     end
 
+    # Merge vectors from two DataFrames. In case of name collision, 
+    # the vectors names are changed to x_1, x_2 ....
+    #
+    # @return {Daru::DataFrame}
+    def merge other_df
+      raise "Number of rows must be equal in this: #{nrows} and other: #{other_df.nrows}" unless nrows == other_df.nrows
+
+      new_fields = (@vectors.to_a + other_df.vectors.to_a)
+                        .recode_repeated
+                        .map(&:to_sym)
+      df_new     = DataFrame.new({}, order: new_fields)
+
+      (0...nrows).to_a.each do |i|
+        row = self.row[i].to_a + other_df.row[i].to_a
+        df_new.add_row(row)
+      end
+
+      df_new.update
+      df_new
+    end
+
+    # Join 2 DataFrames by given fields
+    # type is one of :left and :inner, default is :left
+    #
+    # Untested! Use at your own risk.
+    # 
+    # @return {Daru::DataFrame}
+    def join(other_ds,fields_1=[],fields_2=[],type=:left)
+      fields_new = other_ds.vectors.to_a - fields_2
+      fields     =     self.vectors.to_a + fields_new
+
+      other_ds_hash = {}
+      other_ds.each_row do |row|
+        key = row.to_hash.select { |k,v| fields_2.include?(k) }.values
+        value = row.to_hash.select { |k,v| fields_new.include?(k) }
+
+        if other_ds_hash[key].nil?
+          other_ds_hash[key] = [value]
+        else
+          other_ds_hash[key] << value
+        end
+      end
+
+      new_ds = DataFrame.new({}, order: fields)
+
+      self.each_row do |row|
+        key = row.to_hash.select{|k,v| fields_1.include?(k)}.values
+        new_case = row.to_hash
+
+        if other_ds_hash[key].nil?
+          if type == :left
+            fields_new.each{|field| new_case[field] = nil}
+            new_ds.add_row(Daru::Vector.new(new_case))
+          end
+        else
+          other_ds_hash[key].each do |new_values|
+            new_ds.add_row(Daru::Vector.new(new_case.merge(new_values)))
+          end
+        end
+      end
+
+      new_ds
+    end
+
+
+    # Creates a new dataset for one to many relations
+    # on a dataset, based on pattern of field names.
+    #
+    # for example, you have a survey for number of children
+    # with this structure:
+    #   id, name, child_name_1, child_age_1, child_name_2, child_age_2
+    # with
+    #   ds.one_to_many([:id], "child_%v_%n"
+    # the field of first parameters will be copied verbatim
+    # to new dataset, and fields which responds to second
+    # pattern will be added one case for each different %n.
+    # 
+    # == Usage
+    #   cases=[
+    #     ['1','george','red',10,'blue',20,nil,nil],
+    #     ['2','fred','green',15,'orange',30,'white',20],
+    #     ['3','alfred',nil,nil,nil,nil,nil,nil]
+    #   ]
+    #   ds=Daru::DataFrame.rows(cases, order: [:id, :name, :car_color1, :car_value1, :car_color2, :car_value2, :car_color3, :car_value3])
+    #   ds.one_to_many([:id],'car_%v%n').to_matrix
+    #   => Matrix[
+    #      ["red", "1", 10],
+    #      ["blue", "1", 20],
+    #      ["green", "2", 15],
+    #      ["orange", "2", 30],
+    #      ["white", "2", 20]
+    #      ]
+    #
+    def one_to_many(parent_fields, pattern)
+      re      = Regexp.new pattern.gsub("%v","(.+?)").gsub("%n","(\\d+?)")
+      ds_vars = parent_fields
+      vars    = []
+      max_n   = 0
+      h       = parent_fields.inject({}) { |a,v| 
+        a[v] = Daru::Vector.new([])
+        a 
+      }
+      # Adding _row_id
+      h[:_col_id] = Daru::Vector.new([])
+      ds_vars.push(:_col_id)
+
+      @vectors.each do |f|
+        if f =~ re
+          if !vars.include? $1
+            vars.push($1)
+            h[$1] = Daru::Vector.new([])
+          end
+          max_n = $2.to_i if max_n < $2.to_i
+        end
+      end
+      ds = DataFrame.new(h, order: ds_vars+vars)
+
+      each_row do |row|
+        row_out = {}
+        parent_fields.each do |f|
+          row_out[f]=row[f]
+        end
+
+        max_n.times do |n1|
+          n  = n1+1
+          any_data = false
+          vars.each do |v|
+            data = row[pattern.gsub("%v",v.to_s).gsub("%n",n.to_s).to_sym]
+            row_out[v] = data
+            any_data = true if !data.nil?
+          end
+
+          if any_data
+            row_out[:_col_id] = n
+            ds.add_row(row_out)
+          end
+        end
+      end
+      ds.update
+      ds
+    end
+
+    def add_vectors_by_split_recode(name_, join='-', sep=Daru::SPLIT_TOKEN)
+      split = self[name_].split_by_separator(sep)
+      i = 1
+      split.each { |k,v|
+        new_field = name_.to_s + join + i.to_s
+        v.rename name_.to_s + ":" + k.to_s
+        self[new_field.to_sym] = v
+        i += 1
+      }
+    end
+
+    # Create a sql, basen on a given Dataset
+    # 
+    # == Arguments
+    # 
+    # * table - String specifying name of the table that will created in SQL.
+    # * charset - Character set. Default is "UTF8".
+    # 
+    # == Usage
+    #
+    #  ds = Daru::DataFrame.new({
+    #   :id   => Daru::Vector.new([1,2,3,4,5]),
+    #   :name => Daru::Vector.new(%w{Alex Peter Susan Mary John})
+    #  })
+    #  ds.create_sql('names')
+    #   ==>"CREATE TABLE names (id INTEGER,\n name VARCHAR (255)) CHARACTER SET=UTF8;"
+    #
+    def create_sql(table,charset="UTF8")
+      sql    = "CREATE TABLE #{table} ("
+      fields = self.vectors.to_a.collect do |f|
+        v = self[f]
+        f.to_s + " " + v.db_type
+      end
+
+      sql + fields.join(",\n ")+") CHARACTER SET=#{charset};"
+    end
+
+    # Convert all numeric vectors to GSL::Matrix
+    def to_gsl
+      numerics_as_arrays = []
+      numeric_vectors.each do |n|
+        numerics_as_arrays << self[n].to_a
+      end
+
+      GSL::Matrix.alloc *numerics_as_arrays.transpose
+    end
+
     # Convert all vectors of type *:numeric* into a Matrix.
     def to_matrix
       numerics_as_arrays = []
@@ -644,22 +1608,27 @@ module Daru
       Matrix.columns numerics_as_arrays
     end
 
+    # Return a Nyaplot::DataFrame from the data of this DataFrame.
+    def to_nyaplotdf
+      Nyaplot::DataFrame.new(to_a[0])
+    end
+
     # Convert all vectors of type *:numeric* and not containing nils into an NMatrix.
     def to_nmatrix
       numerics_as_arrays = []
       each_vector do |vector|
         numerics_as_arrays << vector.to_a if(vector.type == :numeric and 
-          vector.nil_positions.size == 0)
+          vector.missing_positions.size == 0)
       end
 
       numerics_as_arrays.transpose.to_nm
     end
     
     # Converts the DataFrame into an array of hashes where key is vector name
-    #   and value is the corresponding element. The 0th index of the array contains 
-    #   the array of hashes while the 1th index contains the indexes of each row 
-    #   of the dataframe. Each element in the index array corresponds to its row 
-    #   in the array of hashes, which has the same index.
+    # and value is the corresponding element. The 0th index of the array contains 
+    # the array of hashes while the 1th index contains the indexes of each row 
+    # of the dataframe. Each element in the index array corresponds to its row 
+    # in the array of hashes, which has the same index.
     def to_a
       arry = [[],[]]
       self.each_row do |row|
@@ -678,9 +1647,26 @@ module Daru
       end
     end
 
+    # Converts DataFrame to a hash with keys as vector names and values as
+    # the corresponding vectors.
+    def to_hash
+      hsh = {}
+      @vectors.each_with_index do |vec_name, idx|
+        hsh[vec_name] = @data[idx]
+      end
+
+      hsh
+    end
+
     # Convert to html for IRuby.
     def to_html threshold=30
-      html  = '<table><tr><th></th>'
+      html = "<table>" + 
+        "<tr>" +
+          "<th colspan=\"#{@vectors.size+1}\">" + 
+            "Daru::DataFrame:#{self.object_id} " + " rows: #{nrows} " + " cols: #{ncols}" 
+          "</th>" +
+        "</tr>"
+      html +='<tr><th></th>'
       @vectors.each { |vector| html += '<th>' + vector.to_s + '</th>' }
       html += '</tr>'
 
@@ -697,6 +1683,15 @@ module Daru
           html += '<tr>'
           (@vectors + 1).size.times { html += '<td>...</td>' }
           html += '</tr>'
+
+          last_index = @index.to_a.last
+          last_row = self.row[last_index]
+          html += '<tr>'
+          html += "<td>" + last_index.to_s + "</td>"
+          (0..(ncols - 1)).to_a.each do |i|
+            html += '<td>' + last_row[i].to_s + '</td>' 
+          end
+          html += '</tr>'
           break
         end
       end
@@ -709,6 +1704,87 @@ module Daru
       to_html
     end
 
+    # Method for updating the metadata (i.e. missing value positions) of the
+    # after assingment/deletion etc. are complete. This is provided so that
+    # time is not wasted in creating the metadata for the vector each time
+    # assignment/deletion of elements is done. Updating data this way is called
+    # lazy loading. To set or unset lazy loading, see the .lazy_update= method.
+    def update
+      @data.each { |v| v.update } if Daru.lazy_update
+    end
+
+    def rename new_name
+      if new_name.is_a?(Numeric)
+        @name = new_name 
+        return
+      end
+      @name = new_name.to_sym
+    end
+
+    # Write this DataFrame to a CSV file.
+    #
+    # == Arguements
+    #
+    # * filename - Path of CSV file where the DataFrame is to be saved.
+    # 
+    # == Options
+    # 
+    # * convert_comma - If set to *true*, will convert any commas in any
+    # of the data to full stops ('.').
+    # All the options accepted by CSV.read() can also be passed into this 
+    # function.
+    def write_csv filename, opts={}
+      Daru::IO.dataframe_write_csv self, filename, opts
+    end
+
+    # Write this dataframe to an Excel Spreadsheet
+    # 
+    # == Arguments
+    # 
+    # * filename - The path of the file where the DataFrame should be written.
+    def write_excel filename, opts={}
+      Daru::IO.dataframe_write_excel self, filename, opts
+    end
+
+    # Insert each case of the Dataset on the selected table
+    #
+    # == Arguments
+    # 
+    # * dbh - DBI database connection object.
+    # * query - Query string.
+    # 
+    # == Usage
+    #
+    #  ds = Daru::DataFrame.new({:id=>Daru::Vector.new([1,2,3]), :name=>Daru::Vector.new(["a","b","c"])})
+    #  dbh = DBI.connect("DBI:Mysql:database:localhost", "user", "password")
+    #  ds.write_sql(dbh,"test")
+    def write_sql dbh, table
+      Daru::IO.dataframe_write_sql self, dbh, table
+    end
+
+
+    # Use marshalling to save dataframe to a file.
+    def save filename
+      Daru::IO.save self, filename
+    end
+
+    def _dump depth
+      Marshal.dump({
+        data:  @data, 
+        index: @index.to_a, 
+        order: @vectors.to_a,
+        name:  @name
+        })
+    end
+
+    def self._load data
+      h = Marshal.load data
+      Daru::DataFrame.new(h[:data], 
+        index: h[:index], 
+        order: h[:order],
+        name:  h[:name])
+    end
+
     # Change dtypes of vectors by supplying a hash of :vector_name => :new_dtype
     # 
     # == Usage
@@ -733,9 +1809,9 @@ module Daru
     # Pretty print in a nice table format for the command line (irb/pry/iruby)
     def inspect spacing=10, threshold=15
       longest = [@name.to_s.size,
-                 @vectors.map(&:to_s).map(&:size).max, 
-                 @index  .map(&:to_s).map(&:size).max,
-                 @data   .map{ |v|  v.map(&:to_s).map(&:size).max }.max].max
+                 (@vectors.map(&:to_s).map(&:size).max || 0), 
+                 (@index  .map(&:to_s).map(&:size).max || 0),
+                 (@data   .map{ |v| v.map(&:to_s).map(&:size).max}.max || 0)].max
 
       name      = @name || 'nil'
       content   = ""
@@ -901,6 +1977,8 @@ module Daru
 
     def access_vector *names
       location = names[0]
+
+      return dup(@vectors[location]) if location.is_a?(Range)
       if @vectors.is_a?(MultiIndex)
         pos = vectors_index_for names
 
@@ -996,41 +2074,68 @@ module Daru
     end
 
     def insert_or_modify_vector name, vector
-      @vectors = reassign_index_as(@vectors + name)
-      v        = nil
+      if vectors.is_a?(Index)        
+        name = name[0]
+      end
 
-      if vector.is_a?(Daru::Vector)
-        v = Daru::Vector.new [], name: set_name(name), index: @index
-        @index.each do |idx|
-          v[idx] = vector[idx]
+      @vectors = @vectors + name if !@vectors.include?(name)
+      v        = nil
+      
+      if @index.empty?
+        v = vector.is_a?(Daru::Vector) ? vector : Daru::Vector.new(vector.to_a)  
+        @index = v.index
+        @data[@vectors[name]] = v
+        set_size
+
+        @data.map! do |v|
+          if v.size == 0
+            Daru::Vector.new([nil]*@size, name: set_name(name), index: @index)
+          else
+            v
+          end
         end
       else
-        raise Exception, "Specified vector of length #{vector.size} cannot be inserted in DataFrame of size #{@size}" if
-          @size != vector.size
+        if vector.is_a?(Daru::Vector)
+          v = Daru::Vector.new [], name: set_name(name), index: @index
+          @index.each do |idx|
+            v[idx] = vector[idx]
+          end
+        else
+          raise Exception, "Specified vector of length #{vector.size} cannot be inserted in DataFrame of size #{@size}" if
+            @size != vector.size
 
-        v = Daru::Vector.new(vector, name: set_name(name), index: @index)
-      end
+          v = Daru::Vector.new(vector, name: set_name(name), index: @index)
+        end
 
-      @data[@vectors[name]] = v
+        @data[@vectors[name]] = v
+      end
     end
 
-    def insert_or_modify_row name, vector      
-      if @index.include? name
-        v = vector.dv(name, @vectors, @dtype) 
-
-        @vectors.each do |vector|
-          @data[@vectors[vector]][name] = v[vector] 
-        end
+    def insert_or_modify_row name, vector    
+      if index.is_a?(MultiIndex)
+        # TODO
       else
-        @index = reassign_index_as(@index + name)
-        v      = Daru::Vector.new(vector, name: set_name(name), index: @vectors)
+        name = name[0]
+        v =
+        if vector.is_a?(Daru::Vector)
+          vector
+        else
+          Daru::Vector.new(vector, name: set_name(name), index: @vectors)
+        end
 
-        @vectors.each do |vector|
-          @data[@vectors[vector]].concat v[vector], name
+        if @index.include? name
+          @vectors.each do |vector|
+            @data[@vectors[vector]][name] = v[vector] 
+          end
+        else
+          @index = reassign_index_as(@index + name)
+          @vectors.each do |vector|
+            @data[@vectors[vector]].concat v[vector], name
+          end
         end
-      end
 
-      set_size
+        set_size
+      end
     end
 
     def create_empty_vectors
@@ -1081,18 +2186,22 @@ module Daru
     def create_vectors_index_with vectors, source
       vectors = source.keys.sort if vectors.nil?
 
+      @vectors =
       unless vectors.is_a?(Index) or vectors.is_a?(MultiIndex)
-        @vectors = Daru::Index.new (vectors + (source.keys - vectors)).uniq.map(&:to_sym)
+        Daru::Index.new((vectors + (source.keys - vectors))
+          .uniq
+          .map { |e| e.respond_to?(:to_sym) ? e.to_sym : e }
+        )
       else
-        @vectors = vectors
+        vectors
       end
     end
 
     def all_vectors_have_equal_indexes? source
-      index = source.values[0].index
+      idx = source.values[0].index
 
       source.all? do |name, vector|
-        index == vector.index
+        idx == vector.index
       end
     end