daru 0.1.0 → 0.1.1

data/lib/daru/core/merge.rb

@@ -0,0 +1,170 @@
+module Daru
+  module Core
+    module MergeHelper
+      class << self
+        def replace_keys_if_duplicates hash, matcher
+          matched = nil
+          hash.keys.each { |d| 
+            if matcher.match(Regexp.new(d.to_s))
+              matched = d
+              break
+            end  
+          }
+
+          if matched
+            hash[matcher] = hash[matched] 
+            hash.delete matched
+          end
+        end
+
+        def resolve_duplicates df_hash1, df_hash2, on
+          hk = df_hash1.keys + df_hash2.keys - on
+          recoded = hk.recode_repeated.map(&:to_sym)
+          diff = (recoded - hk).sort
+
+          diff.each_slice(2) do |a|
+            replace_keys_if_duplicates df_hash1, a[0]
+            replace_keys_if_duplicates df_hash2, a[1]
+          end
+        end
+
+        def hashify df
+          hsh = df.to_hash
+          hsh.each { |k,v| hsh[k] = v.to_a }
+          hsh
+        end
+
+        def inner_join df1, df2, df_hash1, df_hash2, on
+          joined_hash = {}
+          ((df_hash1.keys - on) | on | (df_hash2.keys - on)).each do |k|
+            joined_hash[k] = []
+          end
+
+          (0...df1.size).each do |id1|
+            (0...df2.size).each do |id2|
+              if on.all? { |n| df_hash1[n][id1] == df_hash2[n][id2] }
+                joined_hash.each do |k,v|
+                  v << (df_hash1.has_key?(k) ? df_hash1[k][id1] : df_hash2[k][id2])
+                end
+              end
+            end
+          end
+
+          Daru::DataFrame.new(joined_hash, order: joined_hash.keys)
+        end
+
+        def full_outer_join df1, df2, df_hash1, df_hash2, on
+          left  = left_outer_join df1, df2, df_hash1, df_hash2, on, true
+          right = right_outer_join df1, df2, df_hash1, df_hash2, on, true
+
+          Daru::DataFrame.rows(
+            (left.values.transpose | right.values.transpose), order: left.keys)
+        end
+
+        def left_outer_join df1, df2, df_hash1, df_hash2, on, as_hash=false
+          joined_hash = {}
+          ((df_hash1.keys - on) | on | (df_hash2.keys - on)).each do |k|
+            joined_hash[k] = []
+          end
+
+          
+          (0...df1.size).each do |id1|
+            joined = false
+            (0...df2.size).each do |id2|
+              if on.all? { |n| df_hash1[n][id1] == df_hash2[n][id2] }
+                joined = true
+                joined_hash.each do |k,v|
+                  v << (df_hash1.has_key?(k) ? df_hash1[k][id1] : df_hash2[k][id2])
+                end
+              end
+            end
+
+            unless joined
+              df_hash1.keys.each do |k|
+                joined_hash[k] << df_hash1[k][id1]
+              end
+
+              (joined_hash.keys - df_hash1.keys).each do |k|
+                joined_hash[k] << nil
+              end
+              joined = false
+            end
+          end
+
+          return joined_hash if as_hash
+          Daru::DataFrame.new(joined_hash, order: joined_hash.keys)
+        end
+
+        def right_outer_join df1, df2, df_hash1, df_hash2, on, as_hash=false
+          joined_hash = {}
+          ((df_hash1.keys - on) | on | (df_hash2.keys - on)).each do |k|
+            joined_hash[k] = []
+          end
+
+          (0...df2.size).each do |id1|
+            joined = false
+            (0...df1.size).each do |id2|
+              if on.all? { |n| df_hash2[n][id1] == df_hash1[n][id2] }
+                joined = true
+                joined_hash.each do |k,v|
+                  v << (df_hash2.has_key?(k) ? df_hash2[k][id1] : df_hash1[k][id2])
+                end
+              end
+            end
+
+            unless joined
+              df_hash2.keys.each do |k|
+                joined_hash[k] << df_hash2[k][id1]
+              end
+
+              (joined_hash.keys - df_hash2.keys).each do |k|
+                joined_hash[k] << nil
+              end
+              joined = false
+            end
+          end
+
+          return joined_hash if as_hash
+          Daru::DataFrame.new(joined_hash, order: joined_hash.keys)
+        end
+
+        def verify_dataframes df_hash1, df_hash2, on
+          raise ArgumentError, 
+            "All fields in :on must be present in self" if !on.all? { |e| df_hash1[e] }
+          raise ArgumentError,
+            "All fields in :on must be present in other DF" if !on.all? { |e| df_hash2[e] }
+        end
+      end
+    end
+    # Private module containing methods for join, merge, concat operations on
+    # dataframes and vectors.
+    # @private
+    module Merge
+      class << self
+        def join df1, df2, opts={}
+          helper = MergeHelper
+
+          df_hash1 = helper.hashify df1
+          df_hash2 = helper.hashify df2
+          on = opts[:on]
+
+          helper.verify_dataframes df_hash1, df_hash2, on
+          helper.resolve_duplicates df_hash1, df_hash2, on
+
+          case opts[:how]
+          when :inner
+            helper.inner_join df1, df2, df_hash1, df_hash2, on
+          when :outer
+            helper.full_outer_join df1, df2, df_hash1, df_hash2, on
+          when :left
+            helper.left_outer_join df1, df2, df_hash1, df_hash2, on
+          when :right
+            helper.right_outer_join df1, df2, df_hash1, df_hash2, on
+          else
+            raise ArgumentError, "Unrecognized option in :how => #{opts[:how]}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/daru/core/query.rb

@@ -0,0 +1,95 @@
+module Daru
+  module Core
+    module Query
+      class BoolArray
+        attr_reader :barry
+
+        def initialize barry
+          @barry = barry
+        end
+
+        def & other
+          new_bool = []
+          other_barry = other.barry
+          @barry.each_with_index do |b, i|
+            new_bool << (b and other_barry[i])
+          end
+
+          BoolArray.new(new_bool)
+        end
+
+        alias :and :&
+
+        def | other
+          new_bool = []
+          other_barry = other.barry
+          @barry.each_with_index do |b, i|
+            new_bool << (b or other_barry[i])
+          end
+
+          BoolArray.new(new_bool)
+        end
+
+        alias :or :|
+
+        def !
+          BoolArray.new(@barry.map { |b| !b })
+        end
+
+        def == other
+          @barry == other.barry
+        end
+
+        def to_a
+          @barry
+        end
+
+        def inspect
+          "(#{self.class}:#{self.object_id} bool_arry=#{@barry})"
+        end
+      end
+
+      class << self
+        def apply_scalar_operator operator, data, other
+          arry = data.inject([]) do |memo,d|
+            memo << (d.send(operator, other) ? true : false)
+            memo
+          end
+
+          BoolArray.new(arry)
+        end
+
+        def apply_vector_operator operator, vector, other
+          bool_arry = []
+          vector.each_with_index do |d, i|
+            bool_arry << (d.send(operator, other[i]) ? true : false)
+          end
+
+          BoolArray.new(bool_arry)
+        end
+
+        def df_where data_frame, bool_array
+          vecs = data_frame.map do |vector|
+            vector.where(bool_array)
+          end
+
+          Daru::DataFrame.new(
+            vecs, order: data_frame.vectors, index: vecs[0].index, clone: false)
+        end
+
+        def vector_where data, index, bool_array, dtype
+          new_data = []
+          new_index = []
+          bool_array.to_a.each_with_index do |b, i|
+            if b
+              new_data << data[i]
+              new_index << index[i]
+            end
+          end
+
+          Daru::Vector.new(new_data, index: new_index, dtype: dtype)
+        end
+      end
+    end
+  end
+end

data/lib/daru/dataframe.rb

@@ -1,7 +1,6 @@
 $:.unshift File.dirname(__FILE__)
 
 require 'accessors/dataframe_by_row.rb'
-require 'accessors/dataframe_by_vector.rb'
 require 'maths/arithmetic/dataframe.rb'
 require 'maths/statistics/dataframe.rb'
 require 'plotting/dataframe.rb'
@@ -226,11 +225,11 @@ module Daru
       @data   = []
 
       temp_name = opts[:name]
-      @name   = temp_name.is_a?(Numeric) ? temp_name : (temp_name || SecureRandom.uuid).to_sym
+      @name   = temp_name || SecureRandom.uuid
 
       if source.empty?
-        @vectors = create_index vectors
-        @index   = create_index index
+        @vectors = try_create_index vectors
+        @index   = try_create_index index
         create_empty_vectors
       else
         case source
@@ -239,8 +238,8 @@ module Daru
             raise ArgumentError, "Number of vectors (#{vectors.size}) should \
               equal order size (#{source.size})" if source.size != vectors.size
 
-            @index   = create_index(index || source[0].size)
-            @vectors = create_index(vectors)
+            @index   = try_create_index(index || source[0].size)
+            @vectors = try_create_index(vectors)
 
             @vectors.each_with_index do |vec,idx|
               @data << Daru::Vector.new(source[idx], index: @index)
@@ -253,9 +252,10 @@ module Daru
             initialize(hsh, index: index, order: vectors, name: @name, clone: clone)
           else # array of hashes
             if vectors.nil?
-              @vectors = Daru::Index.new source[0].keys.map(&:to_sym)
+              @vectors = Daru::Index.new source[0].keys
             else
-              @vectors = Daru::Index.new (vectors + (source[0].keys - vectors)).uniq.map(&:to_sym)
+              @vectors = Daru::Index.new(
+                (vectors + (source[0].keys - vectors)).uniq)
             end
             @index = Daru::Index.new(index || source.size)
 
@@ -272,8 +272,9 @@ module Daru
           create_vectors_index_with vectors, source
           if all_daru_vectors_in_source? source
             if !index.nil?
-              @index = create_index index
+              @index = try_create_index index
             elsif all_vectors_have_equal_indexes?(source)
+              vectors_have_same_index = true
               @index = source.values[0].index.dup
             else
               all_indexes = []
@@ -289,17 +290,28 @@ module Daru
 
             if clone
               @vectors.each do |vector|
-                @data << Daru::Vector.new([], name: vector, index: @index)
-
-                @index.each do |idx|
-                  @data[@vectors[vector]][idx] = source[vector][idx]
+                # avoids matching indexes of vectors if all the supplied vectors
+                # have the same index.
+                if vectors_have_same_index 
+                  v = source[vector].dup
+                else
+                  v = Daru::Vector.new([], name: vector, index: @index)
+
+                  @index.each do |idx|
+                    if source[vector].index.include? idx
+                      v[idx] = source[vector][idx]
+                    else
+                      v[idx] = nil
+                    end
+                  end
                 end
+                @data << v
               end
             else
               @data.concat source.values
             end
           else
-            @index = create_index(index || source.values[0].size)
+            @index = try_create_index(index || source.values[0].size)
 
             @vectors.each do |name|
               @data << Daru::Vector.new(source[name].dup, name: set_name(name), index: @index)
@@ -313,6 +325,11 @@ module Daru
       update
     end
 
+    def vector *args
+      $stderr.puts "#vector has been deprecated in favour of #[]. Please use that."
+      self[*names]
+    end
+
     # Access row or vector. Specify name of row/vector followed by axis(:row, :vector).
     # Defaults to *:vector*. Use of this method is not recommended for accessing 
     # rows or vectors. Use df.row[:a] for accessing row with index ':a' or 
@@ -324,7 +341,6 @@ module Daru
       else
         axis = :vector
       end
-      names.map! { |e| e.respond_to?(:to_sym) ? e.to_sym : e }
 
       if axis == :vector
         access_vector *names
@@ -349,7 +365,6 @@ module Daru
 
       name = args[0..-2]
       vector = args[-1]
-      name.map! { |e| e.respond_to?(:to_sym) ? e.to_sym : e }
 
       if axis == :vector
         insert_or_modify_vector name, vector
@@ -360,15 +375,6 @@ module Daru
       end
     end
 
-    # Access a vector or set/create a vector. Refer #[] and #[]= docs for details.
-    # 
-    # == Usage
-    #   df.vector[:a] # access vector named ':a'
-    #   df.vector[:b] = [1,2,3] # set vector ':b' to [1,2,3]
-    def vector
-      Daru::Accessors::DataFrameByVector.new(self)
-    end
-
     # Access a vector by name.
     def column name
       vector[name]
@@ -398,24 +404,13 @@ module Daru
     # * +vectors_to_dup+ - An Array specifying the names of Vectors to 
     # be duplicated. Will duplicate the entire DataFrame if not specified.
     def dup vectors_to_dup=nil
-      vectors_to_dup = @vectors unless vectors_to_dup
+      vectors_to_dup = @vectors.to_a unless vectors_to_dup
 
-      new_order =
-      if vectors.is_a?(MultiIndex)
-        src = []
-        vectors_to_dup.each do |vec|
-          src << @data[@vectors[vec]].dup
-        end
-
-        Daru::MultiIndex.new(vectors_to_dup)
-      else
-        src = {}
-        vectors_to_dup.each do |vector|
-          src[vector] = @data[@vectors[vector]].dup
-        end
-
-        Daru::Index.new(vectors_to_dup)
+      src = []
+      vectors_to_dup.each do |vec|
+        src << @data[@vectors[vec]].to_a
       end
+      new_order = Daru::Index.new(vectors_to_dup)
 
       Daru::DataFrame.new src, order: new_order, index: @index.dup, name: @name, clone: true
     end
@@ -465,6 +460,14 @@ module Daru
       (vecs.nil? ? self : dup(vecs)).row[*(row_indexes - rows_with_nil)]
     end
 
+    # Iterate over each index of the DataFrame.
+    def each_index &block
+      return to_enum(:each_index) unless block_given?
+
+      @index.each(&block)
+      self
+    end
+
     # Iterate over each vector
     def each_vector(&block)
       return to_enum(:each_vector) unless block_given?
@@ -608,7 +611,7 @@ module Daru
     #
     # Recode works similarly to #map, but an important difference between 
     # the two is that recode returns a modified Daru::DataFrame instead 
-    # of an Array. For this reason, #recodeexpects that every run of the 
+    # of an Array. For this reason, #recode expects that every run of the 
     # block to return a Daru::Vector.
     #
     # Just like map and each, recode also accepts an optional _axis_ argument.
@@ -667,7 +670,8 @@ module Daru
       df = self.dup
       df.each_vector_with_index do |v, i|
         ret = yield v
-        ret.is_a?(Daru::Vector) or raise TypeError, "Every iteration must return Daru::Vector not #{ret.class}"
+        ret.is_a?(Daru::Vector) or 
+          raise TypeError, "Every iteration must return Daru::Vector not #{ret.class}"
         df[*i] = ret
       end
 
@@ -841,7 +845,7 @@ module Daru
       idx = named_index_for index
 
       if @index.include? idx
-        @index = reassign_index_as(@index.to_a - [idx])
+        @index = Daru::Index.new(@index.to_a - [idx])
         self.each_vector do |vector|
           vector.delete_at idx
         end
@@ -1015,7 +1019,7 @@ module Daru
         number_of_missing << row.missing_positions.size
       end
 
-      Daru::Vector.new number_of_missing, index: @index, name: "#{@name}_missing_rows".to_sym
+      Daru::Vector.new number_of_missing, index: @index, name: "#{@name}_missing_rows"
     end
 
     # TODO: remove next version
@@ -1087,9 +1091,18 @@ module Daru
 
     # Check if a vector is present
     def has_vector? vector
-      !!@vectors[*vector]
+      @vectors.include? vector
     end
 
+    # Works like Array#any?.
+    #
+    # @param [Symbol] axis (:vector) The axis to iterate over. Can be :vector or
+    #   :row. A Daru::Vector object is yielded in the block.
+    # @example Using any?
+    #   df = Daru::DataFrame.new({a: [1,2,3,4,5], b: ['a', 'b', 'c', 'd', 'e']})
+    #   df.any?(:row) do |row|
+    #     row[:a] < 3 and row[:b] == 'b' 
+    #   end #=> true
     def any? axis=:vector, &block
       if axis == :vector or axis == :column
         @data.any?(&block)
@@ -1103,6 +1116,15 @@ module Daru
       end
     end
 
+    # Works like Array#all?
+    #
+    # @param [Symbol] axis (:vector) The axis to iterate over. Can be :vector or
+    #   :row. A Daru::Vector object is yielded in the block.
+    # @example Using all?
+    #   df = Daru::DataFrame.new({a: [1,2,3,4,5], b: ['a', 'b', 'c', 'd', 'e']})
+    #   df.all?(:row) do |row|
+    #     row[:a] < 10 
+    #   end #=> true
     def all? axis=:vector, &block
       if axis == :vector or axis == :column
         @data.all?(&block)
@@ -1183,38 +1205,126 @@ module Daru
     #   # ["foo", "one", 3]=>[6],
     #   # ["foo", "three", 8]=>[7],
     #   # ["foo", "two", 3]=>[2, 4]}
-    def group_by vectors
-      vectors = [vectors] if vectors.is_a?(Symbol)
+    def group_by *vectors
+      vectors.flatten!
       vectors.each { |v| raise(ArgumentError, "Vector #{v} does not exist") unless
         has_vector?(v) }
         
       Daru::Core::GroupBy.new(self, vectors)
     end
 
-    def reindex_vectors! new_vectors
-      raise ArgumentError, "Number of vectors passed into function (#{new_vectors.size}) should equal that present in the DataFrame (#{@vectors.size})" if 
-        @vectors.size != new_vectors.size
+    def reindex_vectors new_vectors
+      raise ArgumentError, "Must pass the new index of type Index or its "\
+        "subclasses, not #{new_index.class}" unless new_vectors.kind_of?(Daru::Index)
+
+      cl = Daru::DataFrame.new({}, order: new_vectors, index: @index, name: @name)
+      new_vectors.each do |vec|
+        if @vectors.include?(vec)
+          cl[vec] = self[vec]
+        else
+          cl[vec] = [nil]*nrows
+        end
+      end
+
+      cl
+    end
+
+    # Concatenate another DataFrame along corresponding columns.
+    # Very premature implementation. Use with caution.
+    def concat other_df
+      vectors = []
+      @vectors.each do |v|
+        vectors << self[v].to_a.concat(other_df[v].to_a)
+      end
+
+      Daru::DataFrame.new(vectors, order: @vectors)
+    end
+
+    # Set a particular column as the new DF
+    def set_index new_index, opts={}
+      raise ArgumentError, "All elements in new index must be unique." if 
+        @size != self[new_index].uniq.size
+      
+      self.index = Daru::Index.new(self[new_index].to_a)
+      self.delete_vector(new_index) unless opts[:keep]
 
-      @vectors = Daru::Index.new new_vectors.map(&:to_sym), new_vectors.map { |e| @vectors[e] }
+      self
     end
 
-    # Change the index of the DataFrame and its underlying vectors. Destructive.
+    # Change the index of the DataFrame and preserve the labels of the previous
+    # indexing. New index can be Daru::Index or any of its subclasses.
     # 
-    # @param [Symbol, Array] new_index Specify an Array if 
-    def reindex! new_index
-      raise ArgumentError, "Index size must equal dataframe size" if new_index.is_a?(Array) and new_index.size != @size
+    # @param [Daru::Index] new_index The new Index for reindexing the DataFrame.
+    # @example Reindexing DataFrame
+    #   df = Daru::DataFrame.new({a: [1,2,3,4], b: [11,22,33,44]}, 
+    #     index: ['a','b','c','d'])
+    #   #=> 
+    #   ##<Daru::DataFrame:83278130 @name = b19277b8-c548-41da-ad9a-2ad8c060e273 @size = 4>
+    #   #                    a          b 
+    #   #         a          1         11 
+    #   #         b          2         22 
+    #   #         c          3         33 
+    #   #         d          4         44 
+    #   df.reindex Daru::Index.new(['b', 0, 'a', 'g'])
+    #   #=> 
+    #   ##<Daru::DataFrame:83177070 @name = b19277b8-c548-41da-ad9a-2ad8c060e273 @size = 4>
+    #   #                    a          b 
+    #   #         b          2         22 
+    #   #         0        nil        nil 
+    #   #         a          1         11 
+    #   #         g        nil        nil
+    def reindex new_index
+      raise ArgumentError, "Must pass the new index of type Index or its "\
+        "subclasses, not #{new_index.class}" unless new_index.kind_of?(Daru::Index)
 
-      @index = possibly_multi_index?(new_index == :seq ? @size : new_index)
-      @data.map! do |vector|
-        vector.reindex possibly_multi_index?(@index.to_a)
+      cl = Daru::DataFrame.new({}, order: @vectors, index: new_index, name: @name)
+      new_index.each do |idx|
+        if @index.include?(idx)
+          cl.row[idx] = self.row[idx]
+        else
+          cl.row[idx] = [nil]*ncols
+        end
       end
 
+      cl
+    end
+
+    # Reassign index with a new index of type Daru::Index or any of its subclasses.
+    #
+    # @param [Daru::Index] idx New index object on which the rows of the dataframe
+    #   are to be indexed.
+    # @example Reassgining index of a DataFrame
+    #   df = Daru::DataFrame.new({a: [1,2,3,4], b: [11,22,33,44]})
+    #   df.index.to_a #=> [0,1,2,3]
+    # 
+    #   df.index = Daru::Index.new(['a','b','c','d'])
+    #   df.index.to_a #=> ['a','b','c','d']
+    #   df.row['a'].to_a #=> [1,11] 
+    def index= idx
+      @data.each { |vec| vec.index = idx}
+      @index = idx
+
       self
     end
 
-    # Non-destructive version of #reindex!
-    def reindex new_index
-      self.dup.reindex! new_index
+    # Reassign vectors with a new index of type Daru::Index or any of its subclasses.
+    # 
+    # @param [Daru::Index] idx The new index object on which the vectors are to
+    #   be indexed. Must of the same size as ncols.
+    # @example Reassigning vectors of a DataFrame
+    #   df = Daru::DataFrame.new({a: [1,2,3,4], b: [:a,:b,:c,:d], c: [11,22,33,44]})
+    #   df.vectors.to_a #=> [:a, :b, :c]
+    # 
+    #   df.vectors = Daru::Index.new([:foo, :bar, :baz])
+    #   df.vectors.to_a #=> [:foo, :bar, :baz]
+    def vectors= idx
+      raise ArgumentError, "Can only reindex with Index and its subclasses" unless 
+        index.kind_of?(Daru::Index)
+      raise ArgumentError, "Specified index length #{idx.size} not equal to"\
+        "dataframe size #{ncols}" if idx.size != ncols
+
+      @vectors = idx
+      self
     end
 
     # Return the indexes of all the numeric vectors. Will include vectors with nils
@@ -1231,8 +1341,8 @@ module Daru
     def numeric_vector_names
       numerics = []
 
-      each_vector do |vec, i|
-        numerics << vec.name if(vec.type == :numeric)
+      @vectors.each do |v|
+        numerics << v if (self[v].type == :numeric)
       end
       numerics
     end
@@ -1248,7 +1358,7 @@ module Daru
         arr
       end
 
-      order = @vectors.is_a?(MultiIndex) ? MultiIndex.new(nv) : Index.new(nv)
+      order = Index.new(nv)
       Daru::DataFrame.new(arry, clone: cln, order: order, index: @index)
     end
 
@@ -1302,9 +1412,11 @@ module Daru
 
       opts[:by]        = create_logic_blocks vector_order, opts[:by]
       opts[:ascending] = sort_order_array vector_order, opts[:ascending]
-      index = @index.to_a
-      send(opts[:type], vector_order, index, opts[:by], opts[:ascending])
-      reindex! index
+      idx = @index.to_a
+      send(opts[:type], vector_order, idx, opts[:by], opts[:ascending])
+      self.index = Daru::Index.new(idx)
+
+      self
     end
 
     # Non-destructive version of #sort!
@@ -1347,7 +1459,8 @@ module Daru
     #   #     [:bar]         18         26 
     #   #     [:foo]         10         12 
     def pivot_table opts={}
-      raise ArgumentError, "Specify grouping index" if !opts[:index] or opts[:index].empty?
+      raise ArgumentError, 
+        "Specify grouping index" if !opts[:index] or opts[:index].empty?
 
       index   = opts[:index]
       vectors = opts[:vectors] || []
@@ -1389,18 +1502,20 @@ module Daru
           end
         end
 
-        df_index = Daru::MultiIndex.new(symbolize(super_hash.keys))
+        df_index = Daru::MultiIndex.from_tuples super_hash.keys
 
         vector_indexes = []
         super_hash.each_value do |sub_hash|
           vector_indexes.concat sub_hash.keys
         end
-        df_vectors = Daru::MultiIndex.new symbolize(vector_indexes.uniq)
+
+        df_vectors = Daru::MultiIndex.from_tuples vector_indexes.uniq
         pivoted_dataframe = Daru::DataFrame.new({}, index: df_index, order: df_vectors)
 
         super_hash.each do |row_index, sub_h|
           sub_h.each do |vector_index, val|
-            pivoted_dataframe[symbolize(vector_index)][symbolize(row_index)] = val
+            # pivoted_dataframe[symbolize(vector_index)][symbolize(row_index)] = val
+            pivoted_dataframe[vector_index][row_index] = val
           end
         end
         return pivoted_dataframe
@@ -1430,47 +1545,33 @@ module Daru
       df_new
     end
 
-    # Join 2 DataFrames by given fields
-    # type is one of :left and :inner, default is :left
-    #
-    # Untested! Use at your own risk.
-    # 
-    # @return {Daru::DataFrame}
-    def join(other_ds,fields_1=[],fields_2=[],type=:left)
-      fields_new = other_ds.vectors.to_a - fields_2
-      fields     =     self.vectors.to_a + fields_new
-
-      other_ds_hash = {}
-      other_ds.each_row do |row|
-        key = row.to_hash.select { |k,v| fields_2.include?(k) }.values
-        value = row.to_hash.select { |k,v| fields_new.include?(k) }
-
-        if other_ds_hash[key].nil?
-          other_ds_hash[key] = [value]
-        else
-          other_ds_hash[key] << value
-        end
-      end
-
-      new_ds = DataFrame.new({}, order: fields)
-
-      self.each_row do |row|
-        key = row.to_hash.select{|k,v| fields_1.include?(k)}.values
-        new_case = row.to_hash
-
-        if other_ds_hash[key].nil?
-          if type == :left
-            fields_new.each{|field| new_case[field] = nil}
-            new_ds.add_row(Daru::Vector.new(new_case))
-          end
-        else
-          other_ds_hash[key].each do |new_values|
-            new_ds.add_row(Daru::Vector.new(new_case.merge(new_values)))
-          end
-        end
-      end
-
-      new_ds
+    # Join 2 DataFrames with SQL style joins. Currently supports inner, left 
+    # outer, right outer and full outer joins.
+    # 
+    # @param [Daru::DataFrame] other_df Another DataFrame on which the join is
+    #   to be performed.
+    # @param [Hash] opts Options Hash
+    # @option :how [Symbol] Can be one of :inner, :left, :right or :outer.
+    # @option :on [Array] The columns on which the join is to be performed.
+    #   Column names specified here must be common to both DataFrames.
+    # @return [Daru::DataFrame]
+    # @example Inner Join
+    #   left = Daru::DataFrame.new({
+    #     :id   => [1,2,3,4],
+    #     :name => ['Pirate', 'Monkey', 'Ninja', 'Spaghetti']
+    #   })
+    #   right = Daru::DataFrame.new({
+    #     :id => [1,2,3,4],
+    #     :name => ['Rutabaga', 'Pirate', 'Darth Vader', 'Ninja']
+    #   })
+    #   left.join(right, how: :inner, on: [:name])
+    #   #=> 
+    #   ##<Daru::DataFrame:82416700 @name = 74c0811b-76c6-4c42-ac93-e6458e82afb0 @size = 2>
+    #   #                 id_1       name       id_2 
+    #   #         0          1     Pirate          2 
+    #   #         1          3      Ninja          4 
+    def join(other_df,opts={})
+      Daru::Core::Merge.join(self, other_df, opts)
     end
 
 
@@ -1486,7 +1587,7 @@ module Daru
     # to new dataset, and fields which responds to second
     # pattern will be added one case for each different %n.
     # 
-    # == Usage
+    # @example
     #   cases=[
     #     ['1','george','red',10,'blue',20,nil,nil],
     #     ['2','fred','green',15,'orange',30,'white',20],
@@ -1494,17 +1595,16 @@ module Daru
     #   ]
     #   ds=Daru::DataFrame.rows(cases, order: [:id, :name, :car_color1, :car_value1, :car_color2, :car_value2, :car_color3, :car_value3])
     #   ds.one_to_many([:id],'car_%v%n').to_matrix
-    #   => Matrix[
-    #      ["red", "1", 10],
-    #      ["blue", "1", 20],
-    #      ["green", "2", 15],
-    #      ["orange", "2", 30],
-    #      ["white", "2", 20]
-    #      ]
-    #
+    #   #=> Matrix[
+    #   #   ["red", "1", 10],
+    #   #   ["blue", "1", 20],
+    #   #   ["green", "2", 15],
+    #   #   ["orange", "2", 30],
+    #   #   ["white", "2", 20]
+    #   #   ]
     def one_to_many(parent_fields, pattern)
       re      = Regexp.new pattern.gsub("%v","(.+?)").gsub("%n","(\\d+?)")
-      ds_vars = parent_fields
+      ds_vars = parent_fields.dup
       vars    = []
       max_n   = 0
       h       = parent_fields.inject({}) { |a,v| 
@@ -1512,8 +1612,8 @@ module Daru
         a 
       }
       # Adding _row_id
-      h[:_col_id] = Daru::Vector.new([])
-      ds_vars.push(:_col_id)
+      h['_col_id'] = Daru::Vector.new([])
+      ds_vars.push('_col_id')
 
       @vectors.each do |f|
         if f =~ re
@@ -1529,20 +1629,20 @@ module Daru
       each_row do |row|
         row_out = {}
         parent_fields.each do |f|
-          row_out[f]=row[f]
+          row_out[f] = row[f]
         end
 
         max_n.times do |n1|
           n  = n1+1
           any_data = false
           vars.each do |v|
-            data = row[pattern.gsub("%v",v.to_s).gsub("%n",n.to_s).to_sym]
+            data = row[pattern.gsub("%v",v.to_s).gsub("%n",n.to_s)]
             row_out[v] = data
             any_data = true if !data.nil?
           end
 
           if any_data
-            row_out[:_col_id] = n
+            row_out['_col_id'] = n
             ds.add_row(row_out)
           end
         end
@@ -1569,14 +1669,14 @@ module Daru
     # * table - String specifying name of the table that will created in SQL.
     # * charset - Character set. Default is "UTF8".
     # 
-    # == Usage
+    # @example
     #
     #  ds = Daru::DataFrame.new({
     #   :id   => Daru::Vector.new([1,2,3,4,5]),
     #   :name => Daru::Vector.new(%w{Alex Peter Susan Mary John})
     #  })
     #  ds.create_sql('names')
-    #   ==>"CREATE TABLE names (id INTEGER,\n name VARCHAR (255)) CHARACTER SET=UTF8;"
+    #   #=>"CREATE TABLE names (id INTEGER,\n name VARCHAR (255)) CHARACTER SET=UTF8;"
     #
     def create_sql(table,charset="UTF8")
       sql    = "CREATE TABLE #{table} ("
@@ -1639,6 +1739,8 @@ module Daru
       arry
     end
 
+    # Convert to json. If no_index is false then the index will NOT be included
+    # in the JSON thus created.
     def to_json no_index=true
       if no_index
         self.to_a[0].to_json
@@ -1681,7 +1783,7 @@ module Daru
         html += '</tr>'
         if num > threshold
           html += '<tr>'
-          (@vectors + 1).size.times { html += '<td>...</td>' }
+          (@vectors.size + 1).times { html += '<td>...</td>' }
           html += '</tr>'
 
           last_index = @index.to_a.last
@@ -1713,12 +1815,9 @@ module Daru
       @data.each { |v| v.update } if Daru.lazy_update
     end
 
+    # Rename the DataFrame.
     def rename new_name
-      if new_name.is_a?(Numeric)
-        @name = new_name 
-        return
-      end
-      @name = new_name.to_sym
+      @name = new_name
     end
 
     # Write this DataFrame to a CSV file.
@@ -1792,7 +1891,7 @@ module Daru
     #   df.recast a: :nmatrix, c: :nmatrix
     def recast opts={}
       opts.each do |vector_name, dtype|
-        vector[vector_name].cast(dtype: dtype)
+        self[vector_name].cast(dtype: dtype)
       end
     end
 
@@ -1840,16 +1939,24 @@ module Daru
       content
     end
 
+    # Query a DataFrame by passing a Daru::Core::Query::BoolArray object.
+    def where bool_array
+      Daru::Core::Query.df_where self, bool_array
+    end
+
     def == other
-      @index == other.index and @size == other.size and @vectors == other.vectors and 
-      @vectors.all? { |vector| self[vector, :vector] == other[vector, :vector] }
+      self.class == other.class   and 
+      @size      == other.size    and 
+      @index     == other.index   and
+      @vectors   == other.vectors and 
+      @vectors.to_a.all? { |v| self[v] == other[v] }
     end
 
     def method_missing(name, *args, &block)
       if md = name.match(/(.+)\=/)
         insert_or_modify_vector name[/(.+)\=/].delete("=").to_sym, args[0]
       elsif self.has_vector? name
-        self[name, :vector]
+        self[name]
       else
         super(name, *args, &block)
       end
@@ -1859,7 +1966,7 @@ module Daru
 
     def possibly_multi_index? index
       if @index.is_a?(MultiIndex)
-        Daru::MultiIndex.new(index)
+        Daru::MultiIndex.from_tuples(index)
       else
         Daru::Index.new(index)
       end
@@ -1892,7 +1999,7 @@ module Daru
 
     def partition vector_order, index, by, ascending, left_lower, right_upper
       mindex = (left_lower + right_upper) / 2
-      mvalues = vector_order.inject([]) { |a, vector_name| a << vector[vector_name][mindex]; a }
+      mvalues = vector_order.inject([]) { |a, vector_name| a << self[vector_name][mindex]; a }
       i = left_lower
       j = right_upper
       descending = ascending.map { |a| !a }
@@ -1929,7 +2036,7 @@ module Daru
     def keep? current_index, mvalues, vector_order, sort_order, by, vector_order_index
       vector_name = vector_order[vector_order_index]
       if vector_name
-        vec = vector[vector_name]
+        vec = self[vector_name]
         eval = by[vector_name].call(vec[current_index], mvalues[vector_order_index])
 
         if sort_order[vector_order_index] # sort in ascending order
@@ -1980,28 +2087,41 @@ module Daru
 
       return dup(@vectors[location]) if location.is_a?(Range)
       if @vectors.is_a?(MultiIndex)
-        pos = vectors_index_for names
+        pos = @vectors[names]
 
         if pos.is_a?(Integer)
           return @data[pos]
         else # MultiIndex
           new_vectors = pos.map do |tuple|
-            @data[vectors_index_for(names + tuple)]
+            @data[@vectors[tuple]]
           end
-          Daru::DataFrame.new(new_vectors, index: @index, order: Daru::MultiIndex.new(pos.to_a))
+
+          if !location.is_a?(Range) and names.size < @vectors.width
+            pos = pos.drop_left_level names.size
+          end
+
+          Daru::DataFrame.new(
+            new_vectors, index: @index, order: pos)
         end
       else
         unless names[1]
-          pos = vectors_index_for location
-          return @data[pos]
+          pos = @vectors[location]
+
+          if pos.is_a?(Numeric)
+            return @data[pos]
+          else
+            names = pos
+          end
         end
 
-        new_vcs = {}
+        new_vcs = []
         names.each do |name|
-          name = name.to_sym unless name.is_a?(Integer)
-          new_vcs[name] = @data[@vectors[name]]
+          new_vcs << @data[@vectors[name]].to_a
         end
-        Daru::DataFrame.new new_vcs, order: new_vcs.keys, index: @index, name: @name
+
+        order = names.is_a?(Array) ? Daru::Index.new(names) : names
+        Daru::DataFrame.new(new_vcs, order: order, 
+          index: @index, name: @name)
       end
     end
 
@@ -2009,82 +2129,55 @@ module Daru
       location = names[0]
 
       if @index.is_a?(MultiIndex)
-        pos = row_index_for names
+        pos = @index[names]
         if pos.is_a?(Integer)
           return Daru::Vector.new(populate_row_for(pos), index: @vectors, name: pos)
         else
-          new_rows =
-          if location.is_a?(Range)
-            pos.map { |tuple| populate_row_for(tuple) }
-          else
-            pos.map { |tuple| populate_row_for(names + tuple) }
-          end
+          new_rows = pos.map { |tuple| populate_row_for(tuple) }
           
-          Daru::DataFrame.rows(new_rows, order: @vectors, name: @name, 
-            index: Daru::MultiIndex.new(pos.to_a))
+          if !location.is_a?(Range) and names.size < @index.width
+            pos = pos.drop_left_level names.size
+          end
+
+          Daru::DataFrame.rows(
+            new_rows, order: @vectors, name: @name, index: pos)
         end
       else
         if names[1].nil? 
-          if location.is_a?(Range)
-            index_arry = @index.to_a
-
-            range = 
-            if location.first.is_a?(Numeric)
-              location
-            else
-              first_index = index_arry.index location.first
-              last_index  = index_arry.index location.last
-
-              first_index..last_index
+          names = @index[location]
+          if names.is_a?(Numeric)
+            row = []
+            @data.each do |vector|
+              row << vector[location]
             end
 
-            names = index_arry[range]
-          else
-            row  = []
-            name = named_index_for names[0]
-            @vectors.each do |vector|
-              row << @data[@vectors[vector]][name]
-            end
-
-            return Daru::Vector.new(row, index: @vectors, name: set_name(name))
+            return Daru::Vector.new(row, index: @vectors, name: set_name(location))
           end
         end
         # Access multiple rows
         rows = []
         names.each do |name|
-          rows << self.row[name]
+          rows << self.row[name].to_a
         end
         
-        Daru::DataFrame.rows rows, name: @name        
-      end
-    end
-
-    def row_index_for location
-      if @index.include?(location) or location[0].is_a?(Range)
-        @index[location]
-      elsif location[0].is_a?(Integer)
-        location[0]
+        Daru::DataFrame.rows rows, index: names ,name: @name, order: @vectors        
       end
     end
 
     def populate_row_for pos
-      @vectors.map do |vector|
-        @data[@vectors[vector]][pos]
+      @data.map do |vector|
+        vector[pos]
       end
     end
 
     def insert_or_modify_vector name, vector
-      if vectors.is_a?(Index)        
-        name = name[0]
-      end
+      name = name[0] unless @vectors.is_a?(MultiIndex)   
+      v = nil
 
-      @vectors = @vectors + name if !@vectors.include?(name)
-      v        = nil
-      
       if @index.empty?
         v = vector.is_a?(Daru::Vector) ? vector : Daru::Vector.new(vector.to_a)  
         @index = v.index
-        @data[@vectors[name]] = v
+        assign_or_add_vector name, v
         set_size
 
         @data.map! do |v|
@@ -2096,21 +2189,47 @@ module Daru
         end
       else
         if vector.is_a?(Daru::Vector)
-          v = Daru::Vector.new [], name: set_name(name), index: @index
-          @index.each do |idx|
-            v[idx] = vector[idx]
+          if vector.index == @index # so that index-by-index assignment is avoided when possible.
+            v = vector.dup
+          else
+            v = Daru::Vector.new [], name: set_name(name), index: @index
+            @index.each do |idx|
+              if vector.index.include? idx
+                v[idx] = vector[idx]
+              else
+                v[idx] = nil
+              end
+            end
           end
         else
-          raise Exception, "Specified vector of length #{vector.size} cannot be inserted in DataFrame of size #{@size}" if
+          raise SizeError,
+            "Specified vector of length #{vector.size} cannot be inserted in DataFrame of size #{@size}" if
             @size != vector.size
 
           v = Daru::Vector.new(vector, name: set_name(name), index: @index)
         end
 
-        @data[@vectors[name]] = v
+        assign_or_add_vector name, v
       end
     end
 
+    def assign_or_add_vector name, v
+      #FIXME: fix this jugaad. need to make changes in Indexing itself.
+      pos = @vectors[name]
+
+      if !pos.kind_of?(Daru::Index) and pos == name and 
+        (@vectors.include?(name) or (pos.is_a?(Integer) and pos < @data.size))
+        @data[pos] = v
+      elsif pos.kind_of?(Daru::Index)
+        pos.each do |p|
+          @data[@vectors[p]] = v
+        end
+      else
+        @vectors = @vectors | [name] if !@vectors.include?(name)
+        @data[@vectors[name]] = v
+      end      
+    end
+
     def insert_or_modify_row name, vector    
       if index.is_a?(MultiIndex)
         # TODO
@@ -2124,13 +2243,13 @@ module Daru
         end
 
         if @index.include? name
-          @vectors.each do |vector|
-            @data[@vectors[vector]][name] = v[vector] 
+          self.each_vector_with_index do |vector,i|
+            vector[name] = v.index.include?(i) ? v[i] : nil
           end
         else
-          @index = reassign_index_as(@index + name)
-          @vectors.each do |vector|
-            @data[@vectors[vector]].concat v[vector], name
+          @index = @index | [name]
+          self.each_vector_with_index do |vector,i|
+            vector.concat((v.index.include?(i) ? v[i] : nil), name)
           end
         end
 
@@ -2184,14 +2303,11 @@ module Daru
     end
 
     def create_vectors_index_with vectors, source
-      vectors = source.keys.sort if vectors.nil?
+      vectors = source.keys.sort_by { |a| a.to_s } if vectors.nil?
 
       @vectors =
       unless vectors.is_a?(Index) or vectors.is_a?(MultiIndex)
-        Daru::Index.new((vectors + (source.keys - vectors))
-          .uniq
-          .map { |e| e.respond_to?(:to_sym) ? e.to_sym : e }
-        )
+        Daru::Index.new((vectors + (source.keys - vectors)).uniq)
       else
         vectors
       end
@@ -2200,21 +2316,17 @@ module Daru
     def all_vectors_have_equal_indexes? source
       idx = source.values[0].index
 
-      source.all? do |name, vector|
+      source.values.all? do |vector|
         idx == vector.index
       end
     end
 
-    def reassign_index_as new_index
-      Daru::Index.new new_index
-    end
-
-    def create_index index
-      index.is_a?(MultiIndex) ? index : Daru::Index.new(index)
+    def try_create_index index
+      index.kind_of?(Index) ? index : Daru::Index.new(index)
     end
 
     def set_name potential_name
-      potential_name.is_a?(Array) ? potential_name.join.to_sym : potential_name
+      potential_name.is_a?(Array) ? potential_name.join : potential_name
     end
 
     def symbolize arry