mongo 1.6.4 → 1.7.0.rc0

data/lib/mongo/util/sharding_pool_manager.rb

@@ -0,0 +1,143 @@
+
+module Mongo
+  module ShardingNode
+    def set_config
+      begin
+        @config = @connection['admin'].command({:ismaster => 1}, :socket => @socket)
+
+        # warning: instance variable @logger not initialized
+        #if @config['msg'] && @logger
+        #  @connection.log(:warn, "#{config['msg']}")
+        #end
+
+      rescue ConnectionFailure, OperationFailure, OperationTimeout, SocketError, SystemCallError, IOError => ex
+        @connection.log(:warn, "Attempted connection to node #{host_string} raised " +
+            "#{ex.class}: #{ex.message}")
+
+        # Socket may already be nil from issuing command
+        if @socket && !@socket.closed?
+          @socket.close
+        end
+
+        return nil
+      end
+
+      @config
+    end
+
+    # Return a list of sharded cluster nodes from the config - currently just the current node.
+    def node_list
+      connect unless connected?
+      set_config unless @config
+
+      return [] unless config
+
+      ["#{@host}:#{@port}"]
+    end
+
+  end
+
+  class ShardingPoolManager < PoolManager
+
+    attr_reader :connection, :primary, :primary_pool, :hosts, :nodes,
+      :max_bson_size, :members
+
+    # Create a new set of connection pools.
+    #
+    # The pool manager will by default use the original seed list passed
+    # to the connection objects, accessible via connection.seeds. In addition,
+    # the user may pass an additional list of seeds nodes discovered in real
+    # time. The union of these lists will be used when attempting to connect,
+    # with the newly-discovered nodes being used first.
+    def initialize(connection, seeds=[])
+      super
+    end
+
+    def inspect
+      "<Mongo::ShardingPoolManager:0x#{self.object_id.to_s(16)} @seeds=#{@seeds}>"
+    end
+
+    # "Best" should be the member with the fastest ping time
+    # but connect/connect_to_members reinitializes @members
+    def best(members)
+      Array(members.first)
+    end
+
+    def connect
+      close if @previously_connected
+
+      initialize_data
+      members = connect_to_members
+      initialize_pools(best(members))
+
+      @members = members
+      @previously_connected = true
+    end
+
+    # We want to refresh to the member with the fastest ping time
+    # but also want to minimize refreshes
+    # We're healthy if the primary is pingable. If this isn't the case,
+    # or the members have changed, set @refresh_required to true, and return.
+    # The config.mongos find can't be part of the connect call chain due to infinite recursion
+    def check_connection_health
+      begin
+        seeds = @connection['config']['mongos'].find.to_a.map{|doc| doc['_id']}
+        if @seeds != seeds
+          @seeds = seeds
+          @refresh_required = true
+        end
+      rescue Mongo::OperationFailure
+        @refresh_required = true
+      end
+    end
+
+    private
+
+    # Connect to each member of the sharded cluster
+    # as reported by the given seed node, and return
+    # as a list of Mongo::Node objects.
+    def connect_to_members
+      members = []
+
+      seed = get_valid_seed_node
+
+      seed.node_list.each do |host|
+        node = Mongo::Node.new(self.connection, host)
+        node.extend ShardingNode
+        if node.connect && node.set_config
+          members << node
+        end
+      end
+      seed.close
+
+      if members.empty?
+        raise ConnectionFailure, "Failed to connect to any given member."
+      end
+
+      members
+    end
+
+    # Iterate through the list of provided seed
+    # nodes until we've gotten a response from the
+    # sharded cluster we're trying to connect to.
+    #
+    # If we don't get a response, raise an exception.
+    def get_valid_seed_node
+      @seeds.each do |seed|
+        node = Mongo::Node.new(self.connection, seed)
+        node.extend ShardingNode
+        if !node.connect
+          next
+        elsif node.set_config && node.healthy?
+          return node
+        else
+          node.close
+        end
+      end
+
+      raise ConnectionFailure, "Cannot connect to a sharded cluster using seeds " +
+          "#{@seeds.map {|s| "#{s[0]}:#{s[1]}" }.join(', ')}"
+    end
+
+  end
+end

data/lib/mongo/util/support.rb

@@ -23,6 +23,13 @@ module Mongo
     include Mongo::Conversions
     extend self
 
+    READ_PREFERENCES = [:primary, :primary_preferred, :secondary, :secondary_preferred, :nearest]
+
+    # Commands that may be sent to replica-set secondaries, depending on
+    # read preference and tags. All other commands are always run on the primary.
+    SECONDARY_OK_COMMANDS = ['group', 'aggregate', 'collstats', 'dbstats', 'count', 'distinct',
+      'geonear', 'geosearch', 'geowalk']
+
     # Generate an MD5 for authentication.
     #
     # @param [String] username
@@ -58,12 +65,25 @@ module Mongo
       db_name
     end
 
+    def secondary_ok?(selector)
+      command = selector.keys.first.to_s.downcase
+
+      if command == 'mapreduce'
+        map_reduce = selector[command]
+        if map_reduce && map_reduce.is_a?(Hash) && map_reduce.has_key?('out')
+          map_reduce['out'] == 'inline' ? false : true
+        end
+      else
+        SECONDARY_OK_COMMANDS.member?(command)
+      end
+    end
+
     def validate_read_preference(value)
-      if [:primary, :secondary, :secondary_only, nil].include?(value)
+      if READ_PREFERENCES.include?(value)
         return true
       else
         raise MongoArgumentError, "#{value} is not a valid read preference. " +
-          "Please specify either :primary or :secondary or :secondary_only."
+          "Please specify one of the following read preferences as a symbol: #{READ_PREFERENCES}"
       end
     end
 

data/lib/mongo/util/tcp_socket.rb

@@ -43,23 +43,18 @@ module Mongo
       # Block on data to read for @op_timeout seconds
       begin
         ready = IO.select([@socket], nil, [@socket], @op_timeout)
+        unless ready
+          raise OperationTimeout
+        end
       rescue IOError
-        raise OperationFailure
+        raise ConnectionFailure
       end
-      if ready
-        begin
-          @socket.sysread(maxlen, buffer)
-        rescue SystemCallError => ex
-          # Needed because sometimes JRUBY doesn't throw Errno::ECONNRESET as it should
-          # http://jira.codehaus.org/browse/JRUBY-6180
-          raise ConnectionFailure, ex
-        rescue Errno::ENOTCONN, Errno::EBADF, Errno::ECONNRESET, Errno::EPIPE, Errno::ETIMEDOUT, EOFError => ex
-          raise ConnectionFailure, ex
-        rescue Errno::EINTR, Errno::EIO, IOError => ex
-          raise OperationFailure, ex
-        end
-      else
-        raise OperationTimeout
+
+      # Read data from socket
+      begin
+        @socket.sysread(maxlen, buffer)
+      rescue SystemCallError, IOError => ex
+        raise ConnectionFailure, ex
       end
     end
 

data/lib/mongo/util/uri_parser.rb

@@ -76,7 +76,7 @@ module Mongo
                   :wtimeoutms       => lambda {|arg| arg.to_i }
                  }
 
-    attr_reader :nodes, :auths, :connect, :replicaset, :slaveok, :safe, :w, :wtimeout, :fsync, :journal, :connecttimeoutms, :sockettimeoutms, :wtimeoutms
+    attr_reader :auths, :connect, :replicaset, :slaveok, :safe, :w, :wtimeout, :fsync, :journal, :connecttimeoutms, :sockettimeoutms, :wtimeoutms
 
     # Parse a MongoDB URI. This method is used by Connection.from_uri.
     # Returns an array of nodes and an array of db authorizations, if applicable.
@@ -87,7 +87,7 @@ module Mongo
     # @param [Hash,nil] extra_opts Extra options. Will override anything already specified in the URI.
     #
     # @core connections
-    def initialize(uri, extra_opts={})
+    def initialize(uri)
       if uri.start_with?('mongodb://')
         uri = uri[10..-1]
       else
@@ -96,7 +96,7 @@ module Mongo
 
       hosts, opts = uri.split('?')
       parse_hosts(hosts)
-      parse_options(opts, extra_opts)
+      parse_options(opts)
       validate_connect
     end
 
@@ -105,11 +105,12 @@ module Mongo
     # @note Don't confuse this with attribute getter method #connect.
     #
     # @return [Connection,ReplSetConnection]
-    def connection
+    def connection(extra_opts)
+      opts = connection_options.merge! extra_opts
       if replicaset?
-        ReplSetConnection.new(*(nodes+[connection_options]))
+        ReplSetConnection.new(nodes, opts)
       else
-        Connection.new(host, port, connection_options)
+        Connection.new(host, port, opts)
       end
     end
 
@@ -207,6 +208,14 @@ module Mongo
       opts
     end
 
+    def nodes
+      if @nodes.length == 1
+        @nodes
+      else
+        @nodes.collect {|node| "#{node[0]}:#{node[1]}"}
+      end
+    end
+
     private
 
     def parse_hosts(uri_without_proto)
@@ -254,13 +263,13 @@ module Mongo
 
     # This method uses the lambdas defined in OPT_VALID and OPT_CONV to validate
     # and convert the given options.
-    def parse_options(string_opts, extra_opts={})
+    def parse_options(string_opts)
       # initialize instance variables for available options
       OPT_VALID.keys.each { |k| instance_variable_set("@#{k}", nil) }
 
       string_opts ||= ''
 
-      return if string_opts.empty? && extra_opts.empty?
+      return if string_opts.empty?
 
       if string_opts.include?(';') and string_opts.include?('&')
         raise MongoArgumentError, "must not mix URL separators ; and &"
@@ -272,8 +281,6 @@ module Mongo
         memo
       end
 
-      opts.merge! extra_opts
-
       opts.each do |key, value|
         if !OPT_ATTRS.include?(key)
           raise MongoArgumentError, "Invalid Mongo URI option #{key}"

data/lib/mongo/version.rb

@@ -1,3 +1,3 @@
 module Mongo
-  VERSION = "1.6.4"
+  VERSION = "1.7.0.rc0"
 end

data/test/collection_test.rb

@@ -247,7 +247,7 @@ class TestCollection < Test::Unit::TestCase
   def test_maximum_insert_size
     docs = []
     16.times do
-      docs << {'foo' => 'a' * 1_000_000}
+      docs << {'foo' => 'a' * 1024 * 1024}
     end
 
     assert_raise InvalidOperation do
@@ -290,6 +290,16 @@ class TestCollection < Test::Unit::TestCase
     assert_equal 1, @@test.find_one(:_id => id2)["x"]
   end
 
+  def test_update_check_keys
+    @@test.save("x" => 1)
+    @@test.update({"x" => 1}, {"$set" => {"a.b" => 2}})
+    assert_equal 2, @@test.find_one("x" => 1)["a"]["b"]
+
+    assert_raise_error BSON::InvalidKeyName, "key a.b must not contain '.'" do
+      @@test.update({"x" => 1}, {"a.b" => 3})
+    end
+  end
+
   if @@version >= "1.1.3"
     def test_multi_update
       @@test.save("num" => 10)
@@ -525,6 +535,128 @@ class TestCollection < Test::Unit::TestCase
     end
     assert c.closed?
   end
+  
+  def setup_aggregate_data
+    # save some data
+    @@test.save( {
+        "_id" => 1,
+        "title" => "this is my title", 
+        "author" => "bob", 
+        "posted" => Time.utc(1500), 
+        "pageViews" => 5 , 
+        "tags" => [ "fun" , "good" , "fun" ],
+        "comments" => [ 
+                        { "author" => "joe", "text" => "this is cool" }, 
+                        { "author" => "sam", "text" => "this is bad" } 
+            ],
+        "other" => { "foo" => 5 }
+        } )
+    
+    @@test.save( {
+         "_id" => 2,
+         "title" => "this is your title", 
+         "author" => "dave", 
+         "posted" => Time.utc(1600), 
+         "pageViews" => 7, 
+         "tags" => [ "fun" , "nasty" ],
+         "comments" => [ 
+                         { "author" => "barbara" , "text" => "this is interesting" }, 
+                         { "author" => "jenny", "text" => "i like to play pinball", "votes" => 10 } 
+         ],
+          "other" => { "bar" => 14 }
+        })
+        
+    @@test.save( {
+            "_id" => 3,
+            "title" => "this is some other title", 
+            "author" => "jane", 
+            "posted" => Time.utc(1700), 
+            "pageViews" => 6 , 
+            "tags" => [ "nasty", "filthy" ],
+            "comments" => [ 
+                { "author" => "will" , "text" => "i don't like the color" } , 
+                { "author" => "jenny" , "text" => "can i get that in green?" } 
+            ],
+            "other" => { "bar" => 14 }
+        })
+    
+  end
+  
+  if @@version > '2.1.1'
+    def test_reponds_to_aggregate
+      assert_respond_to @@test, :aggregate
+    end
+
+    def test_aggregate_requires_arguments
+      assert_raise MongoArgumentError do
+        @@test.aggregate()
+      end
+    end
+
+    def test_aggregate_requires_valid_arguments
+      assert_raise MongoArgumentError do
+        @@test.aggregate({})
+      end
+    end
+
+    def test_aggregate_pipeline_operator_format
+      assert_raise Mongo::OperationFailure do
+        @@test.aggregate([{"$project" => "_id"}])
+      end
+    end
+
+    def test_aggregate_pipeline_operators_using_strings
+      setup_aggregate_data
+      desired_results = [ {"_id"=>1, "pageViews"=>5, "tags"=>["fun", "good", "fun"]}, 
+                          {"_id"=>2, "pageViews"=>7, "tags"=>["fun", "nasty"]}, 
+                          {"_id"=>3, "pageViews"=>6, "tags"=>["nasty", "filthy"]} ]
+      results = @@test.aggregate([{"$project" => {"tags" => 1, "pageViews" => 1}}])
+      assert_equal desired_results, results
+    end
+
+    def test_aggregate_pipeline_operators_using_symbols
+      setup_aggregate_data
+      desired_results = [ {"_id"=>1, "pageViews"=>5, "tags"=>["fun", "good", "fun"]}, 
+                          {"_id"=>2, "pageViews"=>7, "tags"=>["fun", "nasty"]}, 
+                          {"_id"=>3, "pageViews"=>6, "tags"=>["nasty", "filthy"]} ]
+      results = @@test.aggregate([{"$project" => {:tags => 1, :pageViews => 1}}])
+      assert_equal desired_results, results
+    end
+
+    def test_aggregate_pipeline_multiple_operators
+      setup_aggregate_data
+      results = @@test.aggregate([{"$project" => {"tags" => 1, "pageViews" => 1}}, {"$match" => {"pageViews" => 7}}])
+      assert_equal 1, results.length
+    end
+
+    def test_aggregate_pipeline_unwind                    
+      setup_aggregate_data
+      desired_results = [ {"_id"=>1, "title"=>"this is my title", "author"=>"bob", "posted"=>Time.utc(1500), 
+                          "pageViews"=>5, "tags"=>"fun", "comments"=>[{"author"=>"joe", "text"=>"this is cool"}, 
+                            {"author"=>"sam", "text"=>"this is bad"}], "other"=>{"foo"=>5 } },
+                          {"_id"=>1, "title"=>"this is my title", "author"=>"bob", "posted"=>Time.utc(1500), 
+                            "pageViews"=>5, "tags"=>"good", "comments"=>[{"author"=>"joe", "text"=>"this is cool"}, 
+                            {"author"=>"sam", "text"=>"this is bad"}], "other"=>{"foo"=>5 } },
+                          {"_id"=>1, "title"=>"this is my title", "author"=>"bob", "posted"=>Time.utc(1500), 
+                            "pageViews"=>5, "tags"=>"fun", "comments"=>[{"author"=>"joe", "text"=>"this is cool"}, 
+                              {"author"=>"sam", "text"=>"this is bad"}], "other"=>{"foo"=>5 } },
+                          {"_id"=>2, "title"=>"this is your title", "author"=>"dave", "posted"=>Time.utc(1600), 
+                            "pageViews"=>7, "tags"=>"fun", "comments"=>[{"author"=>"barbara", "text"=>"this is interesting"}, 
+                              {"author"=>"jenny", "text"=>"i like to play pinball", "votes"=>10 }], "other"=>{"bar"=>14 } },
+                          {"_id"=>2, "title"=>"this is your title", "author"=>"dave", "posted"=>Time.utc(1600), 
+                            "pageViews"=>7, "tags"=>"nasty", "comments"=>[{"author"=>"barbara", "text"=>"this is interesting"}, 
+                              {"author"=>"jenny", "text"=>"i like to play pinball", "votes"=>10 }], "other"=>{"bar"=>14 } },
+                          {"_id"=>3, "title"=>"this is some other title", "author"=>"jane", "posted"=>Time.utc(1700), 
+                            "pageViews"=>6, "tags"=>"nasty", "comments"=>[{"author"=>"will", "text"=>"i don't like the color"}, 
+                              {"author"=>"jenny", "text"=>"can i get that in green?"}], "other"=>{"bar"=>14 } },
+                          {"_id"=>3, "title"=>"this is some other title", "author"=>"jane", "posted"=>Time.utc(1700), 
+                            "pageViews"=>6, "tags"=>"filthy", "comments"=>[{"author"=>"will", "text"=>"i don't like the color"}, 
+                              {"author"=>"jenny", "text"=>"can i get that in green?"}], "other"=>{"bar"=>14 } }
+                          ]
+      results = @@test.aggregate([{"$unwind"=> "$tags"}])
+      assert_equal desired_results, results
+    end
+  end
 
   if @@version > "1.1.1"
     def test_map_reduce