mongo 2.0.6 → 2.1.0

data/lib/mongo/bulk_write/transformable.rb

@@ -0,0 +1,132 @@
+# Copyright (C) 2015 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class BulkWrite
+
+    # Defines behaviour around transformations.
+    #
+    # @api private
+    #
+    # @since 2.1.0
+    module Transformable
+
+      # The delete many model constant.
+      #
+      # @since 2.1.0
+      DELETE_MANY = :delete_many.freeze
+
+      # The delete one model constant.
+      #
+      # @since 2.1.0
+      DELETE_ONE = :delete_one.freeze
+
+      # The insert one model constant.
+      #
+      # @since 2.1.0
+      INSERT_ONE = :insert_one.freeze
+
+      # The replace one model constant.
+      #
+      # @since 2.1.0
+      REPLACE_ONE = :replace_one.freeze
+
+      # The update many model constant.
+      #
+      # @since 2.1.0
+      UPDATE_MANY = :update_many.freeze
+
+      # The update one model constant.
+      #
+      # @since 2.1.0
+      UPDATE_ONE = :update_one.freeze
+
+      # Proc to transform delete many ops.
+      #
+      # @since 2.1.0
+      DELETE_MANY_TRANSFORM = ->(doc){
+        { Operation::Q => doc[:filter], Operation::LIMIT => 0 }
+      }
+
+      # Proc to transform delete one ops.
+      #
+      # @since 2.1.0
+      DELETE_ONE_TRANSFORM = ->(doc){
+        { Operation::Q => doc[:filter], Operation::LIMIT => 1 }
+      }
+
+      # Proc to transform insert one ops.
+      #
+      # @since 2.1.0
+      INSERT_ONE_TRANSFORM = ->(doc){
+        doc
+      }
+
+      # Proc to transfor replace one ops.
+      #
+      # @since 2.1.0
+      REPLACE_ONE_TRANSFORM = ->(doc){
+        {
+          Operation::Q => doc[:filter],
+          Operation::U => doc[:replacement],
+          Operation::MULTI => false,
+          Operation::UPSERT => doc.fetch(:upsert, false)
+        }
+      }
+
+      # Proc to transform update many ops.
+      #
+      # @since 2.1.0
+      UPDATE_MANY_TRANSFORM = ->(doc){
+        {
+          Operation::Q => doc[:filter],
+          Operation::U => doc[:update],
+          Operation::MULTI => true,
+          Operation::UPSERT => doc.fetch(:upsert, false)
+        }
+      }
+
+      # Proc to transform update one ops.
+      #
+      # @since 2.1.0
+      UPDATE_ONE_TRANSFORM = ->(doc){
+        {
+          Operation::Q => doc[:filter],
+          Operation::U => doc[:update],
+          Operation::MULTI => false,
+          Operation::UPSERT => doc.fetch(:upsert, false)
+        }
+      }
+
+      # Document mappers from the bulk api input into proper commands.
+      #
+      # @since 2.1.0
+      MAPPERS = {
+        DELETE_MANY => DELETE_MANY_TRANSFORM,
+        DELETE_ONE  => DELETE_ONE_TRANSFORM,
+        INSERT_ONE  => INSERT_ONE_TRANSFORM,
+        REPLACE_ONE => REPLACE_ONE_TRANSFORM,
+        UPDATE_MANY => UPDATE_MANY_TRANSFORM,
+        UPDATE_ONE  => UPDATE_ONE_TRANSFORM
+      }.freeze
+
+      private
+
+      def transform(name, document)
+        validate(name, document)
+        MAPPERS[name].call(document)
+      end
+    end
+  end
+end

data/lib/mongo/bulk_write/unordered_combiner.rb

@@ -0,0 +1,52 @@
+# Copyright (C) 2015 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class BulkWrite
+
+    # Combines groups of bulk write operations in no order.
+    #
+    # @api private
+    #
+    # @since 2.1.0
+    class UnorderedCombiner
+      include Transformable
+      include Validatable
+      include Combineable
+
+      # Combine the requests in order.
+      #
+      # @api private
+      #
+      # @example Combine the requests.
+      #   combiner.combine
+      #
+      # @return [ Array<Hash> ] The combined requests.
+      #
+      # @since 2.1.0
+      def combine
+        combine_requests({}).map do |name, ops|
+          { name => ops }
+        end
+      end
+
+      private
+
+      def add(operations, name, document)
+        (operations[name] ||= []).push(transform(name, document))
+        operations
+      end
+    end
+  end
+end

data/lib/mongo/bulk_write/validatable.rb

@@ -0,0 +1,62 @@
+# Copyright (C) 2015 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class BulkWrite
+
+    # Defines behaviour around validations.
+    #
+    # @api private
+    #
+    # @since 2.1.0
+    module Validatable
+
+      # Validate the document.
+      #
+      # @api private
+      #
+      # @example Validate the document.
+      #   validatable.validate(:insert_one, { _id: 0 })
+      #
+      # @param [ Symbol ] name The operation name.
+      # @param [ Hash, BSON::Document ] document The document.
+      #
+      # @raise [ InvalidBulkOperation ] If not valid.
+      #
+      # @return [ Hash, BSON::Document ] The document.
+      #
+      # @since 2.1.0
+      def validate(name, document)
+        validate_operation(name)
+        validate_document(name, document)
+      end
+
+      private
+
+      def validate_document(name, document)
+        if document.respond_to?(:keys) || document.respond_to?(:data)
+          document
+        else
+          raise Error::InvalidBulkOperation.new(name, document)
+        end
+      end
+
+      def validate_operation(name)
+        unless Transformable::MAPPERS.key?(name)
+          raise Error::InvalidBulkOperationType.new(name)
+        end
+      end
+    end
+  end
+end

data/lib/mongo/bulk_write.rb

@@ -12,41 +12,177 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require 'mongo/bulk_write/bulk_writable'
-require 'mongo/bulk_write/ordered_bulk_write'
-require 'mongo/bulk_write/unordered_bulk_write'
+require 'mongo/bulk_write/result'
+require 'mongo/bulk_write/transformable'
+require 'mongo/bulk_write/validatable'
+require 'mongo/bulk_write/combineable'
+require 'mongo/bulk_write/ordered_combiner'
+require 'mongo/bulk_write/unordered_combiner'
+require 'mongo/bulk_write/result_combiner'
 
 module Mongo
-  module BulkWrite
-    extend self
+  class BulkWrite
+    extend Forwardable
+    include Retryable
 
-    # Get a bulk write object either of type ordered or unordered.
+    # @return [ Mongo::Collection ] collection The collection.
+    attr_reader :collection
+
+    # @return [ Array<Hash, BSON::Document> ] requests The requests.
+    attr_reader :requests
+
+    # @return [ Hash, BSON::Document ] options The options.
+    attr_reader :options
+
+    # Delegate various methods to the collection.
+    def_delegators :@collection, :database, :cluster, :next_primary
+
+    # Execute the bulk write operation.
+    #
+    # @example Execute the bulk write.
+    #   bulk_write.execute
+    #
+    # @return [ Mongo::BulkWrite::Result ] The result.
+    #
+    # @since 2.1.0
+    def execute
+      operation_id = Monitoring.next_operation_id
+      result_combiner = ResultCombiner.new
+      write_with_retry do
+        server = next_primary
+        operations.each do |operation|
+          execute_operation(
+            operation.keys.first,
+            operation.values.first,
+            server,
+            operation_id,
+            result_combiner
+          )
+        end
+      end
+      result_combiner.result
+    end
+
+    # Create the new bulk write operation.
     #
-    # @example Get a bulk write object.
-    #   Mongo::BulkWrite.get(collection, operations, ordered: true)
+    # @api private
     #
-    # @param [ Collection ] collection The collection on which the operations
-    #   will be executed.
+    # @example Create an ordered bulk write.
+    #   Mongo::BulkWrite.new(collection, [{ insert_one: { _id: 1 }}])
     #
-    # @param [ Array<Hash> ] operations The operations to execute.
+    # @example Create an unordered bulk write.
+    #   Mongo::BulkWrite.new(collection, [{ insert_one: { _id: 1 }}], ordered: false)
     #
-    # @param [ Hash ] options The options for the bulk write object.
+    # @example Create an ordered mixed bulk write.
+    #   Mongo::BulkWrite.new(
+    #     collection,
+    #     [
+    #       { insert_one: { _id: 1 }},
+    #       { update_one: { filter: { _id: 0 }, update: { '$set' => { name: 'test' }}}},
+    #       { delete_one: { filter: { _id: 2 }}}
+    #     ]
+    #   )
+    #
+    # @param [ Mongo::Collection ] collection The collection.
+    # @param [ Array<Hash, BSON::Document> ] requests The requests.
+    # @param [ Hash, BSON::Document ] options The options.
+    #
+    # @since 2.1.0
+    def initialize(collection, requests, options = {})
+      @collection = collection
+      @requests = requests
+      @options = options || {}
+    end
+
+    # Is the bulk write ordered?
+    #
+    # @api private
+    #
+    # @example Is the bulk write ordered?
+    #   bulk_write.ordered?
+    #
+    # @return [ true, false ] If the bulk write is ordered.
+    #
+    # @since 2.1.0
+    def ordered?
+      @ordered ||= options.fetch(:ordered, true)
+    end
+
+    # Get the write concern for the bulk write.
     #
-    # @option options [ true, false ] :ordered Whether the operations
-    #   should be executed in order.
-    # @option options [ Hash ] :write_concern The write concern options.
-    #   Can be :w => Integer, :fsync => Boolean, :j => Boolean.
+    # @api private
     #
-    # @return [ OrderedBulkWrite, UnorderedBulkWrite ] The appropriate bulk
-    #   write object.
+    # @example Get the write concern.
+    #   bulk_write.write_concern
     #
-    # @since 2.0.0
-    def get(collection, operations, options)
-      if options.fetch(:ordered, true)
-        OrderedBulkWrite.new(collection, operations, options)
+    # @return [ WriteConcern ] The write concern.
+    #
+    # @since 2.1.0
+    def write_concern
+      @write_concern ||= options[:write_concern] ?
+        WriteConcern.get(options[:write_concern]) : collection.write_concern
+    end
+
+    private
+
+    def base_spec(operation_id)
+      {
+        :db_name => database.name,
+        :coll_name => collection.name,
+        :write_concern => write_concern,
+        :ordered => ordered?,
+        :operation_id => operation_id
+      }
+    end
+
+    def execute_operation(name, values, server, operation_id, combiner)
+      begin
+        if values.size > server.max_write_batch_size
+          split_execute(name, values, server, operation_id, combiner)
+        else
+          combiner.combine!(send(name, values, server, operation_id), values.size)
+        end
+      rescue Error::MaxBSONSize, Error::MaxMessageSize => e
+        split_execute(name, values, server, operation_id, combiner)
+      end
+    end
+
+    def operations
+      if ordered?
+        OrderedCombiner.new(requests).combine
       else
-        UnorderedBulkWrite.new(collection, operations, options)
+        UnorderedCombiner.new(requests).combine
       end
     end
+
+    def split_execute(name, values, server, operation_id, combiner)
+      execute_operation(name, values.shift(values.size / 2), server, operation_id, combiner)
+      execute_operation(name, values, server, operation_id, combiner)
+    end
+
+    def delete(documents, server, operation_id)
+      Operation::Write::Bulk::Delete.new(
+        base_spec(operation_id).merge(:deletes => documents)
+      ).execute(server.context)
+    end
+
+    alias :delete_one :delete
+    alias :delete_many :delete
+
+    def insert_one(documents, server, operation_id)
+      Operation::Write::Bulk::Insert.new(
+        base_spec(operation_id).merge(:documents => documents)
+      ).execute(server.context)
+    end
+
+    def update(documents, server, operation_id)
+      Operation::Write::Bulk::Update.new(
+        base_spec(operation_id).merge(:updates => documents)
+      ).execute(server.context)
+    end
+
+    alias :replace_one :update
+    alias :update_one :update
+    alias :update_many :update
   end
 end

data/lib/mongo/client.rb

@@ -39,6 +39,9 @@ module Mongo
     # Delegate command execution to the current database.
     def_delegators :@database, :command
 
+    # Delegate subscription to monitoring.
+    def_delegators :@monitoring, :subscribe, :unsubscribe
+
     # Determine if this client is equivalent to another object.
     #
     # @example Check client equality.
@@ -119,7 +122,7 @@ module Mongo
     #   attempt a connection.
     # @option options [ Hash ] :read The read preference options. They consist of a
     #   mode specified as a symbol, an array of hashes known as tag_sets,
-    #   and two timing options: local_threshold and server_selection_timeout.
+    #   and local_threshold.
     #   :mode can be one of :secondary, :secondary_preferred, :primary,
     #   :primary_preferred, :nearest.
     # @option options [ Array<Hash, String> ] :roles The list of roles for the
@@ -143,14 +146,21 @@ module Mongo
     # @option options [ String ] :user The user name.
     # @option options [ Hash ] :write The write concern options. Can be :w =>
     #   Integer, :fsync => Boolean, :j => Boolean.
+    # @option options [ true, false ] :monitoring Initializes a client without
+    #   any default monitoring if false is provided.
+    # @option options [ Logger ] :logger A custom logger if desired.
+    # @option options [ true, false ] :truncate_logs Whether to truncate the
+    #   logs at the default 250 characters.
     #
     # @since 2.0.0
-    def initialize(addresses_or_uri, options = {})
+    def initialize(addresses_or_uri, options = Options::Redacted.new)
+      @monitoring = Monitoring.new(options)
       if addresses_or_uri.is_a?(::String)
         create_from_uri(addresses_or_uri, options)
       else
         create_from_addresses(addresses_or_uri, options)
       end
+      yield(self) if block_given?
     end
 
     # Get an inspection of the client as a string.
@@ -175,7 +185,7 @@ module Mongo
     #
     # @since 2.0.0
     def read_preference
-      @read_preference ||= ServerSelector.get(options[:read] || {}, options)
+      @read_preference ||= ServerSelector.get(Options::Redacted.new(options[:read] || {}).merge(options))
     end
 
     # Use the database with the provided name. This will switch the current
@@ -205,13 +215,14 @@ module Mongo
     # @return [ Mongo::Client ] A new client instance.
     #
     # @since 2.0.0
-    def with(new_options = {})
+    def with(new_options = Options::Redacted.new)
       clone.tap do |client|
-        client.options.update(new_options)
+        opts = Options::Redacted.new(new_options) || Options::Redacted.new
+        client.options.update(opts)
         Database.create(client)
         # We can't use the same cluster if some options that would affect it
         # have changed.
-        if cluster_modifying?(new_options)
+        if cluster_modifying?(opts)
           Cluster.create(client)
         end
       end
@@ -230,6 +241,30 @@ module Mongo
       @write_concern ||= WriteConcern.get(options[:write])
     end
 
+    # Close all connections.
+    #
+    # @example Disconnect the client.
+    #   client.close
+    #
+    # @return [ true ] Always true.
+    #
+    # @since 2.1.0
+    def close
+      @cluster.disconnect! and true
+    end
+
+    # Reconnect the client.
+    #
+    # @example Reconnect the client.
+    #   client.reconnect
+    #
+    # @return [ true ] Always true.
+    #
+    # @since 2.1.0
+    def reconnect
+      @cluster.reconnect! and true
+    end
+
     # Get the names of all databases.
     #
     # @example Get the database names.
@@ -239,7 +274,7 @@ module Mongo
     #
     # @since 2.0.5
     def database_names
-      list_databases.collect{ |info| info['name'] }
+      list_databases.collect{ |info| info[Database::NAME] }
     end
 
     # Get info for each database.
@@ -251,26 +286,27 @@ module Mongo
     #
     # @since 2.0.5
     def list_databases
-      use(Database::ADMIN).command(listDatabases: 1).first['databases']
+      use(Database::ADMIN).command(listDatabases: 1).first[Database::DATABASES]
     end
 
     private
 
-    def create_from_addresses(addresses, opts = {})
-      @options = Database::DEFAULT_OPTIONS.merge(opts).freeze
-      @cluster = Cluster.new(addresses, options)
+    def create_from_addresses(addresses, opts = Options::Redacted.new)
+      @options = Options::Redacted.new(Database::DEFAULT_OPTIONS.merge(opts)).freeze
+      @cluster = Cluster.new(addresses, @monitoring, options)
       @database = Database.new(self, options[:database], options)
     end
 
-    def create_from_uri(connection_string, opts = {})
-      uri = URI.new(connection_string)
-      @options = Database::DEFAULT_OPTIONS.merge(uri.client_options.merge(opts)).freeze
-      @cluster = Cluster.new(uri.servers, options)
+    def create_from_uri(connection_string, opts = Options::Redacted.new)
+      uri = URI.new(connection_string, opts)
+      @options = Options::Redacted.new(Database::DEFAULT_OPTIONS.merge(uri.client_options.merge(opts))).freeze
+      @cluster = Cluster.new(uri.servers, @monitoring, options)
       @database = Database.new(self, options[:database], options)
     end
 
     def initialize_copy(original)
       @options = original.options.dup
+      @monitoring = Monitoring.new(@options)
       @database = nil
       @read_preference = nil
       @write_concern = nil
@@ -278,7 +314,7 @@ module Mongo
 
     def cluster_modifying?(new_options)
       cluster_options = new_options.reject do |name|
-        CRUD_OPTIONS.include?(name)
+        CRUD_OPTIONS.include?(name.to_sym)
       end
       cluster_options.any? do |name, value|
         options[name] != value

data/lib/mongo/cluster/topology/replica_set.rb

@@ -60,16 +60,20 @@ module Mongo
         # @return [ ReplicaSet ] The topology.
         def elect_primary(description, servers)
           if description.replica_set_name == replica_set_name
-            log_debug([ "Server #{description.address.to_s} elected as primary in #{replica_set_name}." ])
-            servers.each do |server|
-              if server.primary? && server.address != description.address
-                server.description.unknown!
+            unless detect_stale_primary!(description)
+              log_debug("Server #{description.address.to_s} elected as primary in #{replica_set_name}.")
+              servers.each do |server|
+                if server.primary? && server.address != description.address
+                  server.description.unknown!
+                end
               end
+              update_max_election_id(description)
             end
           else
-            log_warn([
-              "Server #{description.address.to_s} in incorrect replica set: #{description.replica_set_name}."
-            ])
+            log_warn(
+              "Server #{description.address.to_s} has incorrect replica set name: " +
+              "'#{description.replica_set_name}'. The current replica set name is '#{replica_set_name}'."
+            )
           end
           self
         end
@@ -84,6 +88,7 @@ module Mongo
         # @since 2.0.0
         def initialize(options, seeds = [])
           @options = options
+          @max_election_id = 0
         end
 
         # A replica set topology is a replica set.
@@ -153,8 +158,9 @@ module Mongo
         def remove_hosts?(description)
           !description.config.empty? &&
             (description.primary? ||
-              description.hosts.empty? ||
-                !member_of_this_set?(description))
+              description.me_mismatch? ||
+                description.hosts.empty? ||
+                  !member_of_this_set?(description))
         end
 
         # Whether a specific server in the cluster can be removed, given a description.
@@ -215,6 +221,18 @@ module Mongo
 
         private
 
+        def update_max_election_id(description)
+          if description.election_id && description.election_id > @max_election_id
+            @max_election_id = description.election_id
+          end
+        end
+
+        def detect_stale_primary!(description)
+          if description.election_id && description.election_id < @max_election_id
+            description.unknown!
+          end
+        end
+
         def has_primary?(servers)
           servers.find { |s| s.primary? }
         end

data/lib/mongo/cluster/topology/sharded.rb

@@ -184,7 +184,7 @@ module Mongo
         private
 
         def remove_self?(description, server)
-          description.is_server?(server) && !description.mongos?
+          description.is_server?(server) && !(description.mongos? || description.unknown?)
         end
       end
     end

data/lib/mongo/cluster/topology/unknown.rb

@@ -55,7 +55,7 @@ module Mongo
         # @return [ Sharded, ReplicaSet ] The new topology.
         def elect_primary(description, servers)
           if description.mongos?
-            log_debug([ "Mongos #{description.address.to_s} discovered." ])
+            log_debug("Mongos #{description.address.to_s} discovered.")
             Sharded.new(options)
           else
             initialize_replica_set(description, servers)
@@ -204,7 +204,10 @@ module Mongo
         private
 
         def initialize_replica_set(description, servers)
-          log_debug([ "Server #{description.address.to_s} discovered as primary." ])
+          log_debug(
+            "Server #{description.address.to_s} discovered as primary in replica set: " +
+            "'#{description.replica_set_name}'. Changing topology to replica set."
+          )
           servers.each do |server|
             if server.standalone? && server.address != description.address
               server.description.unknown!