mongo 2.1.0.beta → 2.2.0

data/lib/mongo/bulk_write/transformable.rb

@@ -0,0 +1,132 @@
+# Copyright (C) 2015 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class BulkWrite
+
+    # Defines behaviour around transformations.
+    #
+    # @api private
+    #
+    # @since 2.1.0
+    module Transformable
+
+      # The delete many model constant.
+      #
+      # @since 2.1.0
+      DELETE_MANY = :delete_many.freeze
+
+      # The delete one model constant.
+      #
+      # @since 2.1.0
+      DELETE_ONE = :delete_one.freeze
+
+      # The insert one model constant.
+      #
+      # @since 2.1.0
+      INSERT_ONE = :insert_one.freeze
+
+      # The replace one model constant.
+      #
+      # @since 2.1.0
+      REPLACE_ONE = :replace_one.freeze
+
+      # The update many model constant.
+      #
+      # @since 2.1.0
+      UPDATE_MANY = :update_many.freeze
+
+      # The update one model constant.
+      #
+      # @since 2.1.0
+      UPDATE_ONE = :update_one.freeze
+
+      # Proc to transform delete many ops.
+      #
+      # @since 2.1.0
+      DELETE_MANY_TRANSFORM = ->(doc){
+        { Operation::Q => doc[:filter], Operation::LIMIT => 0 }
+      }
+
+      # Proc to transform delete one ops.
+      #
+      # @since 2.1.0
+      DELETE_ONE_TRANSFORM = ->(doc){
+        { Operation::Q => doc[:filter], Operation::LIMIT => 1 }
+      }
+
+      # Proc to transform insert one ops.
+      #
+      # @since 2.1.0
+      INSERT_ONE_TRANSFORM = ->(doc){
+        doc
+      }
+
+      # Proc to transfor replace one ops.
+      #
+      # @since 2.1.0
+      REPLACE_ONE_TRANSFORM = ->(doc){
+        {
+          Operation::Q => doc[:filter],
+          Operation::U => doc[:replacement],
+          Operation::MULTI => false,
+          Operation::UPSERT => doc.fetch(:upsert, false)
+        }
+      }
+
+      # Proc to transform update many ops.
+      #
+      # @since 2.1.0
+      UPDATE_MANY_TRANSFORM = ->(doc){
+        {
+          Operation::Q => doc[:filter],
+          Operation::U => doc[:update],
+          Operation::MULTI => true,
+          Operation::UPSERT => doc.fetch(:upsert, false)
+        }
+      }
+
+      # Proc to transform update one ops.
+      #
+      # @since 2.1.0
+      UPDATE_ONE_TRANSFORM = ->(doc){
+        {
+          Operation::Q => doc[:filter],
+          Operation::U => doc[:update],
+          Operation::MULTI => false,
+          Operation::UPSERT => doc.fetch(:upsert, false)
+        }
+      }
+
+      # Document mappers from the bulk api input into proper commands.
+      #
+      # @since 2.1.0
+      MAPPERS = {
+        DELETE_MANY => DELETE_MANY_TRANSFORM,
+        DELETE_ONE  => DELETE_ONE_TRANSFORM,
+        INSERT_ONE  => INSERT_ONE_TRANSFORM,
+        REPLACE_ONE => REPLACE_ONE_TRANSFORM,
+        UPDATE_MANY => UPDATE_MANY_TRANSFORM,
+        UPDATE_ONE  => UPDATE_ONE_TRANSFORM
+      }.freeze
+
+      private
+
+      def transform(name, document)
+        validate(name, document)
+        MAPPERS[name].call(document)
+      end
+    end
+  end
+end

data/lib/mongo/bulk_write/unordered_combiner.rb

@@ -0,0 +1,52 @@
+# Copyright (C) 2015 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class BulkWrite
+
+    # Combines groups of bulk write operations in no order.
+    #
+    # @api private
+    #
+    # @since 2.1.0
+    class UnorderedCombiner
+      include Transformable
+      include Validatable
+      include Combineable
+
+      # Combine the requests in order.
+      #
+      # @api private
+      #
+      # @example Combine the requests.
+      #   combiner.combine
+      #
+      # @return [ Array<Hash> ] The combined requests.
+      #
+      # @since 2.1.0
+      def combine
+        combine_requests({}).map do |name, ops|
+          { name => ops }
+        end
+      end
+
+      private
+
+      def add(operations, name, document)
+        (operations[name] ||= []).push(transform(name, document))
+        operations
+      end
+    end
+  end
+end

data/lib/mongo/bulk_write/validatable.rb

@@ -0,0 +1,62 @@
+# Copyright (C) 2015 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class BulkWrite
+
+    # Defines behaviour around validations.
+    #
+    # @api private
+    #
+    # @since 2.1.0
+    module Validatable
+
+      # Validate the document.
+      #
+      # @api private
+      #
+      # @example Validate the document.
+      #   validatable.validate(:insert_one, { _id: 0 })
+      #
+      # @param [ Symbol ] name The operation name.
+      # @param [ Hash, BSON::Document ] document The document.
+      #
+      # @raise [ InvalidBulkOperation ] If not valid.
+      #
+      # @return [ Hash, BSON::Document ] The document.
+      #
+      # @since 2.1.0
+      def validate(name, document)
+        validate_operation(name)
+        validate_document(name, document)
+      end
+
+      private
+
+      def validate_document(name, document)
+        if document.respond_to?(:keys) || document.respond_to?(:data)
+          document
+        else
+          raise Error::InvalidBulkOperation.new(name, document)
+        end
+      end
+
+      def validate_operation(name)
+        unless Transformable::MAPPERS.key?(name)
+          raise Error::InvalidBulkOperationType.new(name)
+        end
+      end
+    end
+  end
+end

data/lib/mongo/bulk_write.rb

@@ -12,42 +12,183 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require 'mongo/bulk_write/bulk_writable'
-require 'mongo/bulk_write/ordered_bulk_write'
-require 'mongo/bulk_write/unordered_bulk_write'
 require 'mongo/bulk_write/result'
+require 'mongo/bulk_write/transformable'
+require 'mongo/bulk_write/validatable'
+require 'mongo/bulk_write/combineable'
+require 'mongo/bulk_write/ordered_combiner'
+require 'mongo/bulk_write/unordered_combiner'
+require 'mongo/bulk_write/result_combiner'
 
 module Mongo
-  module BulkWrite
-    extend self
+  class BulkWrite
+    extend Forwardable
+    include Retryable
 
-    # Get a bulk write object either of type ordered or unordered.
+    # @return [ Mongo::Collection ] collection The collection.
+    attr_reader :collection
+
+    # @return [ Array<Hash, BSON::Document> ] requests The requests.
+    attr_reader :requests
+
+    # @return [ Hash, BSON::Document ] options The options.
+    attr_reader :options
+
+    # Delegate various methods to the collection.
+    def_delegators :@collection, :database, :cluster, :next_primary
+
+    def_delegators :database, :client
+
+    # Execute the bulk write operation.
+    #
+    # @example Execute the bulk write.
+    #   bulk_write.execute
+    #
+    # @return [ Mongo::BulkWrite::Result ] The result.
+    #
+    # @since 2.1.0
+    def execute
+      operation_id = Monitoring.next_operation_id
+      result_combiner = ResultCombiner.new
+      write_with_retry do
+        server = next_primary
+        operations.each do |operation|
+          execute_operation(
+            operation.keys.first,
+            operation.values.first,
+            server,
+            operation_id,
+            result_combiner
+          )
+        end
+      end
+      result_combiner.result
+    end
+
+    # Create the new bulk write operation.
+    #
+    # @api private
+    #
+    # @example Create an ordered bulk write.
+    #   Mongo::BulkWrite.new(collection, [{ insert_one: { _id: 1 }}])
+    #
+    # @example Create an unordered bulk write.
+    #   Mongo::BulkWrite.new(collection, [{ insert_one: { _id: 1 }}], ordered: false)
+    #
+    # @example Create an ordered mixed bulk write.
+    #   Mongo::BulkWrite.new(
+    #     collection,
+    #     [
+    #       { insert_one: { _id: 1 }},
+    #       { update_one: { filter: { _id: 0 }, update: { '$set' => { name: 'test' }}}},
+    #       { delete_one: { filter: { _id: 2 }}}
+    #     ]
+    #   )
+    #
+    # @param [ Mongo::Collection ] collection The collection.
+    # @param [ Array<Hash, BSON::Document> ] requests The requests.
+    # @param [ Hash, BSON::Document ] options The options.
+    #
+    # @since 2.1.0
+    def initialize(collection, requests, options = {})
+      @collection = collection
+      @requests = requests
+      @options = options || {}
+    end
+
+    # Is the bulk write ordered?
+    #
+    # @api private
     #
-    # @example Get a bulk write object.
-    #   Mongo::BulkWrite.get(collection, operations, ordered: true)
+    # @example Is the bulk write ordered?
+    #   bulk_write.ordered?
     #
-    # @param [ Collection ] collection The collection on which the operations
-    #   will be executed.
+    # @return [ true, false ] If the bulk write is ordered.
     #
-    # @param [ Array<Hash> ] operations The operations to execute.
+    # @since 2.1.0
+    def ordered?
+      @ordered ||= options.fetch(:ordered, true)
+    end
+
+    # Get the write concern for the bulk write.
     #
-    # @param [ Hash ] options The options for the bulk write object.
+    # @api private
     #
-    # @option options [ true, false ] :ordered Whether the operations
-    #   should be executed in order.
-    # @option options [ Hash ] :write_concern The write concern options.
-    #   Can be :w => Integer, :fsync => Boolean, :j => Boolean.
+    # @example Get the write concern.
+    #   bulk_write.write_concern
     #
-    # @return [ OrderedBulkWrite, UnorderedBulkWrite ] The appropriate bulk
-    #   write object.
+    # @return [ WriteConcern ] The write concern.
     #
-    # @since 2.0.0
-    def get(collection, operations, options)
-      if options.fetch(:ordered, true)
-        OrderedBulkWrite.new(collection, operations, options)
+    # @since 2.1.0
+    def write_concern
+      @write_concern ||= options[:write_concern] ?
+        WriteConcern.get(options[:write_concern]) : collection.write_concern
+    end
+
+    private
+
+    def base_spec(operation_id)
+      {
+        :db_name => database.name,
+        :coll_name => collection.name,
+        :write_concern => write_concern,
+        :ordered => ordered?,
+        :operation_id => operation_id,
+        :bypass_document_validation => !!options[:bypass_document_validation],
+        :options => options,
+        :id_generator => client.options[:id_generator]
+      }
+    end
+
+    def execute_operation(name, values, server, operation_id, combiner)
+      begin
+        if values.size > server.max_write_batch_size
+          split_execute(name, values, server, operation_id, combiner)
+        else
+          combiner.combine!(send(name, values, server, operation_id), values.size)
+        end
+      rescue Error::MaxBSONSize, Error::MaxMessageSize => e
+        raise e if values.size <= 1
+        split_execute(name, values, server, operation_id, combiner)
+      end
+    end
+
+    def operations
+      if ordered?
+        OrderedCombiner.new(requests).combine
       else
-        UnorderedBulkWrite.new(collection, operations, options)
+        UnorderedCombiner.new(requests).combine
       end
     end
+
+    def split_execute(name, values, server, operation_id, combiner)
+      execute_operation(name, values.shift(values.size / 2), server, operation_id, combiner)
+      execute_operation(name, values, server, operation_id, combiner)
+    end
+
+    def delete(documents, server, operation_id)
+      Operation::Write::Bulk::Delete.new(
+        base_spec(operation_id).merge(:deletes => documents)
+      ).execute(server.context)
+    end
+
+    alias :delete_one :delete
+    alias :delete_many :delete
+
+    def insert_one(documents, server, operation_id)
+      Operation::Write::Bulk::Insert.new(
+        base_spec(operation_id).merge(:documents => documents)
+      ).execute(server.context)
+    end
+
+    def update(documents, server, operation_id)
+      Operation::Write::Bulk::Update.new(
+        base_spec(operation_id).merge(:updates => documents)
+      ).execute(server.context)
+    end
+
+    alias :replace_one :update
+    alias :update_one :update
+    alias :update_many :update
   end
 end

data/lib/mongo/client.rb

@@ -20,6 +20,7 @@ module Mongo
   # @since 2.0.0
   class Client
     extend Forwardable
+    include Loggable
 
     # The options that do not affect the behaviour of a cluster and its
     # subcomponents.
@@ -27,6 +28,42 @@ module Mongo
     # @since 2.1.0
     CRUD_OPTIONS = [ :database, :read, :write ].freeze
 
+    # Valid client options.
+    #
+    # @since 2.1.2
+    VALID_OPTIONS = [
+      :auth_mech,
+      :auth_mech_properties,
+      :auth_source,
+      :connect,
+      :connect_timeout,
+      :database,
+      :heartbeat_frequency,
+      :id_generator,
+      :local_threshold,
+      :logger,
+      :max_pool_size,
+      :max_read_retries,
+      :min_pool_size,
+      :monitoring,
+      :password,
+      :read,
+      :read_retry_interval,
+      :replica_set,
+      :server_selection_timeout,
+      :socket_timeout,
+      :ssl,
+      :ssl_ca_cert,
+      :ssl_cert,
+      :ssl_key,
+      :ssl_key_pass_phrase,
+      :ssl_verify,
+      :truncate_logs,
+      :user,
+      :wait_queue_timeout,
+      :write
+    ].freeze
+
     # @return [ Mongo::Cluster ] cluster The cluster of servers for the client.
     attr_reader :cluster
 
@@ -36,8 +73,8 @@ module Mongo
     # @return [ Hash ] options The configuration options.
     attr_reader :options
 
-    # Delegate command execution to the current database.
-    def_delegators :@database, :command
+    # Delegate command and collections execution to the current database.
+    def_delegators :@database, :command, :collections
 
     # Delegate subscription to monitoring.
     def_delegators :@monitoring, :subscribe, :unsubscribe
@@ -122,11 +159,9 @@ module Mongo
     #   attempt a connection.
     # @option options [ Hash ] :read The read preference options. They consist of a
     #   mode specified as a symbol, an array of hashes known as tag_sets,
-    #   and two timing options: local_threshold and server_selection_timeout.
+    #   and local_threshold.
     #   :mode can be one of :secondary, :secondary_preferred, :primary,
     #   :primary_preferred, :nearest.
-    # @option options [ Array<Hash, String> ] :roles The list of roles for the
-    #   user.
     # @option options [ Symbol ] :replica_set The name of the replica set to
     #   connect to. Servers not in this replica set will be ignored.
     # @option options [ true, false ] :ssl Whether to use SSL.
@@ -145,17 +180,26 @@ module Mongo
     #   execute operations on a socket.
     # @option options [ String ] :user The user name.
     # @option options [ Hash ] :write The write concern options. Can be :w =>
-    #   Integer, :fsync => Boolean, :j => Boolean.
+    #   Integer|String, :fsync => Boolean, :j => Boolean.
     # @option options [ true, false ] :monitoring Initializes a client without
     #   any default monitoring if false is provided.
+    # @option options [ Logger ] :logger A custom logger if desired.
+    # @option options [ true, false ] :truncate_logs Whether to truncate the
+    #   logs at the default 250 characters.
+    # @option options [ Integer ] :max_read_retries The maximum number of read
+    #   retries on mongos query failures.
+    # @option options [ Float ] :read_retry_interval The interval, in seconds,
+    #   in which reads on a mongos are retried.
+    # @option options [ Object ] :id_generator A custom object to generate ids
+    #   for documents. Must respond to #generate.
     #
     # @since 2.0.0
-    def initialize(addresses_or_uri, options = {})
+    def initialize(addresses_or_uri, options = Options::Redacted.new)
       @monitoring = Monitoring.new(options)
       if addresses_or_uri.is_a?(::String)
-        create_from_uri(addresses_or_uri, options)
+        create_from_uri(addresses_or_uri, validate_options(options))
       else
-        create_from_addresses(addresses_or_uri, options)
+        create_from_addresses(addresses_or_uri, validate_options(options))
       end
       yield(self) if block_given?
     end
@@ -182,7 +226,7 @@ module Mongo
     #
     # @since 2.0.0
     def read_preference
-      @read_preference ||= ServerSelector.get(options[:read] || {}, options)
+      @read_preference ||= ServerSelector.get(Options::Redacted.new(options[:read] || {}).merge(options))
     end
 
     # Use the database with the provided name. This will switch the current
@@ -212,9 +256,9 @@ module Mongo
     # @return [ Mongo::Client ] A new client instance.
     #
     # @since 2.0.0
-    def with(new_options = {})
+    def with(new_options = Options::Redacted.new)
       clone.tap do |client|
-        opts = new_options || {}
+        opts = validate_options(new_options)
         client.options.update(opts)
         Database.create(client)
         # We can't use the same cluster if some options that would affect it
@@ -271,7 +315,7 @@ module Mongo
     #
     # @since 2.0.5
     def database_names
-      list_databases.collect{ |info| info['name'] }
+      list_databases.collect{ |info| info[Database::NAME] }
     end
 
     # Get info for each database.
@@ -283,19 +327,19 @@ module Mongo
     #
     # @since 2.0.5
     def list_databases
-      use(Database::ADMIN).command(listDatabases: 1).first['databases']
+      use(Database::ADMIN).command(listDatabases: 1).first[Database::DATABASES]
     end
 
     private
 
-    def create_from_addresses(addresses, opts = {})
+    def create_from_addresses(addresses, opts = Options::Redacted.new)
       @options = Database::DEFAULT_OPTIONS.merge(opts).freeze
       @cluster = Cluster.new(addresses, @monitoring, options)
       @database = Database.new(self, options[:database], options)
     end
 
-    def create_from_uri(connection_string, opts = {})
-      uri = URI.new(connection_string)
+    def create_from_uri(connection_string, opts = Options::Redacted.new)
+      uri = URI.new(connection_string, opts)
       @options = Database::DEFAULT_OPTIONS.merge(uri.client_options.merge(opts)).freeze
       @cluster = Cluster.new(uri.servers, @monitoring, options)
       @database = Database.new(self, options[:database], options)
@@ -303,6 +347,7 @@ module Mongo
 
     def initialize_copy(original)
       @options = original.options.dup
+      @monitoring = Monitoring.new(@options)
       @database = nil
       @read_preference = nil
       @write_concern = nil
@@ -310,11 +355,23 @@ module Mongo
 
     def cluster_modifying?(new_options)
       cluster_options = new_options.reject do |name|
-        CRUD_OPTIONS.include?(name)
+        CRUD_OPTIONS.include?(name.to_sym)
       end
       cluster_options.any? do |name, value|
         options[name] != value
       end
     end
+
+    def validate_options(opts = Options::Redacted.new)
+      return Options::Redacted.new unless opts
+      Options::Redacted.new(opts.select do |o|
+        if VALID_OPTIONS.include?(o)
+          true
+        else
+          log_warn("Unsupported client option '#{o}'. It will be ignored.")
+          false
+        end
+      end)
+    end
   end
 end

data/lib/mongo/cluster/topology/replica_set.rb

@@ -61,7 +61,7 @@ module Mongo
         def elect_primary(description, servers)
           if description.replica_set_name == replica_set_name
             unless detect_stale_primary!(description)
-              log_debug([ "Server #{description.address.to_s} elected as primary in #{replica_set_name}." ])
+              log_debug("Server #{description.address.to_s} elected as primary in #{replica_set_name}.")
               servers.each do |server|
                 if server.primary? && server.address != description.address
                   server.description.unknown!
@@ -70,9 +70,10 @@ module Mongo
               update_max_election_id(description)
             end
           else
-            log_warn([
-              "Server #{description.address.to_s} in incorrect replica set: #{description.replica_set_name}."
-            ])
+            log_warn(
+              "Server #{description.address.to_s} has incorrect replica set name: " +
+              "'#{description.replica_set_name}'. The current replica set name is '#{replica_set_name}'."
+            )
           end
           self
         end
@@ -157,8 +158,9 @@ module Mongo
         def remove_hosts?(description)
           !description.config.empty? &&
             (description.primary? ||
-              description.hosts.empty? ||
-                !member_of_this_set?(description))
+              description.me_mismatch? ||
+                description.hosts.empty? ||
+                  !member_of_this_set?(description))
         end
 
         # Whether a specific server in the cluster can be removed, given a description.

data/lib/mongo/cluster/topology/unknown.rb

@@ -55,7 +55,7 @@ module Mongo
         # @return [ Sharded, ReplicaSet ] The new topology.
         def elect_primary(description, servers)
           if description.mongos?
-            log_debug([ "Mongos #{description.address.to_s} discovered." ])
+            log_debug("Mongos #{description.address.to_s} discovered.")
             Sharded.new(options)
           else
             initialize_replica_set(description, servers)
@@ -204,7 +204,10 @@ module Mongo
         private
 
         def initialize_replica_set(description, servers)
-          log_debug([ "Server #{description.address.to_s} discovered as primary." ])
+          log_debug(
+            "Server #{description.address.to_s} discovered as primary in replica set: " +
+            "'#{description.replica_set_name}'. Changing topology to replica set."
+          )
           servers.each do |server|
             if server.standalone? && server.address != description.address
               server.description.unknown!