mongo 2.6.2 → 2.16.0

data/lib/mongo/background_thread.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2019-2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+
+  # The run!, running? and stop! methods used to be part of the public API
+  # in some of the classes which now include this module. Therefore these
+  # methods must be considered part of the driver's public API for backwards
+  # compatibility reasons. However using these methods outside of the driver
+  # is deprecated.
+  #
+  # @note Do not start or stop background threads in finalizers. See
+  #   https://jira.mongodb.org/browse/RUBY-2453 and
+  #   https://bugs.ruby-lang.org/issues/16288. When interpreter exits,
+  #   background threads are stopped first and finalizers are invoked next,
+  #   and MRI's internal data structures are basically corrupt at this point
+  #   if threads are being referenced. Prior to interpreter shutdown this
+  #   means threads cannot be stopped by objects going out of scope, but
+  #   most likely the threads hold references to said objects anyway if
+  #   work is being performed thus the objects wouldn't go out of scope in
+  #   the first place.
+  #
+  # @api private
+  module BackgroundThread
+    include Loggable
+
+    # Start the background thread.
+    #
+    # If the thread is already running, this method does nothing.
+    #
+    # @api public for backwards compatibility only
+    def run!
+      if @stop_requested && @thread
+        wait_for_stop
+        if @thread.alive?
+          log_warn("Starting a new background thread in #{self}, but the previous background thread is still running")
+          @thread = nil
+        end
+        @stop_requested = false
+      end
+      if running?
+        @thread
+      else
+        start!
+      end
+    end
+
+    # @api public for backwards compatibility only
+    def running?
+      if @thread
+        @thread.alive?
+      else
+        false
+      end
+    end
+
+    # Stop the background thread and wait for to terminate for a reasonable
+    # amount of time.
+    #
+    # @return [ true | false ] Whether the thread was terminated.
+    #
+    # @api public for backwards compatibility only
+    def stop!
+      # If the thread was not started, there is nothing to stop.
+      #
+      # Classes including this module may want to perform additional
+      # cleanup, which they can do by overriding this method.
+      return true unless @thread
+
+      # Background threads generally perform operations in a loop.
+      # This flag is meant to be checked on each iteration of the
+      # working loops and the thread should stop working when this flag
+      # is set.
+      @stop_requested = true
+
+      # Besides setting the flag, a particular class may have additional
+      # ways of signaling the background thread to either stop working or
+      # wake up to check the stop flag, for example, setting a semaphore.
+      # This can be accomplished by providing the pre_stop method.
+      pre_stop
+
+      # Now we have requested the graceful termination, and we could wait
+      # for the thread to exit on its own accord. A future version of the
+      # driver may allow a certain amount of time for the thread to quit.
+      # For now, we additionally use the Ruby machinery to request the thread
+      # be terminated, and do so immediately.
+      #
+      # Note that this may cause the background thread to terminate in
+      # the middle of an operation.
+      @thread.kill
+
+      wait_for_stop
+    end
+
+    private
+
+    def start!
+      @thread = Thread.new do
+        catch(:done) do
+          until @stop_requested
+            do_work
+          end
+        end
+      end
+    end
+
+    # Waits for the thread to die, with a timeout.
+    #
+    # Returns true if the thread died, false otherwise.
+    def wait_for_stop
+      # Wait for the thread to die. This is important in order to reliably
+      # clean up resources like connections knowing that no background
+      # thread will reconnect because it is still working.
+      #
+      # However, we do not want to wait indefinitely because in theory
+      # a background thread could be performing, say, network I/O and if
+      # the network is no longer available that could take a long time.
+      start_time = Utils.monotonic_time
+      ([0.1, 0.15] + [0.2] * 5 + [0.3] * 20).each do |interval|
+        begin
+          Timeout.timeout(interval) do
+            @thread.join
+          end
+          break
+        rescue ::Timeout::Error
+        end
+      end
+
+      # Some driver objects can be reconnected, for backwards compatibiilty
+      # reasons. Clear the thread instance variable to support this cleanly.
+      if @thread.alive?
+        log_warn("Failed to stop the background thread in #{self} in #{(Utils.monotonic_time - start_time).to_i} seconds: #{@thread.inspect} (thread status: #{@thread.status})")
+        # On JRuby the thread may be stuck in aborting state
+        # seemingly indefinitely. If the thread is aborting, consider it dead
+        # for our purposes (we will create a new thread if needed, and
+        # the background thread monitor will not detect the aborting thread
+        # as being alive).
+        if @thread.status == 'aborting'
+          @thread = nil
+          @stop_requested = false
+        end
+        false
+      else
+        @thread = nil
+        @stop_requested = false
+        true
+      end
+    end
+
+    # Override this method to do the work in the background thread.
+    def do_work
+    end
+
+    # Override this method to perform additional signaling for the background
+    # thread to stop.
+    def pre_stop
+    end
+  end
+end

data/lib/mongo/bson.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2015-2018 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/bulk_write/combineable.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2015-2018 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +18,7 @@
 module Mongo
   class BulkWrite
 
-    # Defines behaviour around combiners
+    # Defines behavior around combiners
     #
     # @api private
     #
@@ -25,13 +28,6 @@ module Mongo
       # @return [ Array<Hash, BSON::Document> ] requests The provided requests.
       attr_reader :requests
 
-      # @return [ true, false ] has_collation Whether one or more operations has a collation defined.
-      attr_reader :has_collation
-
-      # @return [ true, false ] has_array_filters Whether one or more operations specifies an array
-      # filters option.
-      attr_reader :has_array_filters
-
       # Create the ordered combiner.
       #
       # @api private
@@ -46,6 +42,25 @@ module Mongo
         @requests = requests
         @has_collation = false
         @has_array_filters = false
+        @has_hint = false
+      end
+
+      # @return [ Boolean ] Whether one or more operation specifies the collation
+      #   option.
+      def has_collation?
+        @has_collation
+      end
+
+      # @return [ Boolean ] Whether one or more operation specifies the
+      #   array_filters option.
+      def has_array_filters?
+        @has_array_filters
+      end
+
+      # @return [ Boolean ] Whether one or more operation specifies the
+      #   hint option.
+      def has_hint?
+        @has_hint
       end
 
       private

data/lib/mongo/bulk_write/ordered_combiner.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2015-2018 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/bulk_write/result.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2015-2018 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -180,7 +183,7 @@ module Mongo
       #
       # @since 2.1.0
       def validate!
-        if @results[Error::WRITE_ERRORS] || @results[Error::WRITE_CONCERN_ERRORS]
+        if @results['writeErrors'] || @results['writeConcernErrors']
           raise Error::BulkWriteError.new(@results)
         else
           self

data/lib/mongo/bulk_write/result_combiner.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2015-2018 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -53,8 +56,18 @@ module Mongo
       #
       # @since 2.1.0
       def combine!(result, count)
-        combine_counts!(result)
-        combine_ids!(result)
+        # Errors can be communicated by the server in a variety of fields:
+        # writeError, writeErrors, writeConcernError, writeConcernErrors.
+        # Currently only errors given in writeConcernErrors will cause
+        # counts not to be added, because this behavior is covered by the
+        # retryable writes tests. It is possible that some or all of the
+        # other errors should also be excluded when combining counts and
+        # ids, and it is also possible that only a subset of these error
+        # fields is actually possible in the context of bulk writes.
+        unless result.write_concern_error?
+          combine_counts!(result)
+          combine_ids!(result)
+        end
         combine_errors!(result)
         @count += count
       end
@@ -102,7 +115,7 @@ module Mongo
       def combine_write_errors!(result)
         if write_errors = result.aggregate_write_errors(count)
           results.merge!(
-            Error::WRITE_ERRORS => ((results[Error::WRITE_ERRORS] || []) << write_errors).flatten
+            'writeErrors' => ((results['writeErrors'] || []) << write_errors).flatten
           )
         else
           result.validate!
@@ -111,7 +124,7 @@ module Mongo
 
       def combine_write_concern_errors!(result)
         if write_concern_errors = result.aggregate_write_concern_errors(count)
-          results[Error::WRITE_CONCERN_ERRORS] = (results[Error::WRITE_CONCERN_ERRORS] || []) +
+          results['writeConcernErrors'] = (results['writeConcernErrors'] || []) +
                                                    write_concern_errors
         end
       end

data/lib/mongo/bulk_write/transformable.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2015-2018 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +18,7 @@
 module Mongo
   class BulkWrite
 
-    # Defines behaviour around transformations.
+    # Defines behavior around transformations.
     #
     # @api private
     #
@@ -56,9 +59,12 @@ module Mongo
       #
       # @since 2.1.0
       DELETE_MANY_TRANSFORM = ->(doc){
-        { Operation::Q => doc[:filter],
-          Operation::LIMIT => 0 }.tap do |d|
+        {
+          Operation::Q => doc[:filter],
+          Operation::LIMIT => 0,
+        }.tap do |d|
           d[Operation::COLLATION] = doc[:collation] if doc[:collation]
+          d['hint'] = doc[:hint] if doc[:hint]
         end
       }
 
@@ -66,9 +72,12 @@ module Mongo
       #
       # @since 2.1.0
       DELETE_ONE_TRANSFORM = ->(doc){
-        { Operation::Q => doc[:filter],
-          Operation::LIMIT => 1 }.tap do |d|
+        {
+          Operation::Q => doc[:filter],
+          Operation::LIMIT => 1,
+        }.tap do |d|
           d[Operation::COLLATION] = doc[:collation] if doc[:collation]
+          d['hint'] = doc[:hint] if doc[:hint]
         end
       }
 
@@ -86,10 +95,10 @@ module Mongo
         {
           Operation::Q => doc[:filter],
           Operation::U => doc[:replacement],
-          Operation::MULTI => false,
-          Operation::UPSERT => doc.fetch(:upsert, false)
         }.tap do |d|
+          d['upsert'] = true if doc[:upsert]
           d[Operation::COLLATION] = doc[:collation] if doc[:collation]
+          d['hint'] = doc[:hint] if doc[:hint]
         end
       }
 
@@ -101,10 +110,11 @@ module Mongo
           Operation::Q => doc[:filter],
           Operation::U => doc[:update],
           Operation::MULTI => true,
-          Operation::UPSERT => doc.fetch(:upsert, false)
         }.tap do |d|
+          d['upsert'] = true if doc[:upsert]
           d[Operation::COLLATION] = doc[:collation] if doc[:collation]
           d[Operation::ARRAY_FILTERS] = doc[:array_filters] if doc[:array_filters]
+          d['hint'] = doc[:hint] if doc[:hint]
         end
       }
 
@@ -115,11 +125,11 @@ module Mongo
         {
           Operation::Q => doc[:filter],
           Operation::U => doc[:update],
-          Operation::MULTI => false,
-          Operation::UPSERT => doc.fetch(:upsert, false)
         }.tap do |d|
+          d['upsert'] = true if doc[:upsert]
           d[Operation::COLLATION] = doc[:collation] if doc[:collation]
           d[Operation::ARRAY_FILTERS] = doc[:array_filters] if doc[:array_filters]
+          d['hint'] = doc[:hint] if doc[:hint]
         end
       }
 

data/lib/mongo/bulk_write/unordered_combiner.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2015-2018 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/bulk_write/validatable.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2015-2018 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +18,7 @@
 module Mongo
   class BulkWrite
 
-    # Defines behaviour around validations.
+    # Defines behavior around validations.
     #
     # @api private
     #
@@ -47,6 +50,10 @@ module Mongo
         if document.respond_to?(:keys) && document[:array_filters]
           @has_array_filters = true
         end
+
+        if document.respond_to?(:keys) && document[:hint]
+          @has_hint = true
+        end
       end
 
       private

data/lib/mongo/bulk_write.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2014-2018 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,7 +26,7 @@ require 'mongo/bulk_write/result_combiner'
 module Mongo
   class BulkWrite
     extend Forwardable
-    include Retryable
+    include Operation::ResponseHandling
 
     # @return [ Mongo::Collection ] collection The collection.
     attr_reader :collection
@@ -38,6 +41,8 @@ module Mongo
     def_delegators :@collection,
                    :database,
                    :cluster,
+                   :write_with_retry,
+                   :nro_write_with_retry,
                    :next_primary
 
     def_delegators :database, :client
@@ -56,27 +61,35 @@ module Mongo
       operations = op_combiner.combine
 
       client.send(:with_session, @options) do |session|
+        context = Operation::Context.new(client: client, session: session)
         operations.each do |operation|
           if single_statement?(operation)
+            write_concern = write_concern(session)
             write_with_retry(session, write_concern) do |server, txn_num|
-              execute_operation(
+              server.with_connection(service_id: context.service_id) do |connection|
+                execute_operation(
                   operation.keys.first,
                   operation.values.flatten,
-                  server,
+                  connection,
+                  context,
                   operation_id,
                   result_combiner,
                   session,
                   txn_num)
+              end
             end
           else
-            legacy_write_with_retry do |server|
-              execute_operation(
+            nro_write_with_retry(session, write_concern) do |server|
+              server.with_connection(service_id: context.service_id) do |connection|
+                execute_operation(
                   operation.keys.first,
                   operation.values.flatten,
-                  server,
+                  connection,
+                  context,
                   operation_id,
                   result_combiner,
                   session)
+              end
             end
           end
         end
@@ -139,9 +152,10 @@ module Mongo
     # @return [ WriteConcern ] The write concern.
     #
     # @since 2.1.0
-    def write_concern
+    def write_concern(session = nil)
       @write_concern ||= options[:write_concern] ?
-        WriteConcern.get(options[:write_concern]) : collection.write_concern
+        WriteConcern.get(options[:write_concern]) :
+        collection.write_concern_with_session(session)
     end
 
     private
@@ -158,29 +172,46 @@ module Mongo
       {
         :db_name => database.name,
         :coll_name => collection.name,
-        :write_concern => write_concern,
+        :write_concern => write_concern(session),
         :ordered => ordered?,
         :operation_id => operation_id,
         :bypass_document_validation => !!options[:bypass_document_validation],
+        :max_time_ms => options[:max_time_ms],
         :options => options,
         :id_generator => client.options[:id_generator],
         :session => session
       }
     end
 
-    def execute_operation(name, values, server, operation_id, result_combiner, session, txn_num = nil)
-      raise Error::UnsupportedCollation.new if op_combiner.has_collation && !server.features.collation_enabled?
-      raise Error::UnsupportedArrayFilters.new if op_combiner.has_array_filters && !server.features.array_filters_enabled?
-      begin
-        if values.size > server.max_write_batch_size
-          split_execute(name, values, server, operation_id, result_combiner, session, txn_num)
+    def execute_operation(name, values, connection, context, operation_id, result_combiner, session, txn_num = nil)
+      validate_collation!(connection)
+      validate_array_filters!(connection)
+      validate_hint!(connection)
+
+      unpin_maybe(session) do
+        if values.size > connection.description.max_write_batch_size
+          split_execute(name, values, connection, context, operation_id, result_combiner, session, txn_num)
         else
-          result = send(name, values, server, operation_id, session, txn_num)
-          result_combiner.combine!(result, values.size)
+          result = send(name, values, connection, context, operation_id, session, txn_num)
+
+          add_server_diagnostics(connection) do
+            add_error_labels(connection, context) do
+              result_combiner.combine!(result, values.size)
+            end
+          end
         end
-      rescue Error::MaxBSONSize, Error::MaxMessageSize => e
-        raise e if values.size <= 1
-        split_execute(name, values, server, operation_id, result_combiner, session, txn_num)
+      end
+    # With OP_MSG (3.6+ servers), the size of each section in the message
+    # is independently capped at 16m and each bulk operation becomes
+    # its own section. The size of the entire bulk write is limited to 48m.
+    # With OP_QUERY (pre-3.6 servers), the entire bulk write is sent as a
+    # single document and is thus subject to the 16m document size limit.
+    # This means the splits differ between pre-3.6 and 3.6+ servers, with
+    # 3.6+ servers being able to split less.
+    rescue Error::MaxBSONSize, Error::MaxMessageSize => e
+      raise e if values.size <= 1
+      unpin_maybe(session) do
+        split_execute(name, values, connection, context, operation_id, result_combiner, session, txn_num)
       end
     end
 
@@ -188,37 +219,71 @@ module Mongo
       @op_combiner ||= ordered? ? OrderedCombiner.new(requests) : UnorderedCombiner.new(requests)
     end
 
-    def split_execute(name, values, server, operation_id, result_combiner, session, txn_num)
-      execute_operation(name, values.shift(values.size / 2), server, operation_id, result_combiner, session, txn_num)
+    def split_execute(name, values, connection, context, operation_id, result_combiner, session, txn_num)
+      execute_operation(name, values.shift(values.size / 2), connection, context, operation_id, result_combiner, session, txn_num)
 
       txn_num = session.next_txn_num if txn_num
-      execute_operation(name, values, server, operation_id, result_combiner, session, txn_num)
+      execute_operation(name, values, connection, context, operation_id, result_combiner, session, txn_num)
     end
 
-    def delete_one(documents, server, operation_id, session, txn_num)
+    def delete_one(documents, connection, context, operation_id, session, txn_num)
+      QueryCache.clear_namespace(collection.namespace)
+
       spec = base_spec(operation_id, session).merge(:deletes => documents, :txn_num => txn_num)
-      Operation::Delete.new(spec).bulk_execute(server)
+      Operation::Delete.new(spec).bulk_execute(connection, context: context)
     end
 
-    def delete_many(documents, server, operation_id, session, txn_num)
+    def delete_many(documents, connection, context, operation_id, session, txn_num)
+      QueryCache.clear_namespace(collection.namespace)
+
       spec = base_spec(operation_id, session).merge(:deletes => documents)
-      Operation::Delete.new(spec).bulk_execute(server)
+      Operation::Delete.new(spec).bulk_execute(connection, context: context)
     end
 
-    def insert_one(documents, server, operation_id, session, txn_num)
+    def insert_one(documents, connection, context, operation_id, session, txn_num)
+      QueryCache.clear_namespace(collection.namespace)
+
       spec = base_spec(operation_id, session).merge(:documents => documents, :txn_num => txn_num)
-      Operation::Insert.new(spec).bulk_execute(server)
+      Operation::Insert.new(spec).bulk_execute(connection, context: context)
     end
 
-    def update_one(documents, server, operation_id, session, txn_num)
+    def update_one(documents, connection, context, operation_id, session, txn_num)
+      QueryCache.clear_namespace(collection.namespace)
+
       spec = base_spec(operation_id, session).merge(:updates => documents, :txn_num => txn_num)
-      Operation::Update.new(spec).bulk_execute(server)
+      Operation::Update.new(spec).bulk_execute(connection, context: context)
     end
     alias :replace_one :update_one
 
-    def update_many(documents, server, operation_id, session, txn_num)
+    def update_many(documents, connection, context, operation_id, session, txn_num)
+      QueryCache.clear_namespace(collection.namespace)
+
       spec = base_spec(operation_id, session).merge(:updates => documents)
-      Operation::Update.new(spec).bulk_execute(server)
+      Operation::Update.new(spec).bulk_execute(connection, context: context)
+    end
+
+    private
+
+    def validate_collation!(connection)
+      if op_combiner.has_collation? && !connection.features.collation_enabled?
+        raise Error::UnsupportedCollation.new
+      end
+    end
+
+    def validate_array_filters!(connection)
+      if op_combiner.has_array_filters? && !connection.features.array_filters_enabled?
+        raise Error::UnsupportedArrayFilters.new
+      end
+    end
+
+    def validate_hint!(connection)
+      if op_combiner.has_hint?
+        if write_concern && !write_concern.acknowledged?
+          raise Error::UnsupportedOption.hint_error(unacknowledged_write: true)
+        elsif !connection.features.update_delete_option_validation_enabled?
+          raise Error::UnsupportedOption.hint_error
+        end
+      end
     end
   end
 end