mongo 2.13.2 → 2.15.0

data/lib/mongo/background_thread.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2019-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,6 +23,17 @@ module Mongo
   # compatibility reasons. However using these methods outside of the driver
   # is deprecated.
   #
+  # @note Do not start or stop background threads in finalizers. See
+  #   https://jira.mongodb.org/browse/RUBY-2453 and
+  #   https://bugs.ruby-lang.org/issues/16288. When interpreter exits,
+  #   background threads are stopped first and finalizers are invoked next,
+  #   and MRI's internal data structures are basically corrupt at this point
+  #   if threads are being referenced. Prior to interpreter shutdown this
+  #   means threads cannot be stopped by objects going out of scope, but
+  #   most likely the threads hold references to said objects anyway if
+  #   work is being performed thus the objects wouldn't go out of scope in
+  #   the first place.
+  #
   # @api private
   module BackgroundThread
     include Loggable
@@ -115,7 +129,7 @@ module Mongo
       # However, we do not want to wait indefinitely because in theory
       # a background thread could be performing, say, network I/O and if
       # the network is no longer available that could take a long time.
-      start_time = Time.now
+      start_time = Utils.monotonic_time
       ([0.1, 0.15] + [0.2] * 5 + [0.3] * 20).each do |interval|
         begin
           Timeout.timeout(interval) do
@@ -129,7 +143,7 @@ module Mongo
       # Some driver objects can be reconnected, for backwards compatibiilty
       # reasons. Clear the thread instance variable to support this cleanly.
       if @thread.alive?
-        log_warn("Failed to stop the background thread in #{self} in #{(Time.now - start_time).to_i} seconds: #{@thread.inspect} (thread status: #{@thread.status})")
+        log_warn("Failed to stop the background thread in #{self} in #{(Utils.monotonic_time - start_time).to_i} seconds: #{@thread.inspect} (thread status: #{@thread.status})")
         # On JRuby the thread may be stuck in aborting state
         # seemingly indefinitely. If the thread is aborting, consider it dead
         # for our purposes (we will create a new thread if needed, and

data/lib/mongo/bson.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

data/lib/mongo/bulk_write.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -58,6 +61,7 @@ module Mongo
       operations = op_combiner.combine
 
       client.send(:with_session, @options) do |session|
+        context = Operation::Context.new(client: client, session: session)
         operations.each do |operation|
           if single_statement?(operation)
             write_concern = write_concern(session)
@@ -67,6 +71,7 @@ module Mongo
                   operation.keys.first,
                   operation.values.flatten,
                   connection,
+                  context,
                   operation_id,
                   result_combiner,
                   session,
@@ -80,6 +85,7 @@ module Mongo
                   operation.keys.first,
                   operation.values.flatten,
                   connection,
+                  context,
                   operation_id,
                   result_combiner,
                   session)
@@ -177,28 +183,35 @@ module Mongo
       }
     end
 
-    def execute_operation(name, values, connection, operation_id, result_combiner, session, txn_num = nil)
+    def execute_operation(name, values, connection, context, operation_id, result_combiner, session, txn_num = nil)
       validate_collation!(connection)
       validate_array_filters!(connection)
       validate_hint!(connection)
 
       unpin_maybe(session) do
         if values.size > connection.description.max_write_batch_size
-          split_execute(name, values, connection, operation_id, result_combiner, session, txn_num)
+          split_execute(name, values, connection, context, operation_id, result_combiner, session, txn_num)
         else
-          result = send(name, values, connection, operation_id, session, txn_num)
+          result = send(name, values, connection, context, operation_id, session, txn_num)
 
           add_server_diagnostics(connection) do
-            add_error_labels(client, connection, session) do
+            add_error_labels(connection, context) do
               result_combiner.combine!(result, values.size)
             end
           end
         end
       end
+    # With OP_MSG (3.6+ servers), the size of each section in the message
+    # is independently capped at 16m and each bulk operation becomes
+    # its own section. The size of the entire bulk write is limited to 48m.
+    # With OP_QUERY (pre-3.6 servers), the entire bulk write is sent as a
+    # single document and is thus subject to the 16m document size limit.
+    # This means the splits differ between pre-3.6 and 3.6+ servers, with
+    # 3.6+ servers being able to split less.
     rescue Error::MaxBSONSize, Error::MaxMessageSize => e
       raise e if values.size <= 1
       unpin_maybe(session) do
-        split_execute(name, values, connection, operation_id, result_combiner, session, txn_num)
+        split_execute(name, values, connection, context, operation_id, result_combiner, session, txn_num)
       end
     end
 
@@ -206,37 +219,47 @@ module Mongo
       @op_combiner ||= ordered? ? OrderedCombiner.new(requests) : UnorderedCombiner.new(requests)
     end
 
-    def split_execute(name, values, connection, operation_id, result_combiner, session, txn_num)
-      execute_operation(name, values.shift(values.size / 2), connection, operation_id, result_combiner, session, txn_num)
+    def split_execute(name, values, connection, context, operation_id, result_combiner, session, txn_num)
+      execute_operation(name, values.shift(values.size / 2), connection, context, operation_id, result_combiner, session, txn_num)
 
       txn_num = session.next_txn_num if txn_num
-      execute_operation(name, values, connection, operation_id, result_combiner, session, txn_num)
+      execute_operation(name, values, connection, context, operation_id, result_combiner, session, txn_num)
     end
 
-    def delete_one(documents, connection, operation_id, session, txn_num)
+    def delete_one(documents, connection, context, operation_id, session, txn_num)
+      QueryCache.clear_namespace(collection.namespace)
+
       spec = base_spec(operation_id, session).merge(:deletes => documents, :txn_num => txn_num)
-      Operation::Delete.new(spec).bulk_execute(connection, client: client)
+      Operation::Delete.new(spec).bulk_execute(connection, context: context)
     end
 
-    def delete_many(documents, connection, operation_id, session, txn_num)
+    def delete_many(documents, connection, context, operation_id, session, txn_num)
+      QueryCache.clear_namespace(collection.namespace)
+
       spec = base_spec(operation_id, session).merge(:deletes => documents)
-      Operation::Delete.new(spec).bulk_execute(connection, client: client)
+      Operation::Delete.new(spec).bulk_execute(connection, context: context)
     end
 
-    def insert_one(documents, connection, operation_id, session, txn_num)
+    def insert_one(documents, connection, context, operation_id, session, txn_num)
+      QueryCache.clear_namespace(collection.namespace)
+
       spec = base_spec(operation_id, session).merge(:documents => documents, :txn_num => txn_num)
-      Operation::Insert.new(spec).bulk_execute(connection, client: client)
+      Operation::Insert.new(spec).bulk_execute(connection, context: context)
     end
 
-    def update_one(documents, connection, operation_id, session, txn_num)
+    def update_one(documents, connection, context, operation_id, session, txn_num)
+      QueryCache.clear_namespace(collection.namespace)
+
       spec = base_spec(operation_id, session).merge(:updates => documents, :txn_num => txn_num)
-      Operation::Update.new(spec).bulk_execute(connection, client: client)
+      Operation::Update.new(spec).bulk_execute(connection, context: context)
     end
     alias :replace_one :update_one
 
-    def update_many(documents, connection, operation_id, session, txn_num)
+    def update_many(documents, connection, context, operation_id, session, txn_num)
+      QueryCache.clear_namespace(collection.namespace)
+
       spec = base_spec(operation_id, session).merge(:updates => documents)
-      Operation::Update.new(spec).bulk_execute(connection, client: client)
+      Operation::Update.new(spec).bulk_execute(connection, context: context)
     end
 
     private

data/lib/mongo/bulk_write/combineable.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

data/lib/mongo/bulk_write/ordered_combiner.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

data/lib/mongo/bulk_write/result.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

data/lib/mongo/bulk_write/result_combiner.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

data/lib/mongo/bulk_write/transformable.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

data/lib/mongo/bulk_write/unordered_combiner.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

data/lib/mongo/bulk_write/validatable.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

data/lib/mongo/caching_cursor.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+
+  # A Cursor that attempts to load documents from memory first before hitting
+  # the database if the same query has already been executed.
+  #
+  # @api semiprivate
+  class CachingCursor < Cursor
+
+    # @return [ Array <BSON::Document> ] The cursor's cached documents.
+    # @api private
+    attr_reader :cached_docs
+
+    # We iterate over the cached documents if they exist already in the
+    # cursor otherwise proceed as normal.
+    #
+    # @example Iterate over the documents.
+    #   cursor.each do |doc|
+    #     # ...
+    #   end
+    def each
+      if @cached_docs
+        @cached_docs.each do |doc|
+          yield doc
+        end
+
+        unless closed?
+          # StopIteration raised by try_next ends this loop.
+          loop do
+            document = try_next
+            yield document if document
+          end
+        end
+      else
+        super
+      end
+    end
+
+    # Get a human-readable string representation of +Cursor+.
+    #
+    # @example Inspect the cursor.
+    #   cursor.inspect
+    #
+    # @return [ String ] A string representation of a +Cursor+ instance.
+    def inspect
+      "#<Mongo::CachingCursor:0x#{object_id} @view=#{@view.inspect}>"
+    end
+
+    # Acquires the next document for cursor iteration and then
+    # inserts that document in the @cached_docs array.
+    #
+    # @api private
+    def try_next
+      @cached_docs ||= []
+      document = super
+      @cached_docs << document if document
+
+      document
+    end
+  end
+end

data/lib/mongo/client.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the 'License');
@@ -85,6 +88,7 @@ module Mongo
       :retry_writes,
       :scan,
       :sdam_proc,
+      :server_api,
       :server_selection_timeout,
       :socket_timeout,
       :ssl,
@@ -101,6 +105,7 @@ module Mongo
       :ssl_verify,
       :ssl_verify_certificate,
       :ssl_verify_hostname,
+      :ssl_verify_ocsp_endpoint,
       :truncate_logs,
       :user,
       :wait_queue_timeout,
@@ -113,7 +118,16 @@ module Mongo
     # The compression algorithms supported by the driver.
     #
     # @since 2.5.0
-    VALID_COMPRESSORS = [ Mongo::Protocol::Compressed::ZLIB ].freeze
+    VALID_COMPRESSORS = [
+      Mongo::Protocol::Compressed::ZSTD,
+      Mongo::Protocol::Compressed::SNAPPY,
+      Mongo::Protocol::Compressed::ZLIB
+    ].freeze
+
+    # The known server API versions.
+    VALID_SERVER_API_VERSIONS = %w(
+      1
+    ).freeze
 
     # @return [ Mongo::Cluster ] cluster The cluster of servers for the client.
     attr_reader :cluster
@@ -222,7 +236,7 @@ module Mongo
     # @option options [ Array<String> ] :compressors A list of potential
     #   compressors to use, in order of preference. The driver chooses the
     #   first compressor that is also supported by the server. Currently the
-    #   driver only supports 'zlib'.
+    #   driver only supports 'zstd, 'snappy' and 'zlib'.
     # @option options [ true | false ] :direct_connection Whether to connect
     #   directly to the specified seed, bypassing topology discovery. Exactly
     #   one seed must be provided.
@@ -234,7 +248,7 @@ module Mongo
     #   attempt a connection.
     # @option options [ String ] :database The database to connect to.
     # @option options [ Float ] :heartbeat_frequency The interval, in seconds,
-    #   for the server monitor to refresh its description via ismaster.
+    #   for the server monitor to refresh its description via hello.
     # @option options [ Object ] :id_generator A custom object to generate ids
     #   for documents. Must respond to #generate.
     # @option options [ Integer ] :local_threshold The local threshold boundary
@@ -311,11 +325,16 @@ module Mongo
     #   in particular the cluster is nil at this time. sdam_proc should
     #   limit itself to calling #subscribe and #unsubscribe methods on the
     #   client only.
+    # @option options [ Hash ] :server_api The requested server API version.
+    #   This hash can have the following items:
+    #   - *:version* -- string
+    #   - *:strict* -- boolean
+    #   - *:deprecation_errors* -- boolean
     # @option options [ Integer ] :server_selection_timeout The timeout in seconds
     #   for selecting a server for an operation.
     # @option options [ Float ] :socket_timeout The timeout, in seconds, to
     #   execute operations on a socket.
-    # @option options [ true, false ] :ssl Whether to use SSL.
+    # @option options [ true, false ] :ssl Whether to use TLS.
     # @option options [ String ] :ssl_ca_cert The file containing concatenated
     #   certificate authority certificates used to validate certs passed from the
     #   other end of the connection. Intermediate certificates should NOT be
@@ -462,6 +481,20 @@ module Mongo
         @srv_records = nil
       end
 
+      options = self.class.canonicalize_ruby_options(options)
+
+      # The server API version is specified to be a string.
+      # However, it is very annoying to always provide the number 1 as a string,
+      # therefore cast to the string type here.
+      if server_api = options[:server_api]
+        if server_api.is_a?(Hash)
+          server_api = Options::Redacted.new(server_api)
+          if (version = server_api[:version]).is_a?(Integer)
+            options[:server_api] = server_api.merge(version: version.to_s)
+          end
+        end
+      end
+
       # Special handling for sdam_proc as it is only used during client
       # construction
       sdam_proc = options.delete(:sdam_proc)
@@ -478,6 +511,12 @@ module Mongo
       end
       options = merged_options
 
+      options.keys.each do |k|
+        if options[k].nil?
+          options.delete(k)
+        end
+      end
+
       @options = validate_new_options!(options)
 =begin WriteConcern object support
       if @options[:write_concern].is_a?(WriteConcern::Base)
@@ -490,7 +529,9 @@ module Mongo
       validate_options!(addresses)
       validate_authentication_options!
 
-      @database = Database.new(self, @options[:database], @options)
+      database_options = @options.dup
+      database_options.delete(:server_api)
+      @database = Database.new(self, @options[:database], database_options)
 
       # Temporarily set monitoring so that event subscriptions can be
       # set up without there being a cluster
@@ -605,15 +646,12 @@ module Mongo
 
     # Get a summary of the client state.
     #
-    # @note This method is experimental and subject to change.
+    # @note The exact format and layout of the returned summary string is
+    #   not part of the driver's public API and may be changed at any time.
     #
-    # @example Inspect the client.
-    #   client.summary
-    #
-    # @return [ String ] Summary string.
+    # @return [ String ] The summary string.
     #
     # @since 2.7.0
-    # @api experimental
     def summary
       "#<Client cluster=#{cluster.summary}>"
     end
@@ -691,9 +729,9 @@ module Mongo
     # @return [ Mongo::Client ] A new client instance.
     #
     # @since 2.0.0
-    def with(new_options = Options::Redacted.new)
+    def with(new_options = nil)
       clone.tap do |client|
-        opts = client.update_options(new_options)
+        opts = client.update_options(new_options || Options::Redacted.new)
         Database.create(client)
         # We can't use the same cluster if some options that would affect it
         # have changed.
@@ -720,6 +758,8 @@ module Mongo
     def update_options(new_options)
       old_options = @options
 
+      new_options = self.class.canonicalize_ruby_options(new_options || {})
+
       validate_new_options!(new_options).tap do |opts|
         # Our options are frozen
         options = @options.dup
@@ -769,7 +809,6 @@ module Mongo
       options[:read_concern]
     end
 
-
     # Get the write concern for this client. If no option was provided, then a
     # default single server acknowledgement will be used.
     #
@@ -1022,6 +1061,23 @@ module Mongo
       end
     end
 
+    class << self
+      # Lowercases auth mechanism properties, if given, in the specified
+      # options, then converts the options to an instance of Options::Redacted.
+      #
+      # @api private
+      def canonicalize_ruby_options(options)
+        Options::Redacted.new(Hash[options.map do |k, v|
+          if k == :auth_mech_properties || k == 'auth_mech_properties'
+            if v
+              v = Hash[v.map { |pk, pv| [pk.downcase, pv] }]
+            end
+          end
+          [k, v]
+        end])
+      end
+    end
+
     private
 
     # Create a new encrypter object using the client's auto encryption options
@@ -1105,8 +1161,42 @@ module Mongo
     # The argument may contain a subset of options that the client will
     # eventually have; this method validates each of the provided options
     # but does not check for interactions between combinations of options.
-    def validate_new_options!(opts = Options::Redacted.new)
+    def validate_new_options!(opts)
       return Options::Redacted.new unless opts
+      if opts[:read_concern]
+        # Raise an error for non user-settable options
+        if opts[:read_concern][:after_cluster_time]
+          raise Mongo::Error::InvalidReadConcern.new(
+            'The after_cluster_time read_concern option cannot be specified by the user'
+          )
+        end
+
+        given_keys = opts[:read_concern].keys.map(&:to_s)
+        allowed_keys = ['level']
+        invalid_keys = given_keys - allowed_keys
+        # Warn that options are invalid but keep it and forward to the server
+        unless invalid_keys.empty?
+          log_warn("Read concern has invalid keys: #{invalid_keys.join(',')}.")
+        end
+      end
+
+      if server_api = opts[:server_api]
+        unless server_api.is_a?(Hash)
+          raise ArgumentError, ":server_api value must be a hash: #{server_api}"
+        end
+
+        extra_keys = server_api.keys - %w(version strict deprecation_errors)
+        unless extra_keys.empty?
+          raise ArgumentError, "Unknown keys under :server_api: #{extra_keys.map(&:inspect).join(', ')}"
+        end
+
+        if version = server_api[:version]
+          unless VALID_SERVER_API_VERSIONS.include?(version)
+            raise ArgumentError, "Unknown server API version: #{version}"
+          end
+        end
+      end
+
       Lint.validate_underscore_read_preference(opts[:read])
       Lint.validate_read_concern_option(opts[:read_concern])
       opts.each.inject(Options::Redacted.new) do |_options, (k, v)|
@@ -1116,6 +1206,15 @@ module Mongo
           validate_read!(key, opts)
           if key == :compressors
             compressors = valid_compressors(v)
+
+            if compressors.include?('snappy')
+              validate_snappy_compression!
+            end
+
+            if compressors.include?('zstd')
+              validate_zstd_compression!
+            end
+
             _options[key] = compressors unless compressors.empty?
           else
             _options[key] = v
@@ -1272,11 +1371,30 @@ module Mongo
                        "This compressor will not be used.")
           false
         else
+
           true
         end
       end
     end
 
+    def validate_snappy_compression!
+      return if defined?(Snappy)
+      require 'snappy'
+    rescue LoadError => e
+      raise Error::UnmetDependency, "Cannot enable snappy compression because the snappy gem " \
+        "has not been installed. Add \"gem 'snappy'\" to your Gemfile and run " \
+        "\"bundle install\" to install the gem. (#{e.class}: #{e})"
+    end
+
+    def validate_zstd_compression!
+      return if defined?(Zstd)
+      require 'zstd-ruby'
+    rescue LoadError => e
+      raise Error::UnmetDependency, "Cannot enable zstd compression because the zstd-ruby gem " \
+        "has not been installed. Add \"gem 'zstd-ruby'\" to your Gemfile and run " \
+        "\"bundle install\" to install the gem. (#{e.class}: #{e})"
+    end
+
     def validate_max_min_pool_size!(option, opts)
       if option == :min_pool_size && opts[:min_pool_size]
         max = opts[:max_pool_size] || Server::ConnectionPool::DEFAULT_MAX_SIZE