mongo 2.13.0 → 2.15.0.alpha

data/lib/mongo/auth/user/view.rb

@@ -44,13 +44,13 @@ module Mongo
         # @since 2.0.0
         def create(user_or_name, options = {})
           user = generate(user_or_name, options)
-          client.send(:with_session, options) do |session|
+          execute_operation(options) do |session|
             Operation::CreateUser.new(
               user: user,
               db_name: database.name,
               session: session,
               write_concern: options[:write_concern] && WriteConcern.get(options[:write_concern]),
-            ).execute(next_primary(nil, session), client: client)
+            )
           end
         end
 
@@ -81,13 +81,13 @@ module Mongo
         #
         # @since 2.0.0
         def remove(name, options = {})
-          client.send(:with_session, options) do |session|
+          execute_operation(options) do |session|
             Operation::RemoveUser.new(
               user_name: name,
               db_name: database.name,
               session: session,
               write_concern: options[:write_concern] && WriteConcern.get(options[:write_concern]),
-            ).execute(next_primary(nil, session), client: client)
+            )
           end
         end
 
@@ -106,14 +106,14 @@ module Mongo
         #
         # @since 2.0.0
         def update(user_or_name, options = {})
-          client.send(:with_session, options) do |session|
-            user = generate(user_or_name, options)
+          user = generate(user_or_name, options)
+          execute_operation(options) do |session|
             Operation::UpdateUser.new(
               user: user,
               db_name: database.name,
               session: session,
               write_concern: options[:write_concern] && WriteConcern.get(options[:write_concern]),
-            ).execute(next_primary(nil, session), client: client)
+            )
           end
         end
 
@@ -137,18 +137,25 @@ module Mongo
         private
 
         def user_query(name, options = {})
-          client.send(:with_session, options) do |session|
+          execute_operation(options) do |session|
             Operation::UsersInfo.new(
               user_name: name,
               db_name: database.name,
               session: session
-            ).execute(next_primary(nil, session), client: client)
+            )
           end
         end
 
         def generate(user, options)
           user.is_a?(String) ? Auth::User.new({ user: user }.merge(options)) : user
         end
+
+        def execute_operation(options)
+          client.send(:with_session, options) do |session|
+            op = yield session
+            op.execute(next_primary(nil, session), context: Operation::Context.new(client: client, session: session))
+          end
+        end
       end
     end
   end

data/lib/mongo/auth/x509/conversation.rb

@@ -34,34 +34,13 @@ module Mongo
         # @param [ Server::Connection ] connection The connection being
         #   authenticated.
         #
-        # @return [ Protocol::Query ] The first X.509 conversation message.
+        # @return [ Protocol::Message ] The first X.509 conversation message.
         #
         # @since 2.0.0
         def start(connection)
-          login = client_first_document
-          if connection && connection.features.op_msg_enabled?
-            selector = login
-            # The only valid database for X.509 authentication is $external.
-            if user.auth_source != '$external'
-              user_name_msg = if user.name
-                " #{user.name}"
-              else
-                ''
-              end
-              raise Auth::InvalidConfiguration, "User#{user_name_msg} specifies auth source '#{user.auth_source}', but the only valid auth source for X.509 is '$external'"
-            end
-            selector[Protocol::Msg::DATABASE_IDENTIFIER] = '$external'
-            cluster_time = connection.mongos? && connection.cluster_time
-            selector[Operation::CLUSTER_TIME] = cluster_time if cluster_time
-            Protocol::Msg.new([], {}, selector)
-          else
-            Protocol::Query.new(
-              Auth::EXTERNAL,
-              Database::COMMAND,
-              login,
-              limit: -1
-            )
-          end
+          validate_external_auth_source
+          selector = client_first_document
+          build_message(connection, '$external', selector)
         end
 
         # Returns the hash to provide to the server in the handshake

data/lib/mongo/background_thread.rb

@@ -20,6 +20,17 @@ module Mongo
   # compatibility reasons. However using these methods outside of the driver
   # is deprecated.
   #
+  # @note Do not start or stop background threads in finalizers. See
+  #   https://jira.mongodb.org/browse/RUBY-2453 and
+  #   https://bugs.ruby-lang.org/issues/16288. When interpreter exits,
+  #   background threads are stopped first and finalizers are invoked next,
+  #   and MRI's internal data structures are basically corrupt at this point
+  #   if threads are being referenced. Prior to interpreter shutdown this
+  #   means threads cannot be stopped by objects going out of scope, but
+  #   most likely the threads hold references to said objects anyway if
+  #   work is being performed thus the objects wouldn't go out of scope in
+  #   the first place.
+  #
   # @api private
   module BackgroundThread
     include Loggable

data/lib/mongo/bulk_write.rb

@@ -58,6 +58,7 @@ module Mongo
       operations = op_combiner.combine
 
       client.send(:with_session, @options) do |session|
+        context = Operation::Context.new(client: client, session: session)
         operations.each do |operation|
           if single_statement?(operation)
             write_concern = write_concern(session)
@@ -67,6 +68,7 @@ module Mongo
                   operation.keys.first,
                   operation.values.flatten,
                   connection,
+                  context,
                   operation_id,
                   result_combiner,
                   session,
@@ -80,6 +82,7 @@ module Mongo
                   operation.keys.first,
                   operation.values.flatten,
                   connection,
+                  context,
                   operation_id,
                   result_combiner,
                   session)
@@ -177,28 +180,35 @@ module Mongo
       }
     end
 
-    def execute_operation(name, values, connection, operation_id, result_combiner, session, txn_num = nil)
+    def execute_operation(name, values, connection, context, operation_id, result_combiner, session, txn_num = nil)
       validate_collation!(connection)
       validate_array_filters!(connection)
       validate_hint!(connection)
 
       unpin_maybe(session) do
         if values.size > connection.description.max_write_batch_size
-          split_execute(name, values, connection, operation_id, result_combiner, session, txn_num)
+          split_execute(name, values, connection, context, operation_id, result_combiner, session, txn_num)
         else
-          result = send(name, values, connection, operation_id, session, txn_num)
+          result = send(name, values, connection, context, operation_id, session, txn_num)
 
           add_server_diagnostics(connection) do
-            add_error_labels(client, connection, session) do
+            add_error_labels(connection, context) do
               result_combiner.combine!(result, values.size)
             end
           end
         end
       end
+    # With OP_MSG (3.6+ servers), the size of each section in the message
+    # is independently capped at 16m and each bulk operation becomes
+    # its own section. The size of the entire bulk write is limited to 48m.
+    # With OP_QUERY (pre-3.6 servers), the entire bulk write is sent as a
+    # single document and is thus subject to the 16m document size limit.
+    # This means the splits differ between pre-3.6 and 3.6+ servers, with
+    # 3.6+ servers being able to split less.
     rescue Error::MaxBSONSize, Error::MaxMessageSize => e
       raise e if values.size <= 1
       unpin_maybe(session) do
-        split_execute(name, values, connection, operation_id, result_combiner, session, txn_num)
+        split_execute(name, values, connection, context, operation_id, result_combiner, session, txn_num)
       end
     end
 
@@ -206,37 +216,47 @@ module Mongo
       @op_combiner ||= ordered? ? OrderedCombiner.new(requests) : UnorderedCombiner.new(requests)
     end
 
-    def split_execute(name, values, connection, operation_id, result_combiner, session, txn_num)
-      execute_operation(name, values.shift(values.size / 2), connection, operation_id, result_combiner, session, txn_num)
+    def split_execute(name, values, connection, context, operation_id, result_combiner, session, txn_num)
+      execute_operation(name, values.shift(values.size / 2), connection, context, operation_id, result_combiner, session, txn_num)
 
       txn_num = session.next_txn_num if txn_num
-      execute_operation(name, values, connection, operation_id, result_combiner, session, txn_num)
+      execute_operation(name, values, connection, context, operation_id, result_combiner, session, txn_num)
     end
 
-    def delete_one(documents, connection, operation_id, session, txn_num)
+    def delete_one(documents, connection, context, operation_id, session, txn_num)
+      QueryCache.clear_namespace(collection.namespace)
+
       spec = base_spec(operation_id, session).merge(:deletes => documents, :txn_num => txn_num)
-      Operation::Delete.new(spec).bulk_execute(connection, client: client)
+      Operation::Delete.new(spec).bulk_execute(connection, context: context)
     end
 
-    def delete_many(documents, connection, operation_id, session, txn_num)
+    def delete_many(documents, connection, context, operation_id, session, txn_num)
+      QueryCache.clear_namespace(collection.namespace)
+
       spec = base_spec(operation_id, session).merge(:deletes => documents)
-      Operation::Delete.new(spec).bulk_execute(connection, client: client)
+      Operation::Delete.new(spec).bulk_execute(connection, context: context)
     end
 
-    def insert_one(documents, connection, operation_id, session, txn_num)
+    def insert_one(documents, connection, context, operation_id, session, txn_num)
+      QueryCache.clear_namespace(collection.namespace)
+
       spec = base_spec(operation_id, session).merge(:documents => documents, :txn_num => txn_num)
-      Operation::Insert.new(spec).bulk_execute(connection, client: client)
+      Operation::Insert.new(spec).bulk_execute(connection, context: context)
     end
 
-    def update_one(documents, connection, operation_id, session, txn_num)
+    def update_one(documents, connection, context, operation_id, session, txn_num)
+      QueryCache.clear_namespace(collection.namespace)
+
       spec = base_spec(operation_id, session).merge(:updates => documents, :txn_num => txn_num)
-      Operation::Update.new(spec).bulk_execute(connection, client: client)
+      Operation::Update.new(spec).bulk_execute(connection, context: context)
     end
     alias :replace_one :update_one
 
-    def update_many(documents, connection, operation_id, session, txn_num)
+    def update_many(documents, connection, context, operation_id, session, txn_num)
+      QueryCache.clear_namespace(collection.namespace)
+
       spec = base_spec(operation_id, session).merge(:updates => documents)
-      Operation::Update.new(spec).bulk_execute(connection, client: client)
+      Operation::Update.new(spec).bulk_execute(connection, context: context)
     end
 
     private

data/lib/mongo/caching_cursor.rb

@@ -0,0 +1,74 @@
+# Copyright (C) 2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+
+  # A Cursor that attempts to load documents from memory first before hitting
+  # the database if the same query has already been executed.
+  #
+  # @api semiprivate
+  class CachingCursor < Cursor
+
+    # @return [ Array <BSON::Document> ] The cursor's cached documents.
+    # @api private
+    attr_reader :cached_docs
+
+    # We iterate over the cached documents if they exist already in the
+    # cursor otherwise proceed as normal.
+    #
+    # @example Iterate over the documents.
+    #   cursor.each do |doc|
+    #     # ...
+    #   end
+    def each
+      if @cached_docs
+        @cached_docs.each do |doc|
+          yield doc
+        end
+
+        unless closed?
+          # StopIteration raised by try_next ends this loop.
+          loop do
+            document = try_next
+            yield document if document
+          end
+        end
+      else
+        super
+      end
+    end
+
+    # Get a human-readable string representation of +Cursor+.
+    #
+    # @example Inspect the cursor.
+    #   cursor.inspect
+    #
+    # @return [ String ] A string representation of a +Cursor+ instance.
+    def inspect
+      "#<Mongo::CachingCursor:0x#{object_id} @view=#{@view.inspect}>"
+    end
+
+    # Acquires the next document for cursor iteration and then
+    # inserts that document in the @cached_docs array.
+    #
+    # @api private
+    def try_next
+      @cached_docs ||= []
+      document = super
+      @cached_docs << document if document
+
+      document
+    end
+  end
+end

data/lib/mongo/client.rb

@@ -85,6 +85,7 @@ module Mongo
       :retry_writes,
       :scan,
       :sdam_proc,
+      :server_api,
       :server_selection_timeout,
       :socket_timeout,
       :ssl,
@@ -101,6 +102,7 @@ module Mongo
       :ssl_verify,
       :ssl_verify_certificate,
       :ssl_verify_hostname,
+      :ssl_verify_ocsp_endpoint,
       :truncate_logs,
       :user,
       :wait_queue_timeout,
@@ -113,7 +115,16 @@ module Mongo
     # The compression algorithms supported by the driver.
     #
     # @since 2.5.0
-    VALID_COMPRESSORS = [ Mongo::Protocol::Compressed::ZLIB ].freeze
+    VALID_COMPRESSORS = [
+      Mongo::Protocol::Compressed::ZSTD,
+      Mongo::Protocol::Compressed::SNAPPY,
+      Mongo::Protocol::Compressed::ZLIB
+    ].freeze
+
+    # The known server API versions.
+    VALID_SERVER_API_VERSIONS = %w(
+      1
+    ).freeze
 
     # @return [ Mongo::Cluster ] cluster The cluster of servers for the client.
     attr_reader :cluster
@@ -222,7 +233,7 @@ module Mongo
     # @option options [ Array<String> ] :compressors A list of potential
     #   compressors to use, in order of preference. The driver chooses the
     #   first compressor that is also supported by the server. Currently the
-    #   driver only supports 'zlib'.
+    #   driver only supports 'zstd, 'snappy' and 'zlib'.
     # @option options [ true | false ] :direct_connection Whether to connect
     #   directly to the specified seed, bypassing topology discovery. Exactly
     #   one seed must be provided.
@@ -311,11 +322,16 @@ module Mongo
     #   in particular the cluster is nil at this time. sdam_proc should
     #   limit itself to calling #subscribe and #unsubscribe methods on the
     #   client only.
+    # @option options [ Hash ] :server_api The requested server API version.
+    #   This hash can have the following items:
+    #   - *:version* -- string
+    #   - *:strict* -- boolean
+    #   - *:deprecation_errors* -- boolean
     # @option options [ Integer ] :server_selection_timeout The timeout in seconds
     #   for selecting a server for an operation.
     # @option options [ Float ] :socket_timeout The timeout, in seconds, to
     #   execute operations on a socket.
-    # @option options [ true, false ] :ssl Whether to use SSL.
+    # @option options [ true, false ] :ssl Whether to use TLS.
     # @option options [ String ] :ssl_ca_cert The file containing concatenated
     #   certificate authority certificates used to validate certs passed from the
     #   other end of the connection. Intermediate certificates should NOT be
@@ -462,11 +478,42 @@ module Mongo
         @srv_records = nil
       end
 
+      options = self.class.canonicalize_ruby_options(options)
+
+      # The server API version is specified to be a string.
+      # However, it is very annoying to always provide the number 1 as a string,
+      # therefore cast to the string type here.
+      if server_api = options[:server_api]
+        if server_api.is_a?(Hash)
+          server_api = Options::Redacted.new(server_api)
+          if (version = server_api[:version]).is_a?(Integer)
+            options[:server_api] = server_api.merge(version: version.to_s)
+          end
+        end
+      end
+
       # Special handling for sdam_proc as it is only used during client
       # construction
       sdam_proc = options.delete(:sdam_proc)
 
-      options = default_options(options).merge(options)
+      # For gssapi service_name, the default option is given in a hash
+      # (one level down from the top level).
+      merged_options = default_options(options)
+      options.each do |k, v|
+        default_v = merged_options[k]
+        if Hash === default_v
+          v = default_v.merge(v)
+        end
+        merged_options[k] = v
+      end
+      options = merged_options
+
+      options.keys.each do |k|
+        if options[k].nil?
+          options.delete(k)
+        end
+      end
+
       @options = validate_new_options!(options)
 =begin WriteConcern object support
       if @options[:write_concern].is_a?(WriteConcern::Base)
@@ -479,7 +526,9 @@ module Mongo
       validate_options!(addresses)
       validate_authentication_options!
 
-      @database = Database.new(self, @options[:database], @options)
+      database_options = @options.dup
+      database_options.delete(:server_api)
+      @database = Database.new(self, @options[:database], database_options)
 
       # Temporarily set monitoring so that event subscriptions can be
       # set up without there being a cluster
@@ -594,15 +643,12 @@ module Mongo
 
     # Get a summary of the client state.
     #
-    # @note This method is experimental and subject to change.
-    #
-    # @example Inspect the client.
-    #   client.summary
+    # @note The exact format and layout of the returned summary string is
+    #   not part of the driver's public API and may be changed at any time.
     #
-    # @return [ String ] Summary string.
+    # @return [ String ] The summary string.
     #
     # @since 2.7.0
-    # @api experimental
     def summary
       "#<Client cluster=#{cluster.summary}>"
     end
@@ -632,7 +678,7 @@ module Mongo
     #
     # @return [ BSON::Document ] The user-defined read preference.
     #   The document may have the following fields:
-    #   - *:read* -- read preference specified as a symbol; valid values are
+    #   - *:mode* -- read preference specified as a symbol; valid values are
     #     *:primary*, *:primary_preferred*, *:secondary*, *:secondary_preferred*
     #     and *:nearest*.
     #   - *:tag_sets* -- an array of hashes.
@@ -680,9 +726,9 @@ module Mongo
     # @return [ Mongo::Client ] A new client instance.
     #
     # @since 2.0.0
-    def with(new_options = Options::Redacted.new)
+    def with(new_options = nil)
       clone.tap do |client|
-        opts = client.update_options(new_options)
+        opts = client.update_options(new_options || Options::Redacted.new)
         Database.create(client)
         # We can't use the same cluster if some options that would affect it
         # have changed.
@@ -709,6 +755,8 @@ module Mongo
     def update_options(new_options)
       old_options = @options
 
+      new_options = self.class.canonicalize_ruby_options(new_options || {})
+
       validate_new_options!(new_options).tap do |opts|
         # Our options are frozen
         options = @options.dup
@@ -758,7 +806,6 @@ module Mongo
       options[:read_concern]
     end
 
-
     # Get the write concern for this client. If no option was provided, then a
     # default single server acknowledgement will be used.
     #
@@ -1011,6 +1058,23 @@ module Mongo
       end
     end
 
+    class << self
+      # Lowercases auth mechanism properties, if given, in the specified
+      # options, then converts the options to an instance of Options::Redacted.
+      #
+      # @api private
+      def canonicalize_ruby_options(options)
+        Options::Redacted.new(Hash[options.map do |k, v|
+          if k == :auth_mech_properties || k == 'auth_mech_properties'
+            if v
+              v = Hash[v.map { |pk, pv| [pk.downcase, pv] }]
+            end
+          end
+          [k, v]
+        end])
+      end
+    end
+
     private
 
     # Create a new encrypter object using the client's auto encryption options
@@ -1094,8 +1158,42 @@ module Mongo
     # The argument may contain a subset of options that the client will
     # eventually have; this method validates each of the provided options
     # but does not check for interactions between combinations of options.
-    def validate_new_options!(opts = Options::Redacted.new)
+    def validate_new_options!(opts)
       return Options::Redacted.new unless opts
+      if opts[:read_concern]
+        # Raise an error for non user-settable options
+        if opts[:read_concern][:after_cluster_time]
+          raise Mongo::Error::InvalidReadConcern.new(
+            'The after_cluster_time read_concern option cannot be specified by the user'
+          )
+        end
+
+        given_keys = opts[:read_concern].keys.map(&:to_s)
+        allowed_keys = ['level']
+        invalid_keys = given_keys - allowed_keys
+        # Warn that options are invalid but keep it and forward to the server
+        unless invalid_keys.empty?
+          log_warn("Read concern has invalid keys: #{invalid_keys.join(',')}.")
+        end
+      end
+
+      if server_api = opts[:server_api]
+        unless server_api.is_a?(Hash)
+          raise ArgumentError, ":server_api value must be a hash: #{server_api}"
+        end
+
+        extra_keys = server_api.keys - %w(version strict deprecation_errors)
+        unless extra_keys.empty?
+          raise ArgumentError, "Unknown keys under :server_api: #{extra_keys.map(&:inspect).join(', ')}"
+        end
+
+        if version = server_api[:version]
+          unless VALID_SERVER_API_VERSIONS.include?(version)
+            raise ArgumentError, "Unknown server API version: #{version}"
+          end
+        end
+      end
+
       Lint.validate_underscore_read_preference(opts[:read])
       Lint.validate_read_concern_option(opts[:read_concern])
       opts.each.inject(Options::Redacted.new) do |_options, (k, v)|
@@ -1105,6 +1203,15 @@ module Mongo
           validate_read!(key, opts)
           if key == :compressors
             compressors = valid_compressors(v)
+
+            if compressors.include?('snappy')
+              validate_snappy_compression!
+            end
+
+            if compressors.include?('zstd')
+              validate_zstd_compression!
+            end
+
             _options[key] = compressors unless compressors.empty?
           else
             _options[key] = v
@@ -1261,11 +1368,30 @@ module Mongo
                        "This compressor will not be used.")
           false
         else
+
           true
         end
       end
     end
 
+    def validate_snappy_compression!
+      return if defined?(Snappy)
+      require 'snappy'
+    rescue LoadError => e
+      raise Error::UnmetDependency, "Cannot enable snappy compression because the snappy gem " \
+        "has not been installed. Add \"gem 'snappy'\" to your Gemfile and run " \
+        "\"bundle install\" to install the gem. (#{e.class}: #{e})"
+    end
+
+    def validate_zstd_compression!
+      return if defined?(Zstd)
+      require 'zstd-ruby'
+    rescue LoadError => e
+      raise Error::UnmetDependency, "Cannot enable zstd compression because the zstd-ruby gem " \
+        "has not been installed. Add \"gem 'zstd-ruby'\" to your Gemfile and run " \
+        "\"bundle install\" to install the gem. (#{e.class}: #{e})"
+    end
+
     def validate_max_min_pool_size!(option, opts)
       if option == :min_pool_size && opts[:min_pool_size]
         max = opts[:max_pool_size] || Server::ConnectionPool::DEFAULT_MAX_SIZE