mongo 2.14.0 → 2.18.1

data/lib/mongo/query_cache.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2020 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -176,7 +179,8 @@ module Mongo
       #
       # @api private
       def get(**opts)
-        limit = opts[:limit]
+        limit = normalized_limit(opts[:limit])
+
         _namespace_key = namespace_key(**opts)
         _cache_key = cache_key(**opts)
 
@@ -186,7 +190,7 @@ module Mongo
         caching_cursor = namespace_hash[_cache_key]
         return nil unless caching_cursor
 
-        caching_cursor_limit = caching_cursor.view.limit
+        caching_cursor_limit = normalized_limit(caching_cursor.view.limit)
 
         # There are two scenarios in which a caching cursor could fulfill the
         # query:
@@ -196,6 +200,7 @@ module Mongo
         #
         # Otherwise, return nil because the stored cursor will not satisfy
         # the query.
+
         if limit && (caching_cursor_limit.nil? || caching_cursor_limit >= limit)
           caching_cursor
         elsif limit.nil? && caching_cursor_limit.nil?
@@ -205,6 +210,14 @@ module Mongo
         end
       end
 
+      def normalized_limit(limit)
+        return nil unless limit
+        # For the purposes of caching, a limit of 0 means no limit, as mongo treats it as such.
+        return nil if limit == 0
+        # For the purposes of caching, a negative limit is the same as as a positive limit.
+        limit.abs
+      end
+
       private
 
       def cache_key(**opts)
@@ -238,5 +251,50 @@ module Mongo
         end
       end
     end
+
+    # Rack middleware that activates the query cache for each request.
+    class Middleware
+
+      # Instantiate the middleware.
+      #
+      # @example Create the new middleware.
+      #   Middleware.new(app)
+      #
+      # @param [ Object ] app The rack application stack.
+      def initialize(app)
+        @app = app
+      end
+
+      # Enable query cache and execute the request.
+      #
+      # @example Execute the request.
+      #   middleware.call(env)
+      #
+      # @param [ Object ] env The environment.
+      #
+      # @return [ Object ] The result of the call.
+      def call(env)
+        QueryCache.cache do
+          @app.call(env)
+        end
+      ensure
+        QueryCache.clear
+      end
+
+      # ActiveJob middleware that activates the query cache for each job.
+      module ActiveJob
+        def self.included(base)
+          base.class_eval do
+            around_perform do |_job, block|
+              QueryCache.cache do
+                block.call
+              end
+            ensure
+              QueryCache.clear
+            end
+          end
+        end
+      end
+    end
   end
 end

data/lib/mongo/retryable.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -188,25 +191,29 @@ module Mongo
     # @note This only retries operations on not master failures, since it is
     #   the only case we can be sure a partial write did not already occur.
     #
-    # @param [ nil | Session ] session Optional session to use with the operation.
     # @param [ nil | Hash | WriteConcern::Base ] write_concern The write concern.
-    # @param [ true | false ] ending_transaction True if the write operation is abortTransaction or
-    #   commitTransaction, false otherwise.
+    # @param [ true | false ] ending_transaction True if the write operation is
+    #   abortTransaction or commitTransaction, false otherwise.
+    # @param [ Context ] context The context for the operation.
     # @param [ Proc ] block The block to execute.
     #
-    # @yieldparam [ Server ] server The server to which the write should be sent.
+    # @yieldparam [ Connection ] connection The connection through which the
+    #   write should be sent.
     # @yieldparam [ Integer ] txn_num Transaction number (NOT the ACID kind).
+    # @yieldparam [ Operation::Context ] context The operation context.
     #
     # @return [ Result ] The result of the operation.
     #
     # @since 2.1.0
-    def write_with_retry(session, write_concern, ending_transaction = false, &block)
+    def write_with_retry(write_concern, ending_transaction: false, context:, &block)
+      session = context.session
+
       if ending_transaction && !session
         raise ArgumentError, 'Cannot end a transaction without a session'
       end
 
       unless ending_transaction || retry_write_allowed?(session, write_concern)
-        return legacy_write_with_retry(nil, session, &block)
+        return legacy_write_with_retry(nil, context: context, &block)
       end
 
       # If we are here, session is not nil. A session being nil would have
@@ -215,23 +222,46 @@ module Mongo
       server = select_server(cluster, ServerSelector.primary, session)
 
       unless ending_transaction || server.retry_writes?
-        return legacy_write_with_retry(server, session, &block)
+        return legacy_write_with_retry(server, context: context, &block)
       end
 
-      txn_num = if session.in_transaction?
-        session.txn_num
-      else
-        session.next_txn_num
-      end
+      txn_num = nil
+
       begin
-        yield(server, txn_num, false)
-      rescue Error::SocketError, Error::SocketTimeoutError => e
+        connection_succeeded = false
+        server.with_connection(connection_global_id: context.connection_global_id) do |connection|
+          connection_succeeded = true
+
+          session.materialize_if_needed
+          txn_num = if session.in_transaction?
+            session.txn_num
+          else
+            session.next_txn_num
+          end
+
+          # The context needs to be duplicated here because we will be using
+          # it later for the retry as well.
+          yield(connection, txn_num, context.dup)
+        end
+      rescue Error::SocketError, Error::SocketTimeoutError, Auth::Unauthorized => e
         e.add_note('modern retry')
         e.add_note("attempt 1")
         if !e.label?('RetryableWriteError')
-          raise e
+          # If we get an auth error, it was raised when connecting the connection
+          # and therefore we didn't have the connection yet to add labels.
+          # Therefore, check if it is retryable, and if it is, add the label
+          # and retry it. We also want to retry this if there was a Socket error
+          # when trying to create the connection.
+          if !connection_succeeded && !session.in_transaction? && e.write_retryable?
+            e.add_label('RetryableWriteError')
+          else
+            raise e
+          end
         end
-        retry_write(e, session, txn_num, &block)
+
+        # Context#with creates a new context, which is not necessary here
+        # but the API is less prone to misuse this way.
+        retry_write(e, txn_num, context: context.with(is_retry: true), &block)
       rescue Error::OperationFailure => e
         e.add_note('modern retry')
         e.add_note("attempt 1")
@@ -241,7 +271,9 @@ module Mongo
           raise e
         end
 
-        retry_write(e, session, txn_num, &block)
+        # Context#with creates a new context, which is not necessary here
+        # but the API is less prone to misuse this way.
+        retry_write(e, txn_num, context: context.with(is_retry: true), &block)
       end
     end
 
@@ -255,23 +287,30 @@ module Mongo
     # delegates to legacy_write_with_retry which performs write retries using
     # legacy logic.
     #
-    # @param [ nil | Session ] session Optional session to use with the operation.
     # @param [ nil | Hash | WriteConcern::Base ] write_concern The write concern.
+    # @param [ Context ] context The context for the operation.
     #
-    # @yieldparam [ Server ] server The server to which the write should be sent.
+    # @yieldparam [ Connection ] connection The connection through which the
+    #   write should be sent.
+    # @yieldparam [ nil ] txn_num nil as transaction number.
+    # @yieldparam [ Operation::Context ] context The operation context.
     #
     # @api private
-    def nro_write_with_retry(session, write_concern, &block)
+    def nro_write_with_retry(write_concern, context:, &block)
+      session = context.session
+
+      server = select_server(cluster, ServerSelector.primary, session)
       if session && session.client.options[:retry_writes]
-        server = select_server(cluster, ServerSelector.primary, session)
         begin
-          yield server
+          server.with_connection(connection_global_id: context.connection_global_id) do |connection|
+            yield connection, nil, context
+          end
         rescue Error::SocketError, Error::SocketTimeoutError, Error::OperationFailure => e
           e.add_note('retries disabled')
           raise e
         end
       else
-        legacy_write_with_retry(nil, session, &block)
+        legacy_write_with_retry(server, context: context, &block)
       end
     end
 
@@ -284,12 +323,17 @@ module Mongo
     # @param [ Server ] server The server which should be used for the
     #   operation. If not provided, the current primary will be retrieved from
     #   the cluster.
-    # @param [ nil | Session ] session Optional session to use with the operation.
+    # @param [ Context ] context The context for the operation.
     #
-    # @yieldparam [ Server ] server The server to which the write should be sent.
+    # @yieldparam [ Connection ] connection The connection through which the
+    #    write should be sent.
+    # @yieldparam [ nil ] txn_num nil as transaction number.
+    # @yieldparam [ Operation::Context ] context The operation context.
     #
     # @api private
-    def legacy_write_with_retry(server = nil, session = nil)
+    def legacy_write_with_retry(server = nil, context:)
+      session = context.session
+
       # This is the pre-session retry logic, and is not subject to
       # current retryable write specifications.
       # In particular it does not retry on SocketError and SocketTimeoutError.
@@ -297,7 +341,10 @@ module Mongo
       begin
         attempt += 1
         server ||= select_server(cluster, ServerSelector.primary, session)
-        yield server
+        server.with_connection(connection_global_id: context.connection_global_id) do |connection|
+          # Legacy retries do not use txn_num
+          yield connection, nil, context.dup
+        end
       rescue Error::OperationFailure => e
         e.add_note('legacy retry')
         e.add_note("attempt #{attempt}")
@@ -328,7 +375,7 @@ module Mongo
           raise e
         end
         retry_read(e, server_selector, session, &block)
-      rescue Error::OperationFailure => e
+      rescue Error::OperationFailure, Auth::Unauthorized => e
         e.add_note('modern retry')
         e.add_note("attempt 1")
         if session.in_transaction? || !e.write_retryable?
@@ -420,7 +467,9 @@ module Mongo
       raise original_error
     end
 
-    def retry_write(original_error, session, txn_num, &block)
+    def retry_write(original_error, txn_num, context:, &block)
+      session = context.session
+
       # We do not request a scan of the cluster here, because error handling
       # for the error which triggered the retry should have updated the
       # server description and/or topology as necessary (specifically,
@@ -441,7 +490,9 @@ module Mongo
         raise Error::RaiseOriginalError
       end
       log_retry(original_error, message: 'Write retry')
-      yield(server, txn_num, true)
+      server.with_connection(connection_global_id: context.connection_global_id) do |connection|
+        yield(connection, txn_num, context)
+      end
     rescue Error::SocketError, Error::SocketTimeoutError => e
       e.add_note('modern retry')
       e.add_note('attempt 2')
@@ -477,7 +528,7 @@ module Mongo
       else
         "Retry"
       end
-      Logger.logger.warn "#{message} due to: #{e.class.name} #{e.message}"
+      Logger.logger.warn "#{message} due to: #{e.class.name}: #{e.message}"
     end
 
     # Retry writes on MMAPv1 should raise an actionable error; append actionable

data/lib/mongo/semaphore.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2018-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

data/lib/mongo/server/app_metadata.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2016-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +17,7 @@
 
 module Mongo
   class Server
-    # Application metadata that is sent to the server in an ismaster command,
+    # Application metadata that is sent to the server during a handshake,
     #   when a new connection is established.
     #
     # @api private
@@ -43,6 +46,11 @@ module Mongo
       # @api private
       AUTH_OPTION_KEYS = [:user, :auth_source, :auth_mech].freeze
 
+      # Possible connection purposes.
+      #
+      # @api private
+      PURPOSES = %i(application monitor push_monitor).freeze
+
       # Instantiate the new AppMetadata object.
       #
       # @api private
@@ -60,10 +68,16 @@ module Mongo
       # @option options [ Array<String> ] :compressors A list of potential
       #   compressors to use, in order of preference. The driver chooses the
       #   first compressor that is also supported by the server. Currently the
-      #   driver only supports 'zlib'.
+      #   driver only supports 'zstd', 'snappy' and 'zlib'.
       # @option options [ String ] :platform Platform information to include in
       #   the metadata printed to the mongod logs upon establishing a connection
       #   in server versions >= 3.4.
+      # @option options [ Symbol ] :purpose The purpose of this connection.
+      # @option options [ Hash ] :server_api The requested server API version.
+      #   This hash can have the following items:
+      #   - *:version* -- string
+      #   - *:strict* -- boolean
+      #   - *:deprecation_errors* -- boolean
       # @option options [ String ] :user The user name.
       # @option options [ Array<Hash> ] :wrapping_libraries Information about
       #   libraries such as ODMs that are wrapping the driver. Specify the
@@ -71,11 +85,17 @@ module Mongo
       #   :platform.
       #
       # @since 2.4.0
-      def initialize(options)
+      def initialize(options = {})
         @app_name = options[:app_name].to_s if options[:app_name]
         @platform = options[:platform]
+        if @purpose = options[:purpose]
+          unless PURPOSES.include?(@purpose)
+            raise ArgumentError, "Invalid purpose: #{@purpose}"
+          end
+        end
         @compressors = options[:compressors] || []
         @wrapping_libraries = options[:wrapping_libraries]
+        @server_api = options[:server_api]
 
         if options[:user] && !options[:auth_mech]
           auth_db = options[:auth_source] || 'admin'
@@ -83,25 +103,37 @@ module Mongo
         end
       end
 
+      # @return [ Symbol ] The purpose of the connection for which this
+      #   app metadata is created.
+      #
+      # @api private
+      attr_reader :purpose
+
+      # @return [ Hash | nil ] The requested server API version.
+      #
+      #   Thes hash can have the following items:
+      #   - *:version* -- string
+      #   - *:strict* -- boolean
+      #   - *:deprecation_errors* -- boolean
+      #
+      # @api private
+      attr_reader :server_api
+
       # @return [ Array<Hash> | nil ] Information about libraries wrapping
       #   the driver.
       attr_reader :wrapping_libraries
 
-      # Get the bytes of the ismaster message including this metadata.
+      # Get the metadata as BSON::Document to be sent to
+      # as part of the handshake. The document should
+      # be appended to a suitable handshake command.
       #
-      # @api private
+      # This method ensures that the metadata are valid.
       #
-      # @example Get the ismaster message bytes.
-      #   metadata.ismaster_bytes
+      # @return [BSON::Document] Valid document for connection's handshake.
       #
-      # @return [ String ] The raw bytes.
+      # @raise [ Error::InvalidApplicationName ] When the metadata are invalid.
       #
-      # @since 2.4.0
-      # @deprecated
-      def ismaster_bytes
-        @ismaster_bytes ||= validate! && serialize.to_s
-      end
-
+      # @api private
       def validated_document
         validate!
         document
@@ -109,6 +141,10 @@ module Mongo
 
       private
 
+      # Check whether it is possible to build a valid app metadata document
+      # with params provided on intialization.
+      #
+      # @raise [ Error::InvalidApplicationName ] When the metadata are invalid.
       def validate!
         if @app_name && @app_name.bytesize > MAX_APP_NAME_SIZE
           raise Error::InvalidApplicationName.new(@app_name, MAX_APP_NAME_SIZE)
@@ -116,6 +152,10 @@ module Mongo
         true
       end
 
+      # Get BSON::Document to be used as value for `client` key in
+      # handshake document.
+      #
+      # @return [BSON::Document] Document describing client for handshake.
       def full_client_document
         BSON::Document.new.tap do |doc|
           doc[:application] = { name: @app_name } if @app_name
@@ -125,27 +165,39 @@ module Mongo
         end
       end
 
-      def serialize
-        Protocol::Query.new(Database::ADMIN, Database::COMMAND, document, :limit => -1).serialize
-      end
 
+      # Get the metadata as BSON::Document to be sent to
+      # as part of the handshake. The document should
+      # be appended to a suitable handshake command.
+      #
+      # @return [BSON::Document] Document for connection's handshake.
       def document
-        client_document = full_client_document
-        while client_document.to_bson.to_s.size > MAX_DOCUMENT_SIZE do
-          if client_document[:os][:name] || client_document[:os][:architecture]
-            client_document[:os].delete(:name)
-            client_document[:os].delete(:architecture)
-          elsif client_document[:platform]
-            client_document.delete(:platform)
-          else
-            client_document = nil
+        @document ||= begin
+          client_document = full_client_document
+          while client_document.to_bson.to_s.size > MAX_DOCUMENT_SIZE do
+            if client_document[:os][:name] || client_document[:os][:architecture]
+              client_document[:os].delete(:name)
+              client_document[:os].delete(:architecture)
+            elsif client_document[:platform]
+              client_document.delete(:platform)
+            else
+              client_document = nil
+            end
           end
+          document = BSON::Document.new(
+            {
+              compression: @compressors,
+              client: client_document,
+            }
+          )
+          document[:saslSupportedMechs] = @request_auth_mech if @request_auth_mech
+          if @server_api
+            document.update(
+              Utils.transform_server_api(@server_api)
+            )
+          end
+          document
         end
-        document = Server::Monitor::Connection::ISMASTER
-        document = document.merge(compression: @compressors)
-        document[:client] = client_document
-        document[:saslSupportedMechs] = @request_auth_mech if @request_auth_mech
-        document
       end
 
       def driver_doc
@@ -192,12 +244,16 @@ module Mongo
           ruby_versions = ["Ruby #{RUBY_VERSION}"]
           platforms = [RUBY_PLATFORM]
         end
-        platform = [
+        platforms = [
           @platform,
           *ruby_versions,
           *platforms,
           RbConfig::CONFIG['build'],
-        ].compact.join(', ')
+        ]
+        if @purpose
+          platforms << @purpose.to_s[0].upcase
+        end
+        platform = platforms.compact.join(', ')
         platforms = [platform]
         if wrapping_libraries
           wrapping_libraries.each do |library|

data/lib/mongo/server/connection.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,6 +24,7 @@ module Mongo
     class Connection < ConnectionBase
       include Monitoring::Publishable
       include Retryable
+      include Id
       extend Forwardable
 
       # The ping command.
@@ -83,12 +87,26 @@ module Mongo
       # @param [ Mongo::Server ] server The server the connection is for.
       # @param [ Hash ] options The connection options.
       #
-      # @option options [ Integer ] :generation Connection pool's generation
-      #   for this connection.
+      # @option options [ Integer ] :generation The generation of this
+      #   connection. The generation should only be specified in this option
+      #   when not in load-balancing mode, and it should be the generation
+      #   of the connection pool when the connection is created. In
+      #   load-balancing mode, the generation is set on the connection
+      #   after the handshake completes.
+      # @option options [ Hash ] :server_api The requested server API version.
+      #   This hash can have the following items:
+      #   - *:version* -- string
+      #   - *:strict* -- boolean
+      #   - *:deprecation_errors* -- boolean
       #
       # @since 2.0.0
       def initialize(server, options = {})
+        if server.load_balancer? && options[:generation]
+          raise ArgumentError, "Generation cannot be set when server is a load balancer"
+        end
+
         @id = server.next_connection_id
+        @global_id = self.class.next_id
         @monitoring = server.monitoring
         @options = options.freeze
         @server = server
@@ -96,6 +114,7 @@ module Mongo
         @last_checkin = nil
         @auth_mechanism = nil
         @pid = Process.pid
+        @pinned = false
 
         publish_cmap_event(
           Monitoring::Event::Cmap::ConnectionCreated.new(address, id)
@@ -113,6 +132,10 @@ module Mongo
       # @since 2.9.0
       attr_reader :id
 
+      # @return [ Integer ] The global ID for the connection. This will be unique
+      # across all connections.
+      attr_reader :global_id
+
       # The connection pool from which this connection was created.
       # May be nil.
       #
@@ -138,6 +161,32 @@ module Mongo
         !!@error
       end
 
+      # Whether the connection is used by a transaction or cursor operations.
+      #
+      # Pinned connections should not be disconnected and removed from a
+      # connection pool if they are idle or stale.
+      #
+      # # @return [ true | false ] Whether connection is pinned.
+      #
+      # @api private
+      def pinned?
+        @pinned
+      end
+
+      # Mark the connection as pinned.
+      #
+      # @api private
+      def pin
+        @pinned = true
+      end
+
+      # Mark the connection as not pinned.
+      #
+      # @api private
+      def unpin
+        @pinned = false
+      end
+
       # Establishes a network connection to the target address.
       #
       # If the connection is already established, this method does nothing.
@@ -165,6 +214,15 @@ module Mongo
           # authenticated and be usable.
           @socket, @description, @compressor = do_connect
 
+          if server.load_balancer?
+            if Lint.enabled?
+              unless service_id
+                raise Error::InternalDriverError, "The connection is to a load balancer and it must have service_id set here, but does not"
+              end
+            end
+            @generation = connection_pool.generation_manager.generation(service_id: service_id)
+          end
+
           publish_cmap_event(
             Monitoring::Event::Cmap::ConnectionReady.new(address, id)
           )
@@ -177,7 +235,7 @@ module Mongo
       # Separate method to permit easier mocking in the test suite.
       #
       # @return [ Array<Socket, Server::Description> ] Connected socket and
-      #   a server description instance from the ismaster response of the
+      #   a server description instance from the hello response of the
       #   returned socket.
       private def do_connect
         socket = add_server_diagnostics do
@@ -306,7 +364,12 @@ module Mongo
           yield
         rescue Error::SocketError => e
           @error = e
-          @server.unknown!(generation: e.generation, stop_push_monitor: true)
+          @server.unknown!(
+            generation: e.generation,
+            # or description.service_id?
+            service_id: e.service_id,
+            stop_push_monitor: true,
+          )
           raise
         rescue Error::SocketTimeoutError => e
           @error = e