mongo 2.10.5 → 2.15.1

data/lib/mongo/protocol/serializers.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -57,10 +60,11 @@ module Mongo
         # Deserializes the header value from the IO stream
         #
         # @param [ String ] buffer Buffer containing the message header.
+        # @param [ Hash ] options This method currently accepts no options.
         #
         # @return [ Array<Fixnum> ] Array consisting of the deserialized
         #   length, request id, response id, and op code.
-        def self.deserialize(buffer)
+        def self.deserialize(buffer, options = {})
           buffer.get_bytes(16).unpack(HEADER_PACK)
         end
       end
@@ -123,9 +127,10 @@ module Mongo
         # Deserializes a 32-bit Fixnum from the IO stream
         #
         # @param [ String ] buffer Buffer containing the 32-bit integer
+        # @param [ Hash ] options This method currently accepts no options.
         #
         # @return [ Fixnum ] Deserialized Int32
-        def self.deserialize(buffer)
+        def self.deserialize(buffer, options = {})
           buffer.get_int32
         end
       end
@@ -156,9 +161,10 @@ module Mongo
         # Deserializes a 64-bit Fixnum from the IO stream
         #
         # @param [ String ] buffer Buffer containing the 64-bit integer.
+        # @param [ Hash ] options This method currently accepts no options.
         #
         # @return [Fixnum] Deserialized Int64.
-        def self.deserialize(buffer)
+        def self.deserialize(buffer, options = {})
           buffer.get_int64
         end
       end
@@ -198,19 +204,24 @@ module Mongo
         # Deserializes a section of an OP_MSG from the IO stream.
         #
         # @param [ BSON::ByteBuffer ] buffer Buffer containing the sections.
+        # @param [ Hash ] options
+        #
+        # @option options [ Boolean ] :deserialize_as_bson Whether to perform
+        #   section deserialization using BSON types instead of native Ruby types
+        #   wherever possible.
         #
         # @return [ Array<BSON::Document> ] Deserialized sections.
         #
         # @since 2.5.0
-        def self.deserialize(buffer)
+        def self.deserialize(buffer, options = {})
           end_length = (@flag_bits & Msg::FLAGS.index(:checksum_present)) == 1 ? 32 : 0
           sections = []
           until buffer.length == end_length
             case byte = buffer.get_byte
             when PayloadZero::TYPE_BYTE
-              sections << PayloadZero.deserialize(buffer)
+              sections << PayloadZero.deserialize(buffer, options)
             when PayloadOne::TYPE_BYTE
-              sections += PayloadOne.deserialize(buffer)
+              sections += PayloadOne.deserialize(buffer, options)
             else
               raise Error::UnknownPayloadType.new(byte)
             end
@@ -260,12 +271,18 @@ module Mongo
           # Deserializes a section of payload type 0 of an OP_MSG from the IO stream.
           #
           # @param [ BSON::ByteBuffer ] buffer Buffer containing the sections.
+          # @param [ Hash ] options
+          #
+          # @option options [ Boolean ] :deserialize_as_bson Whether to perform
+          #   section deserialization using BSON types instead of native Ruby types
+          #   wherever possible.
           #
           # @return [ Array<BSON::Document> ] Deserialized section.
           #
           # @since 2.5.0
-          def self.deserialize(buffer)
-            BSON::Document.from_bson(buffer)
+          def self.deserialize(buffer, options = {})
+            mode = options[:deserialize_as_bson] ? :bson : nil
+            BSON::Document.from_bson(buffer, **{ mode: mode })
           end
         end
 
@@ -313,6 +330,8 @@ module Mongo
           #
           # @since 2.5.0
           def self.deserialize(buffer)
+            raise NotImplementedError
+
             start_size = buffer.length
             section_size = buffer.get_int32 # get the size
             end_size = start_size - section_size
@@ -340,18 +359,26 @@ module Mongo
         def self.serialize(buffer, value, max_bson_size = nil, validating_keys = BSON::Config.validating_keys?)
           start_size = buffer.length
           value.to_bson(buffer, validating_keys)
-          if max_bson_size && buffer.length - start_size > max_bson_size
-            raise Error::MaxBSONSize.new(max_bson_size)
+          serialized_size = buffer.length - start_size
+          if max_bson_size && serialized_size > max_bson_size
+            raise Error::MaxBSONSize,
+              "The document exceeds maximum allowed BSON object size after serialization. Serialized size: #{serialized_size} bytes, maximum allowed size: #{max_bson_size} bytes"
           end
         end
 
         # Deserializes a document from the IO stream
         #
         # @param [ String ] buffer Buffer containing the BSON encoded document.
+        # @param [ Hash ] options
+        #
+        # @option options [ Boolean ] :deserialize_as_bson Whether to perform
+        #   section deserialization using BSON types instead of native Ruby types
+        #   wherever possible.
         #
         # @return [ Hash ] The decoded BSON document.
-        def self.deserialize(buffer)
-          BSON::Document.from_bson(buffer)
+        def self.deserialize(buffer, options = {})
+          mode = options[:deserialize_as_bson] ? :bson : nil
+          BSON::Document.from_bson(buffer, **{ mode: mode })
         end
 
         # Whether there can be a size limit on this type after serialization.
@@ -385,11 +412,12 @@ module Mongo
         # Deserializes a byte from the byte buffer.
         #
         # @param [ BSON::ByteBuffer ] buffer Buffer containing the value to read.
+        # @param [ Hash ] options This method currently accepts no options.
         #
         # @return [ String ] The byte.
         #
         # @since 2.5.0
-        def self.deserialize(buffer)
+        def self.deserialize(buffer, options = {})
           buffer.get_byte
         end
       end
@@ -415,12 +443,15 @@ module Mongo
         # Deserializes bytes from the byte buffer.
         #
         # @param [ BSON::ByteBuffer ] buffer Buffer containing the value to read.
-        # @param [ Integer ] num_bytes Number of bytes to read.
+        # @param [ Hash ] options The method options.
+        #
+        # @option options [ Integer ] num_bytes Number of bytes to read.
         #
         # @return [ String ] The bytes.
         #
         # @since 2.5.0
-        def self.deserialize(buffer, num_bytes = nil)
+        def self.deserialize(buffer, options = {})
+          num_bytes = options[:num_bytes]
           buffer.get_bytes(num_bytes || buffer.length)
         end
       end

data/lib/mongo/protocol/update.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/protocol.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Wire Protocol Base
 require 'mongo/protocol/serializers'
 require 'mongo/protocol/registry'

data/lib/mongo/query_cache.rb

@@ -0,0 +1,275 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2020 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  module QueryCache
+    class << self
+
+      # Set whether the cache is enabled.
+      #
+      # @example Set if the cache is enabled.
+      #   QueryCache.enabled = true
+      #
+      # @param [ true, false ] value The enabled value.
+      def enabled=(value)
+        Thread.current["[mongo]:query_cache:enabled"] = value
+      end
+
+      # Is the query cache enabled on the current thread?
+      #
+      # @example Is the query cache enabled?
+      #   QueryCache.enabled?
+      #
+      # @return [ true, false ] If the cache is enabled.
+      def enabled?
+        !!Thread.current["[mongo]:query_cache:enabled"]
+      end
+
+      # Execute the block while using the query cache.
+      #
+      # @example Execute with the cache.
+      #   QueryCache.cache { collection.find }
+      #
+      # @return [ Object ] The result of the block.
+      def cache
+        enabled = enabled?
+        self.enabled = true
+        begin
+          yield
+        ensure
+          self.enabled = enabled
+        end
+      end
+
+      # Execute the block with the query cache disabled.
+      #
+      # @example Execute without the cache.
+      #   QueryCache.uncached { collection.find }
+      #
+      # @return [ Object ] The result of the block.
+      def uncached
+        enabled = enabled?
+        self.enabled = false
+        begin
+          yield
+        ensure
+          self.enabled = enabled
+        end
+      end
+
+      # Get the cached queries.
+      #
+      # @example Get the cached queries from the current thread.
+      #   QueryCache.cache_table
+      #
+      # @return [ Hash ] The hash of cached queries.
+      private def cache_table
+        Thread.current["[mongo]:query_cache"] ||= {}
+      end
+
+      # Clear the query cache.
+      #
+      # @example Clear the cache.
+      #   QueryCache.clear
+      #
+      # @return [ nil ] Always nil.
+      def clear
+        Thread.current["[mongo]:query_cache"] = nil
+      end
+
+      # Clear the section of the query cache storing cursors with results
+      # from this namespace.
+      #
+      # @param [ String ] namespace The namespace to be cleared, in the format
+      #   "database.collection".
+      #
+      # @return [ nil ] Always nil.
+      #
+      # @api private
+      def clear_namespace(namespace)
+        cache_table.delete(namespace)
+        # The nil key is where cursors are stored that could potentially read from
+        # multiple collections. This key should be cleared on every write operation
+        # to prevent returning stale data.
+        cache_table.delete(nil)
+        nil
+      end
+
+      # Store a CachingCursor instance in the query cache associated with the
+      # specified query options.
+      #
+      # @param [ Mongo::CachingCursor ] cursor The CachingCursor instance to store.
+      #
+      # @option opts [ String | nil ] namespace The namespace of the query,
+      #   in the format "database_name.collection_name".
+      # @option opts [ Array, Hash ] selector The selector passed to the query.
+      #   For most queries, this will be a Hash, but for aggregations, this
+      #   will be an Array representing the aggregation pipeline. May not be nil.
+      # @option opts [ Integer | nil ] skip The skip value of the query.
+      # @option opts [ Hash | nil ] sort The order of the query results
+      #   (e.g. { name: -1 }).
+      # @option opts [ Integer | nil ] limit The limit value of the query.
+      # @option opts [ Hash | nil ] projection The projection of the query
+      #   results (e.g. { name: 1 }).
+      # @option opts [ Hash | nil ] collation The collation of the query
+      #   (e.g. { "locale" => "fr_CA" }).
+      # @option opts [ Hash | nil ] read_concern The read concern of the query
+      #   (e.g. { level: :majority }).
+      # @option opts [ Hash | nil ] read_preference The read preference of
+      #   the query (e.g. { mode: :secondary }).
+      # @option opts [ Boolean | nil ] multi_collection Whether the query
+      #   results could potentially come from multiple collections. When true,
+      #   these results will be stored under the nil namespace key and cleared
+      #   on every write command.
+      #
+      # @return [ true ] Always true.
+      #
+      # @api private
+      def set(cursor, **opts)
+        _cache_key = cache_key(**opts)
+        _namespace_key = namespace_key(**opts)
+
+        cache_table[_namespace_key] ||= {}
+        cache_table[_namespace_key][_cache_key] = cursor
+
+        true
+      end
+
+      # For the given query options, retrieve a cached cursor that can be used
+      # to obtain the correct query results, if one exists in the cache.
+      #
+      # @option opts [ String | nil ] namespace The namespace of the query,
+      #   in the format "database_name.collection_name".
+      # @option opts [ Array, Hash ] selector The selector passed to the query.
+      #   For most queries, this will be a Hash, but for aggregations, this
+      #   will be an Array representing the aggregation pipeline. May not be nil.
+      # @option opts [ Integer | nil ] skip The skip value of the query.
+      # @option opts [ Hash | nil ] sort The order of the query results
+      #   (e.g. { name: -1 }).
+      # @option opts [ Integer | nil ] limit The limit value of the query.
+      # @option opts [ Hash | nil ] projection The projection of the query
+      #   results (e.g. { name: 1 }).
+      # @option opts [ Hash | nil ] collation The collation of the query
+      #   (e.g. { "locale" => "fr_CA" }).
+      # @option opts [ Hash | nil ] read_concern The read concern of the query
+      #   (e.g. { level: :majority }).
+      # @option opts [ Hash | nil ] read_preference The read preference of
+      #   the query (e.g. { mode: :secondary }).
+      # @option opts [ Boolean | nil ] multi_collection Whether the query
+      #   results could potentially come from multiple collections. When true,
+      #   these results will be stored under the nil namespace key and cleared
+      #   on every write command.
+      #
+      # @return [ Mongo::CachingCursor | nil ] Returns a CachingCursor if one
+      #   exists in the query cache, otherwise returns nil.
+      #
+      # @api private
+      def get(**opts)
+        limit = opts[:limit]
+        _namespace_key = namespace_key(**opts)
+        _cache_key = cache_key(**opts)
+
+        namespace_hash = cache_table[_namespace_key]
+        return nil unless namespace_hash
+
+        caching_cursor = namespace_hash[_cache_key]
+        return nil unless caching_cursor
+
+        caching_cursor_limit = caching_cursor.view.limit
+
+        # There are two scenarios in which a caching cursor could fulfill the
+        # query:
+        # 1. The query has a limit, and the stored cursor has no limit or
+        #    a larger limit.
+        # 2. The query has no limit and the stored cursor has no limit.
+        #
+        # Otherwise, return nil because the stored cursor will not satisfy
+        # the query.
+        if limit && (caching_cursor_limit.nil? || caching_cursor_limit >= limit)
+          caching_cursor
+        elsif limit.nil? && caching_cursor_limit.nil?
+          caching_cursor
+        else
+          nil
+        end
+      end
+
+      private
+
+      def cache_key(**opts)
+        unless opts[:namespace]
+          raise ArgumentError.new("Cannot generate cache key without namespace")
+        end
+        unless opts[:selector]
+          raise ArgumentError.new("Cannot generate cache key without selector")
+        end
+
+        [
+          opts[:namespace],
+          opts[:selector],
+          opts[:skip],
+          opts[:sort],
+          opts[:projection],
+          opts[:collation],
+          opts[:read_concern],
+          opts[:read_preference]
+        ]
+      end
+
+      # If the cached results can come from multiple collections, store this
+      # cursor under the nil namespace to be cleared on every write operation.
+      # Otherwise, store it under the specified namespace.
+      def namespace_key(**opts)
+        if opts[:multi_collection]
+          nil
+        else
+          opts[:namespace]
+        end
+      end
+    end
+
+    # Rack middleware that activates the query cache for each request.
+    class Middleware
+
+      # Instantiate the middleware.
+      #
+      # @example Create the new middleware.
+      #   Middleware.new(app)
+      #
+      # @param [ Object ] app The rack application stack.
+      def initialize(app)
+        @app = app
+      end
+
+      # Enable query cache and execute the request.
+      #
+      # @example Execute the request.
+      #   middleware.call(env)
+      #
+      # @param [ Object ] env The environment.
+      #
+      # @return [ Object ] The result of the call.
+      def call(env)
+        QueryCache.cache do
+          @app.call(env)
+        end
+      ensure
+        QueryCache.clear
+      end
+    end
+  end
+end

data/lib/mongo/retryable.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2015-2019 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -59,7 +62,14 @@ module Mongo
     def read_with_retry_cursor(session, server_selector, view, &block)
       read_with_retry(session, server_selector) do |server|
         result = yield server
-        Cursor.new(view, result, server, session: session)
+
+        # RUBY-2367: This will be updated to allow the query cache to
+        # cache cursors with multi-batch results.
+        if QueryCache.enabled? && !view.collection.system_collection?
+          CachingCursor.new(view, result, server, session: session)
+        else
+          Cursor.new(view, result, server, session: session)
+        end
       end
     end
 
@@ -144,7 +154,7 @@ module Mongo
     # @note This only retries read operations on socket errors.
     #
     # @param [ Hash ] options Options.
-    # @param [ Proc ] block The block to execute.
+    # @yield Calls the provided block with no arguments
     #
     # @option options [ String ] :retry_message Message to log when retrying.
     #
@@ -221,7 +231,7 @@ module Mongo
       rescue Error::SocketError, Error::SocketTimeoutError => e
         e.add_note('modern retry')
         e.add_note("attempt 1")
-        if session.in_transaction? && !ending_transaction
+        if !e.label?('RetryableWriteError')
           raise e
         end
         retry_write(e, session, txn_num, &block)
@@ -230,7 +240,7 @@ module Mongo
         e.add_note("attempt 1")
         if e.unsupported_retryable_write?
           raise_unsupported_error(e)
-        elsif (session.in_transaction? && !ending_transaction) || !e.write_retryable?
+        elsif !e.label?('RetryableWriteError')
           raise e
         end
 
@@ -298,7 +308,7 @@ module Mongo
         if attempt > client.max_write_retries
           raise e
         end
-        if e.write_retryable? && !(session && session.in_transaction?)
+        if e.label?('RetryableWriteError')
           log_retry(e, message: 'Legacy write retry')
           cluster.scan!(false)
           retry
@@ -381,9 +391,11 @@ module Mongo
     def retry_read(original_error, server_selector, session, &block)
       begin
         server = select_server(cluster, server_selector, session)
-      rescue => e
+      rescue Error, Error::AuthError => e
         original_error.add_note("later retry failed: #{e.class}: #{e}")
-        raise original_error
+
+        # See the corresponding note below in retry_write.
+        raise Error::RaiseOriginalError
       end
 
       log_retry(original_error, message: 'Read retry')
@@ -402,11 +414,13 @@ module Mongo
         end
         e.add_note("attempt 2")
         raise e
-      rescue => e
+      rescue Error, Error::AuthError => e
         e.add_note('modern retry')
         original_error.add_note("later retry failed: #{e.class}: #{e}")
         raise original_error
       end
+    rescue Error::RaiseOriginalError
+      raise original_error
     end
 
     def retry_write(original_error, session, txn_num, &block)
@@ -420,7 +434,14 @@ module Mongo
         # Do not need to add "modern retry" here, it should already be on
         # the first exception.
         original_error.add_note('did not retry because server selected for retry does not supoprt retryable writes')
-        raise original_error
+
+        # When we want to raise the original error, we must not run the
+        # rescue blocks below that add diagnostics because the diagnostics
+        # added would either be rendundant (e.g. modern retry note) or wrong
+        # (e.g. "attempt 2", we are raising the exception produced in the
+        # first attempt and haven't attempted the second time). Use the
+        # special marker class to bypass the ordinarily applicable rescues.
+        raise Error::RaiseOriginalError
       end
       log_retry(original_error, message: 'Write retry')
       yield(server, txn_num, true)
@@ -430,18 +451,20 @@ module Mongo
       raise e
     rescue Error::OperationFailure => e
       e.add_note('modern retry')
-      if e.write_retryable?
+      if e.label?('RetryableWriteError')
         e.add_note('attempt 2')
         raise e
       else
         original_error.add_note("later retry failed: #{e.class}: #{e}")
         raise original_error
       end
-    rescue => e
+    rescue Error, Error::AuthError => e
       # Do not need to add "modern retry" here, it should already be on
       # the first exception.
       original_error.add_note("later retry failed: #{e.class}: #{e}")
       raise original_error
+    rescue Error::RaiseOriginalError
+      raise original_error
     end
 
     # This is a separate method to make it possible for the test suite to
@@ -457,7 +480,7 @@ module Mongo
       else
         "Retry"
       end
-      Logger.logger.warn "#{message} due to: #{e.class.name} #{e.message}"
+      Logger.logger.warn "#{message} due to: #{e.class.name}: #{e.message}"
     end
 
     # Retry writes on MMAPv1 should raise an actionable error; append actionable

data/lib/mongo/semaphore.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2018-2019 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2018-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,7 +28,7 @@ module Mongo
 
     # Waits for the semaphore to be signaled up to timeout seconds.
     # If semaphore is not signaled, returns after timeout seconds.
-    def wait(timeout)
+    def wait(timeout = nil)
       @lock.synchronize do
         @cv.wait(@lock, timeout)
       end