mongo 2.4.1 → 2.14.0

data/lib/mongo/collection/view/change_stream.rb

@@ -0,0 +1,369 @@
+# Copyright (C) 2017-2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'mongo/collection/view/change_stream/retryable'
+
+module Mongo
+  class Collection
+    class View
+
+      # Provides behavior around a `$changeStream` pipeline stage in the
+      # aggregation framework. Specifying this stage allows users to request
+      # that notifications are sent for all changes to a particular collection
+      # or database.
+      #
+      # @note Only available in server versions 3.6 and higher.
+      # @note ChangeStreams do not work properly with JRuby because of the
+      #  issue documented here: https://github.com/jruby/jruby/issues/4212.
+      #  Namely, JRuby eagerly evaluates #next on an Enumerator in a background
+      #  green thread, therefore calling #next on the change stream will cause
+      #  getMores to be called in a loop in the background.
+      #
+      #
+      # @since 2.5.0
+      class ChangeStream < Aggregation
+        include Retryable
+
+        # @return [ String ] The fullDocument option default value.
+        #
+        # @since 2.5.0
+        FULL_DOCUMENT_DEFAULT = 'default'.freeze
+
+        # @return [ Symbol ] Used to indicate that the change stream should listen for changes on
+        #   the entire database rather than just the collection.
+        #
+        # @since 2.6.0
+        DATABASE = :database
+
+        # @return [ Symbol ] Used to indicate that the change stream should listen for changes on
+        #   the entire cluster rather than just the collection.
+        #
+        # @since 2.6.0
+        CLUSTER = :cluster
+
+        # @return [ BSON::Document ] The change stream options.
+        #
+        # @since 2.5.0
+        attr_reader :options
+
+        # Initialize the change stream for the provided collection view, pipeline
+        # and options.
+        #
+        # @example Create the new change stream view.
+        #   ChangeStream.new(view, pipeline, options)
+        #
+        # @param [ Collection::View ] view The collection view.
+        # @param [ Array<Hash> ] pipeline The pipeline of operators to filter the change notifications.
+        # @param [ Hash ] options The change stream options.
+        #
+        # @option options [ String ] :full_document Allowed values: 'default', 'updateLookup'. Defaults to 'default'.
+        #   When set to 'updateLookup', the change notification for partial updates will include both a delta
+        #   describing the changes to the document, as well as a copy of the entire document that was changed
+        #   from some time after the change occurred.
+        # @option options [ BSON::Document, Hash ] :resume_after Specifies the logical starting point for the
+        #   new change stream.
+        # @option options [ Integer ] :max_await_time_ms The maximum amount of time for the server to wait
+        #   on new documents to satisfy a change stream query.
+        # @option options [ Integer ] :batch_size The number of documents to return per batch.
+        # @option options [ BSON::Document, Hash ] :collation The collation to use.
+        # @option options [ BSON::Timestamp ] :start_at_operation_time Only
+        #   return changes that occurred at or after the specified timestamp. Any
+        #   command run against the server will return a cluster time that can
+        #   be used here. Only recognized by server versions 4.0+.
+        # @option options [ Bson::Document, Hash ] :start_after Similar to :resume_after, this
+        #   option takes a resume token and starts a new change stream returning the first
+        #   notification after the token. This will allow users to watch collections that have been
+        #   dropped and recreated or newly renamed collections without missing any notifications.
+        #
+        #   The server will report an error if `startAfter` and `resumeAfter` are both specified.
+        #
+        # @since 2.5.0
+        def initialize(view, pipeline, changes_for, options = {})
+          @view = view
+          @changes_for = changes_for
+          @change_stream_filters = pipeline && pipeline.dup
+          @options = options && options.dup.freeze
+          @start_after = @options[:start_after]
+
+          # The resume token tracked by the change stream, used only
+          # when there is no cursor, or no cursor resume token
+          @resume_token = @start_after || @options[:resume_after]
+
+          create_cursor!
+
+          # We send different parameters when we resume a change stream
+          # compared to when we send the first query
+          @resuming = true
+        end
+
+        # Iterate through documents returned by the change stream.
+        #
+        # This method retries once per error on resumable errors
+        # (two consecutive errors result in the second error being raised,
+        # an error which is recovered from resets the error count to zero).
+        #
+        # @example Iterate through the stream of documents.
+        #   stream.each do |document|
+        #     p document
+        #   end
+        #
+        # @return [ Enumerator ] The enumerator.
+        #
+        # @since 2.5.0
+        #
+        # @yieldparam [ BSON::Document ] Each change stream document.
+        def each
+          raise StopIteration.new if closed?
+          loop do
+            document = try_next
+            yield document if document
+          end
+        rescue StopIteration => e
+          return self
+        end
+
+        # Return one document from the change stream, if one is available.
+        #
+        # Retries once on a resumable error.
+        #
+        # Raises StopIteration if the change stream is closed.
+        #
+        # This method will wait up to max_await_time_ms milliseconds
+        # for changes from the server, and if no changes are received
+        # it will return nil.
+        #
+        # @return [ BSON::Document | nil ] A change stream document.
+        # @since 2.6.0
+        def try_next
+          raise StopIteration.new if closed?
+          begin
+            doc = @cursor.try_next
+          rescue Mongo::Error => e
+            if !e.change_stream_resumable?
+              raise
+            end
+
+            # Rerun initial aggregation.
+            # Any errors here will stop iteration and break out of this
+            # method.
+
+            # Save cursor's resume token so we can use it
+            # to create a new cursor
+            @resume_token = @cursor.resume_token
+
+            close
+            create_cursor!
+            retry
+          end
+
+          # We need to verify each doc has an _id, so we
+          # have a resume token to work with
+          if doc && doc['_id'].nil?
+            raise Error::MissingResumeToken
+          end
+          doc
+        end
+
+        def to_enum
+          enum = super
+          enum.send(:instance_variable_set, '@obj', self)
+          class << enum
+            def try_next
+              @obj.try_next
+            end
+          end
+          enum
+        end
+
+        # Close the change stream.
+        #
+        # @example Close the change stream.
+        #   stream.close
+        #
+        # @note This method attempts to close the cursor used by the change
+        #   stream, which in turn closes the server-side change stream cursor.
+        #   This method ignores any errors that occur when closing the
+        #   server-side cursor.
+        #
+        # @return [ nil ] Always nil.
+        #
+        # @since 2.5.0
+        def close
+          unless closed?
+            begin
+              @cursor.close
+            rescue Error::OperationFailure
+              # ignore
+            end
+            @cursor = nil
+          end
+        end
+
+        # Is the change stream closed?
+        #
+        # @example Determine whether the change stream is closed.
+        #   stream.closed?
+        #
+        # @return [ true, false ] If the change stream is closed.
+        #
+        # @since 2.5.0
+        def closed?
+          @cursor.nil?
+        end
+
+        # Get a formatted string for use in inspection.
+        #
+        # @example Inspect the change stream object.
+        #   stream.inspect
+        #
+        # @return [ String ] The change stream inspection.
+        #
+        # @since 2.5.0
+        def inspect
+          "#<Mongo::Collection::View:ChangeStream:0x#{object_id} filters=#{@change_stream_filters} " +
+            "options=#{@options} resume_token=#{resume_token}>"
+        end
+
+        # Returns the resume token that the stream will
+        # use to automatically resume, if one exists.
+        #
+        # @example Get the change stream resume token.
+        #   stream.resume_token
+        #
+        # @return [ BSON::Document | nil ] The change stream resume token.
+        #
+        # @since 2.10.0
+        def resume_token
+          cursor_resume_token = @cursor.resume_token if @cursor
+          cursor_resume_token || @resume_token
+        end
+
+        private
+
+        def for_cluster?
+          @changes_for == CLUSTER
+        end
+
+        def for_database?
+          @changes_for == DATABASE
+        end
+
+        def for_collection?
+          !for_cluster? && !for_database?
+        end
+
+        def create_cursor!
+          # clear the cache because we may get a newer or an older server
+          # (rolling upgrades)
+          @start_at_operation_time_supported = nil
+
+          session = client.send(:get_session, @options)
+          start_at_operation_time = nil
+          start_at_operation_time_supported = nil
+          @cursor = read_with_retry_cursor(session, server_selector, view) do |server|
+            start_at_operation_time_supported = server.description.server_version_gte?('4.0')
+
+            result = send_initial_query(server, session)
+            if doc = result.replies.first && result.replies.first.documents.first
+              start_at_operation_time = doc['operationTime']
+            else
+              # The above may set @start_at_operation_time to nil
+              # if it was not in the document for some reason,
+              # for consistency set it to nil here as well.
+              # NB: since this block may be executed more than once, each
+              # execution must write to start_at_operation_time either way.
+              start_at_operation_time = nil
+            end
+            result
+          end
+          @start_at_operation_time = start_at_operation_time
+          @start_at_operation_time_supported = start_at_operation_time_supported
+        end
+
+        def pipeline
+          [{ '$changeStream' => change_doc }] + @change_stream_filters
+        end
+
+        def aggregate_spec(session)
+          super(session).tap do |spec|
+            spec[:selector][:aggregate] = 1 unless for_collection?
+          end
+        end
+
+        def change_doc
+          {}.tap do |doc|
+            if @options[:full_document]
+              doc[:fullDocument] = @options[:full_document]
+            end
+
+            if resuming?
+              # We have a resume token once we retrieved any documents.
+              # However, if the first getMore fails and the user didn't pass
+              # a resume token we won't have a resume token to use.
+              # Use start_at_operation time in this case
+              if resume_token
+                # Spec says we need to remove both startAtOperationTime and startAfter if
+                # either was passed in by user, thus we won't forward them
+                doc[:resumeAfter] = resume_token
+              elsif @start_at_operation_time_supported && @start_at_operation_time
+                # It is crucial to check @start_at_operation_time_supported
+                # here - we may have switched to an older server that
+                # does not support operation times and therefore shouldn't
+                # try to send one to it!
+                #
+                # @start_at_operation_time is already a BSON::Timestamp
+                doc[:startAtOperationTime] = @start_at_operation_time
+              else
+                # Can't resume if we don't have either
+                raise Mongo::Error::MissingResumeToken
+              end
+            else
+              if @start_after
+                doc[:startAfter] = @start_after
+              elsif resume_token
+                doc[:resumeAfter] = resume_token
+              end
+
+              if options[:start_at_operation_time]
+                doc[:startAtOperationTime] = time_to_bson_timestamp(
+                  options[:start_at_operation_time])
+              end
+            end
+
+            doc[:allChangesForCluster] = true if for_cluster?
+          end
+        end
+
+        def send_initial_query(server, session)
+          initial_query_op(session).execute(server, client: client)
+        end
+
+        def time_to_bson_timestamp(time)
+          if time.is_a?(Time)
+            seconds = time.to_f
+            BSON::Timestamp.new(seconds.to_i, ((seconds - seconds.to_i) * 1000000).to_i)
+          elsif time.is_a?(BSON::Timestamp)
+            time
+          else
+            raise ArgumentError, 'Time must be a Time or a BSON::Timestamp instance'
+          end
+        end
+
+        def resuming?
+          !!@resuming
+        end
+      end
+    end
+  end
+end

data/lib/mongo/collection/view/explainable.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2016 MongoDB, Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@ module Mongo
   class Collection
     class View
 
-      # Defines explain related behaviour for collection view.
+      # Defines explain related behavior for collection view.
       #
       # @since 2.0.0
       module Explainable
@@ -36,16 +36,25 @@ module Mongo
         # @since 2.2.0
         ALL_PLANS_EXECUTION = 'allPlansExecution'.freeze
 
-        # Get the explain plan for the query.
+        # Get the query plan for the query.
         #
-        # @example Get the explain plan for the query.
-        #   view.explain
+        # @example Get the query plan for the query with execution statistics.
+        #   view.explain(verbosity: :execution_stats)
         #
-        # @return [ Hash ] A single document with the explain plan.
+        # @option opts [ true | false ] :verbose The level of detail
+        #   to return for MongoDB 2.6 servers.
+        # @option opts [ String | Symbol ] :verbosity The type of information
+        #   to return for MongoDB 3.0 and newer servers. If the value is a
+        #   symbol, it will be stringified and converted from underscore
+        #   style to camel case style (e.g. :query_planner => "queryPlanner").
+        #
+        # @return [ Hash ] A single document with the query plan.
+        #
+        # @see https://docs.mongodb.com/manual/reference/method/db.collection.explain/#db.collection.explain
         #
         # @since 2.0.0
-        def explain
-          self.class.new(collection, selector, options.merge(explain_options)).first
+        def explain(**opts)
+          self.class.new(collection, selector, options.merge(explain_options(**opts))).first
         end
 
         private
@@ -54,9 +63,19 @@ module Mongo
           !!options[:explain]
         end
 
-        def explain_options
+        # @option opts [ true | false ] :verbose The level of detail
+        #   to return for MongoDB 2.6 servers.
+        # @option opts [ String | Symbol ] :verbosity The type of information
+        #   to return for MongoDB 3.0 and newer servers. If the value is a
+        #   symbol, it will be stringified and converted from underscore
+        #   style to camel case style (e.g. :query_planner => "queryPlanner").
+        def explain_options(**opts)
           explain_limit = limit || 0
-          { :limit => -explain_limit.abs, :explain => true }
+          # Note: opts will never be nil here.
+          if Symbol === opts[:verbosity]
+            opts[:verbosity] = Utils.camelize(opts[:verbosity])
+          end
+          { limit: -explain_limit.abs, explain: opts }
         end
       end
     end

data/lib/mongo/collection/view/immutable.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2016 MongoDB, Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@ module Mongo
   class Collection
     class View
 
-      # Defines behaviour around views being configurable and immutable.
+      # Defines behavior around views being configurable and immutable.
       #
       # @since 2.0.0
       module Immutable

data/lib/mongo/collection/view/iterable.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2016 MongoDB, Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@ module Mongo
   class Collection
     class View
 
-      # Defines iteration related behaviour for collection views, including
+      # Defines iteration related behavior for collection views, including
       # cursor instantiation.
       #
       # @since 2.0.0
@@ -35,50 +35,133 @@ module Mongo
         #
         # @yieldparam [ Hash ] Each matching document.
         def each
-          @cursor = nil
-          read_with_retry do
-            server = read.select_server(cluster, false)
-            result = send_initial_query(server)
-            @cursor = Cursor.new(view, result, server)
+          @cursor = if use_query_cache? && cached_cursor
+            cached_cursor
+          else
+            session = client.send(:get_session, @options)
+            select_cursor(session)
+          end
+
+          if use_query_cache?
+            # No need to store the cursor in the query cache if there is
+            # already a cached cursor stored at this key.
+            QueryCache.set(@cursor, **cache_options) unless cached_cursor
+
+            # If a query with a limit is performed, the query cache will
+            # re-use results from an earlier query with the same or larger
+            # limit, and then impose the lower limit during iteration.
+            limit_for_cached_query = respond_to?(:limit) ? limit : nil
+          end
+
+          if block_given?
+            # Ruby versions 2.5 and older do not support arr[0..nil] syntax, so
+            # this must be a separate conditional.
+            cursor_to_iterate = if limit_for_cached_query
+              @cursor.to_a[0...limit_for_cached_query]
+            else
+              @cursor
+            end
+
+            cursor_to_iterate.each do |doc|
+              yield doc
+            end
+          else
+            @cursor.to_enum
           end
-          @cursor.each do |doc|
-            yield doc
-          end if block_given?
-          @cursor.to_enum
         end
 
-        # Stop the iteration by sending a KillCursors command to the server.
+        # Cleans up resources associated with this query.
+        #
+        # If there is a server cursor associated with this query, it is
+        # closed by sending a KillCursors command to the server.
         #
-        # @example Stop the iteration.
-        #   view.close_query
+        # @note This method propagates any errors that occur when closing the
+        #   server-side cursor.
+        #
+        # @return [ nil ] Always nil.
+        #
+        # @raise [ Error::OperationFailure ] If the server cursor close fails.
         #
         # @since 2.1.0
         def close_query
-          @cursor.send(:kill_cursors) if @cursor && !@cursor.closed?
+          if @cursor
+            @cursor.close
+          end
         end
         alias :kill_cursors :close_query
 
         private
 
-        def initial_query_op(server)
-          if server.features.find_command_enabled?
-            initial_command_op
+        def select_cursor(session)
+          if respond_to?(:write?, true) && write?
+            server = server_selector.select_server(cluster, nil, session)
+            result = send_initial_query(server, session)
+
+            # RUBY-2367: This will be updated to allow the query cache to
+            # cache cursors with multi-batch results.
+            if use_query_cache?
+              CachingCursor.new(view, result, server, session: session)
+            else
+              Cursor.new(view, result, server, session: session)
+            end
+          else
+            read_with_retry_cursor(session, server_selector, view) do |server|
+              send_initial_query(server, session)
+            end
+          end
+        end
+
+        def cached_cursor
+          QueryCache.get(**cache_options)
+        end
+
+        def cache_options
+          # NB: this hash is passed as keyword argument and must have symbol
+          # keys.
+          {
+            namespace: collection.namespace,
+            selector: selector,
+            skip: skip,
+            sort: sort,
+            limit: limit,
+            projection: projection,
+            collation: collation,
+            read_concern: read_concern,
+            read_preference: read_preference
+
+          }
+        end
+
+        def initial_query_op(server, session)
+          if server.with_connection { |connection| connection.features }.find_command_enabled?
+            initial_command_op(session)
           else
-            Operation::Read::Query.new(Builder::OpQuery.new(self).specification)
+            # Server versions that do not have the find command feature
+            # (versions older than 3.2) do not support the allow_disk_use option
+            # but perform no validation and will not raise an error if it is
+            # specified. If the allow_disk_use option is specified, raise an error
+            # to alert the user.
+            raise Error::UnsupportedOption.allow_disk_use_error if options.key?(:allow_disk_use)
+            Operation::Find.new(Builder::OpQuery.new(self).specification)
           end
         end
 
-        def initial_command_op
+        def initial_command_op(session)
+          builder = Builder::FindCommand.new(self, session)
           if explained?
-            Operation::Commands::Command.new(Builder::FindCommand.new(self).explain_specification)
+            Operation::Explain.new(builder.explain_specification)
           else
-            Operation::Commands::Find.new(Builder::FindCommand.new(self).specification)
+            Operation::Find.new(builder.specification)
           end
         end
 
-        def send_initial_query(server)
+        def send_initial_query(server, session = nil)
           validate_collation!(server, collation)
-          initial_query_op(server).execute(server)
+          initial_query_op(server, session).execute(server, client: client)
+        end
+
+        def use_query_cache?
+          QueryCache.enabled? && !collection.system_collection?
         end
       end
     end