mongo 2.5.3 → 2.6.0

data/lib/mongo/cluster/app_metadata.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2016 MongoDB, Inc.
+# Copyright (C) 2016-2018 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -55,6 +55,11 @@ module Mongo
         @app_name = cluster.options[:app_name].to_s if cluster.options[:app_name]
         @platform = cluster.options[:platform]
         @compressors = cluster.options[:compressors] || []
+
+        if cluster.options[:user] && !cluster.options[:auth_mech]
+          auth_db = cluster.options[:auth_source] || 'admin'
+          @request_auth_mech = "#{auth_db}.#{cluster.options[:user]}"
+        end
       end
 
       # Get the bytes of the ismaster message including this metadata.
@@ -108,6 +113,7 @@ module Mongo
         document = Server::Monitor::Connection::ISMASTER
         document = document.merge(compression: @compressors)
         document[:client] = client_document
+        document[:saslSupportedMechs] = @request_auth_mech if @request_auth_mech
         document
       end
 

data/lib/mongo/cluster/periodic_executor.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2017 MongoDB, Inc.
+# Copyright (C) 2014-2018 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/cluster/reapers/socket_reaper.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2017 MongoDB, Inc.
+# Copyright (C) 2014-2018 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/cluster/topology.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2017 MongoDB, Inc.
+# Copyright (C) 2014-2018 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,7 +20,17 @@ require 'mongo/cluster/topology/unknown'
 module Mongo
   class Cluster
 
-    # Defines behaviour for getting servers.
+    # Defines behavior for getting servers.
+    #
+    # Topologies are associated with their clusters - for example, a
+    # ReplicaSet topology contains the replica set name. A topology
+    # object therefore cannot be used with multiple cluster objects.
+    #
+    # At the same time, topology objects do not know anything about
+    # specific *servers* in a cluster, despite what their constructor
+    # may suggest. Which means, in particular, that topology change events
+    # require the application to maintain cluster references on its own
+    # if it wants to track server changes within a replica set.
     #
     # @since 2.0.0
     module Topology

data/lib/mongo/cluster/topology/replica_set.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2017 MongoDB, Inc.
+# Copyright (C) 2014-2018 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -123,6 +123,14 @@ module Mongo
         # @param [ Monitoring ] monitoring The monitoring.
         # @param [ Array<String> ] seeds The seeds.
         #
+        # @option options [ Symbol ] :replica_set Name of the replica set to
+        #   connect to. Can be left blank (either nil or the empty string are
+        #   accepted) to discover the name from the seeds. If the seeds
+        #   belong to different replica sets there is no guarantee which
+        #   replica set is selected - in particular, the driver may choose
+        #   the replica set name of a secondary if it returns its response
+        #   prior to a primary belonging to a different replica set.
+        #
         # @since 2.0.0
         def initialize(options, monitoring, seeds = [])
           @options = options

data/lib/mongo/cluster/topology/sharded.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2017 MongoDB, Inc.
+# Copyright (C) 2014-2018 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/cluster/topology/single.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2017 MongoDB, Inc.
+# Copyright (C) 2014-2018 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/cluster/topology/unknown.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2017 MongoDB, Inc.
+# Copyright (C) 2015-2018 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/collection.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2017 MongoDB, Inc.
+# Copyright (C) 2014-2018 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -109,7 +109,7 @@ module Mongo
     #
     # @since 2.0.0
     def server_selector
-     @server_selector ||= ServerSelector.get(read_preference || database.server_selector)
+      @server_selector ||= ServerSelector.get(read_preference || database.server_selector)
     end
 
     # Get the read preference on this collection.
@@ -295,12 +295,13 @@ module Mongo
     #
     # @since 2.1.0
     def aggregate(pipeline, options = {})
-      View.new(self, {}).aggregate(pipeline, options)
+      View.new(self, {}, options).aggregate(pipeline, options)
     end
 
-    # As of version 3.6 of the MongoDB server, a ``$changeStream`` pipeline stage is supported
-    # in the aggregation framework. This stage allows users to request that notifications are sent for
-    # all changes to a particular collection.
+    # As of version 3.6 of the MongoDB server, a ``$changeStream`` pipeline
+    # stage is supported in the aggregation framework. This stage allows users
+    # to request that notifications are sent for all changes to a particular
+    # collection.
     #
     # @example Get change notifications for a given collection.
     #   collection.watch([{ '$match' => { operationType: { '$in' => ['insert', 'replace'] } } }])
@@ -308,30 +309,36 @@ module Mongo
     # @param [ Array<Hash> ] pipeline Optional additional filter operators.
     # @param [ Hash ] options The change stream options.
     #
-    # @option options [ String ] :full_document Allowed values: ‘default’, ‘updateLookup’. Defaults to ‘default’.
-    #   When set to ‘updateLookup’, the change notification for partial updates will include both a delta
-    #   describing the changes to the document, as well as a copy of the entire document that was changed
-    #   from some time after the change occurred.
-    # @option options [ BSON::Document, Hash ] :resume_after Specifies the logical starting point for the
-    #   new change stream.
-    # @option options [ Integer ] :max_await_time_ms The maximum amount of time for the server to wait
-    #   on new documents to satisfy a change stream query.
-    # @option options [ Integer ] :batch_size The number of documents to return per batch.
+    # @option options [ String ] :full_document Allowed values: ‘default’,
+    #   ‘updateLookup’. Defaults to ‘default’. When set to ‘updateLookup’,
+    #   the change notification for partial updates will include both a delta
+    #   describing the changes to the document, as well as a copy of the entire
+    #   document that was changed from some time after the change occurred.
+    # @option options [ BSON::Document, Hash ] :resume_after Specifies the
+    #   logical starting point for the new change stream.
+    # @option options [ Integer ] :max_await_time_ms The maximum amount of time
+    #   for the server to wait on new documents to satisfy a change stream query.
+    # @option options [ Integer ] :batch_size The number of documents to return
+    #   per batch.
     # @option options [ BSON::Document, Hash ] :collation The collation to use.
     # @option options [ Session ] :session The session to use.
+    # @option options [ BSON::Timestamp ] :start_at_operation_time Only return
+    #   changes that occurred at or after the specified timestamp. Any command run
+    #   against the server will return a cluster time that can be used here.
+    #   Only recognized by server versions 4.0+.
     #
     # @note A change stream only allows 'majority' read concern.
-    # @note This helper method is preferable to running a raw aggregation with a $changeStream stage,
-    #   for the purpose of supporting resumability.
+    # @note This helper method is preferable to running a raw aggregation with
+    #   a $changeStream stage, for the purpose of supporting resumability.
     #
     # @return [ ChangeStream ] The change stream object.
     #
     # @since 2.5.0
     def watch(pipeline = [], options = {})
-      View::ChangeStream.new(View.new(self, {}, options), pipeline, options)
+      View::ChangeStream.new(View.new(self, {}, options), pipeline, nil, options)
     end
 
-    # Get a count of matching documents in the collection.
+    # Gets the number of matching documents in the collection.
     #
     # @example Get the count.
     #   collection.count(name: 1)
@@ -350,10 +357,59 @@ module Mongo
     # @return [ Integer ] The document count.
     #
     # @since 2.1.0
+    #
+    # @deprecated Use #count_documents or estimated_document_count instead. However, note that the
+    #   following operators will need to be substituted when switching to #count_documents:
+    #     * $where should be replaced with $expr (only works on 3.6+)
+    #     * $near should be replaced with $geoWithin with $center
+    #     * $nearSphere should be replaced with $geoWithin with $centerSphere
     def count(filter = nil, options = {})
       View.new(self, filter || {}, options).count(options)
     end
 
+    # Gets the number of of matching documents in the collection. Unlike the deprecated #count
+    # method, this will return the exact number of documents matching the filter rather than the estimate.
+    #
+    # @example Get the number of documents in the collection.
+    #   collection_view.count_documents
+    #
+    # @param [ Hash ] filter A filter for matching documents.
+    # @param [ Hash ] options Options for the operation.
+    #
+    # @option opts :skip [ Integer ] The number of documents to skip.
+    # @option opts :hint [ Hash ] Override default index selection and force
+    #   MongoDB to use a specific index for the query. Requires server version 3.6+.
+    # @option opts :limit [ Integer ] Max number of docs to count.
+    # @option opts :max_time_ms [ Integer ] The maximum amount of time to allow the
+    #   command to run.
+    # @option opts [ Hash ] :read The read preference options.
+    # @option opts [ Hash ] :collation The collation to use.
+    #
+    # @return [ Integer ] The document count.
+    #
+    # @since 2.6.0
+    def count_documents(filter, options = {})
+      View.new(self, filter, options).count_documents(options)
+    end
+
+    # Gets an estimate of the count of documents in a collection using collection metadata.
+    #
+    # @example Get the number of documents in the collection.
+    #   collection_view.estimated_document_count
+    #
+    # @param [ Hash ] options Options for the operation.
+    #
+    # @option opts :max_time_ms [ Integer ] The maximum amount of time to allow the command to
+    #   run.
+    # @option opts [ Hash ] :read The read preference options.
+    #
+    # @return [ Integer ] The document count.
+    #
+    # @since 2.6.0
+    def estimated_document_count(options = {})
+      View.new(self, {}, options).estimated_document_count(options)
+    end
+
     # Get a list of distinct values for a specific field.
     #
     # @example Get the distinct values.

data/lib/mongo/collection/view.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2017 MongoDB, Inc.
+# Copyright (C) 2014-2018 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/collection/view/aggregation.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2017 MongoDB, Inc.
+# Copyright (C) 2014-2018 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/collection/view/builder.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2017 MongoDB, Inc.
+# Copyright (C) 2015-2018 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/collection/view/builder/aggregation.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2017 MongoDB, Inc.
+# Copyright (C) 2015-2018 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -29,8 +29,8 @@ module Mongo
           MAPPINGS = BSON::Document.new(
             :allow_disk_use => 'allowDiskUse',
             :max_time_ms => 'maxTimeMS',
-            # This is intentional; max_await_time_ms is an alias for maxTimeMS used on getmore
-            # commands for change streams.
+            # This is intentional; max_await_time_ms is an alias for maxTimeMS
+            # used on getMore commands for change streams.
             :max_await_time_ms => 'maxTimeMS',
             :explain => 'explain',
             :bypass_document_validation => 'bypassDocumentValidation',

data/lib/mongo/collection/view/builder/find_command.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2017 MongoDB, Inc.
+# Copyright (C) 2015-2018 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/collection/view/builder/flags.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2017 MongoDB, Inc.
+# Copyright (C) 2015-2018 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/collection/view/builder/map_reduce.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2017 MongoDB, Inc.
+# Copyright (C) 2015-2018 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/collection/view/builder/modifiers.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2017 MongoDB, Inc.
+# Copyright (C) 2015-2018 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/collection/view/builder/op_query.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2017 MongoDB, Inc.
+# Copyright (C) 2015-2018 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/collection/view/change_stream.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2017 MongoDB, Inc.
+# Copyright (C) 2017-2018 MongoDB, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,14 +19,16 @@ module Mongo
     class View
 
       # Provides behaviour around a `$changeStream` pipeline stage in the
-      # aggregation framework. Specifying this stage allows users to request that
-      # notifications are sent for all changes to a particular collection or database.
+      # aggregation framework. Specifying this stage allows users to request
+      # that notifications are sent for all changes to a particular collection
+      # or database.
       #
       # @note Only available in server versions 3.6 and higher.
-      # @note ChangeStreams do not work properly with JRuby because of the issue documented
-      #  here: https://github.com/jruby/jruby/issues/4212
-      #  Namely, JRuby eagerly evaluates #next on an Enumerator in a background green thread.
-      #  So calling #next on the change stream will cause getmores to be called in a loop in the background.
+      # @note ChangeStreams do not work properly with JRuby because of the
+      #  issue documented here: https://github.com/jruby/jruby/issues/4212.
+      #  Namely, JRuby eagerly evaluates #next on an Enumerator in a background
+      #  green thread, therefore calling #next on the change stream will cause
+      #  getMores to be called in a loop in the background.
       #
       #
       # @since 2.5.0
@@ -38,6 +40,18 @@ module Mongo
         # @since 2.5.0
         FULL_DOCUMENT_DEFAULT = 'default'.freeze
 
+        # @return [ Symbol ] Used to indicate that the change stream should listen for changes on
+        #   the entire database rather than just the collection.
+        #
+        # @since 2.6.0
+        DATABASE = :database
+
+        # @return [ Symbol ] Used to indicate that the change stream should listen for changes on
+        #   the entire cluster rather than just the collection.
+        #
+        # @since 2.6.0
+        CLUSTER = :cluster
+
         # @return [ BSON::Document ] The change stream options.
         #
         # @since 2.5.0
@@ -53,8 +67,8 @@ module Mongo
         # @param [ Array<Hash> ] pipeline The pipeline of operators to filter the change notifications.
         # @param [ Hash ] options The change stream options.
         #
-        # @option options [ String ] :full_document Allowed values: ‘default’, ‘updateLookup’. Defaults to ‘default’.
-        #   When set to ‘updateLookup’, the change notification for partial updates will include both a delta
+        # @option options [ String ] :full_document Allowed values: 'default', 'updateLookup'. Defaults to 'default'.
+        #   When set to 'updateLookup', the change notification for partial updates will include both a delta
         #   describing the changes to the document, as well as a copy of the entire document that was changed
         #   from some time after the change occurred.
         # @option options [ BSON::Document, Hash ] :resume_after Specifies the logical starting point for the
@@ -63,18 +77,31 @@ module Mongo
         #   on new documents to satisfy a change stream query.
         # @option options [ Integer ] :batch_size The number of documents to return per batch.
         # @option options [ BSON::Document, Hash ] :collation The collation to use.
+        # @option options [ BSON::Timestamp ] :start_at_operation_time Only
+        #   return changes that occurred at or after the specified timestamp. Any
+        #   command run against the server will return a cluster time that can
+        #   be used here. Only recognized by server versions 4.0+.
         #
         # @since 2.5.0
-        def initialize(view, pipeline, options = {})
+        def initialize(view, pipeline, changes_for, options = {})
           @view = view
+          @changes_for = changes_for
           @change_stream_filters = pipeline && pipeline.dup
           @options = options && options.dup.freeze
           @resume_token = @options[:resume_after]
-          read_with_one_retry { create_cursor! }
+          create_cursor!
+
+          # We send different parameters when we resume a change stream
+          # compared to when we send the first query
+          @resuming = true
         end
 
         # Iterate through documents returned by the change stream.
         #
+        # This method retries once per error on resumable errors
+        # (two consecutive errors result in the second error being raised,
+        # an error which is recovered from resets the error count to zero).
+        #
         # @example Iterate through the stream of documents.
         #   stream.each do |document|
         #     p document
@@ -87,20 +114,82 @@ module Mongo
         # @yieldparam [ BSON::Document ] Each change stream document.
         def each
           raise StopIteration.new if closed?
+          retried = false
           begin
             @cursor.each do |doc|
               cache_resume_token(doc)
               yield doc
             end if block_given?
             @cursor.to_enum
-          rescue => e
+          rescue Mongo::Error => e
+            if retried || !e.change_stream_resumable?
+              raise
+            end
+
+            retried = true
+            # Rerun initial aggregation.
+            # Any errors here will stop iteration and break out of this
+            # method
             close
-            if retryable?(e)
+            create_cursor!
+            retry
+          end
+        end
+
+        # Return one document from the change stream, if one is available.
+        #
+        # Retries once on a resumable error.
+        #
+        # Raises StopIteration if the change stream is closed.
+        #
+        # This method will wait up to max_await_time_ms milliseconds
+        # for changes from the server, and if no changes are received
+        # it will return nil.
+        #
+        # @note This method is experimental and subject to change.
+        #
+        # @return [ BSON::Document | nil ] A change stream document.
+        # @api private
+        def try_next
+          raise StopIteration.new if closed?
+          retried = false
+
+          begin
+            doc = @cursor.try_next
+          rescue Mongo::Error => e
+            unless e.change_stream_resumable?
+              raise
+            end
+
+            if retried
+              # Rerun initial aggregation.
+              # Any errors here will stop iteration and break out of this
+              # method
+              close
               create_cursor!
+              retried = false
+            else
+              # Attempt to retry a getMore once
+              retried = true
               retry
             end
-            raise
           end
+
+          if doc
+            cache_resume_token(doc)
+          end
+          doc
+        end
+
+        def to_enum
+          enum = super
+          enum.send(:instance_variable_set, '@obj', self)
+          class << enum
+            def try_next
+              @obj.try_next
+            end
+          end
+          enum
         end
 
         # Close the change stream.
@@ -145,28 +234,115 @@ module Mongo
 
         private
 
+        def for_cluster?
+          @changes_for == CLUSTER
+        end
+
+        def for_database?
+          @changes_for == DATABASE
+        end
+
+        def for_collection?
+          !for_cluster? && !for_database?
+        end
+
         def cache_resume_token(doc)
+          # Always record both resume token and operation time,
+          # in case we get an older or newer server during rolling
+          # upgrades/downgrades
           unless @resume_token = (doc[:_id] && doc[:_id].dup)
-            raise Error::MissingResumeToken.new
+            raise Error::MissingResumeToken
           end
         end
 
         def create_cursor!
+          # clear the cache because we may get a newer or an older server
+          # (rolling upgrades)
+          @start_at_operation_time_supported = nil
+
           session = client.send(:get_session, @options)
           server = server_selector.select_server(cluster)
           result = send_initial_query(server, session)
+          if doc = result.replies.first && result.replies.first.documents.first
+            @start_at_operation_time = doc['operationTime']
+          else
+            # The above may set @start_at_operation_time to nil
+            # if it was not in the document for some reason,
+            # for consistency set it to nil here as well
+            @start_at_operation_time = nil
+          end
           @cursor = Cursor.new(view, result, server, disable_retry: true, session: session)
         end
 
         def pipeline
-          change_doc = { fullDocument: ( @options[:full_document] || FULL_DOCUMENT_DEFAULT ) }
-          change_doc[:resumeAfter] = @resume_token if @resume_token
           [{ '$changeStream' => change_doc }] + @change_stream_filters
         end
 
+        def aggregate_spec(session)
+          super(session).tap do |spec|
+            spec[:selector][:aggregate] = 1 unless for_collection?
+          end
+        end
+
+        def change_doc
+          { fullDocument: ( @options[:full_document] || FULL_DOCUMENT_DEFAULT ) }.tap do |doc|
+            if resuming?
+              # We have a resume token once we retrieved any documents.
+              # However, if the first getMore fails and the user didn't pass
+              # a resume token we won't have a resume token to use.
+              # Use start_at_operation time in this case
+              if @resume_token
+                # Spec says we need to remove startAtOperationTime if
+                # one was passed in by user, thus we won't forward it
+              elsif start_at_operation_time_supported? && @start_at_operation_time
+                # It is crucial to check @start_at_operation_time_supported
+                # here - we may have switched to an older server that
+                # does not support operation times and therefore shouldn't
+                # try to send one to it!
+                #
+                # @start_at_operation_time is already a BSON::Timestamp
+                doc[:startAtOperationTime] = @start_at_operation_time
+              else
+                # Can't resume if we don't have either
+                raise Mongo::Error::MissingResumeToken
+              end
+            else
+              if options[:start_at_operation_time]
+                doc[:startAtOperationTime] = time_to_bson_timestamp(
+                  options[:start_at_operation_time])
+              end
+            end
+            doc[:resumeAfter] = @resume_token if @resume_token
+            doc[:allChangesForCluster] = true if for_cluster?
+          end
+        end
+
         def send_initial_query(server, session)
           initial_query_op(session).execute(server)
         end
+
+        def time_to_bson_timestamp(time)
+          if time.is_a?(Time)
+            seconds = time.to_f
+            BSON::Timestamp.new(seconds.to_i, ((seconds - seconds.to_i) * 1000000).to_i)
+          elsif time.is_a?(BSON::Timestamp)
+            time
+          else
+            raise ArgumentError, 'Time must be a Time or a BSON::Timestamp instance'
+          end
+        end
+
+        def resuming?
+          !!@resuming
+        end
+
+        def start_at_operation_time_supported?
+          if @start_at_operation_time_supported.nil?
+            server = server_selector.select_server(cluster)
+            @start_at_operation_time_supported = server.description.max_wire_version >= 7
+          end
+          @start_at_operation_time_supported
+        end
       end
     end
   end