mongo 2.4.1 → 2.14.0

data/lib/mongo/cursor.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2016 MongoDB, Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,8 +19,9 @@ module Mongo
   # Client-side representation of an iterator over a query result set on
   # the server.
   #
-  # A +Cursor+ is not created directly by a user. Rather, +CollectionView+
-  # creates a +Cursor+ in an Enumerable module method.
+  # +Cursor+ objects are not directly exposed to application code. Rather,
+  # +Collection::View+ exposes the +Enumerable+ interface to the applications,
+  # and the enumerator is backed by a +Cursor+ instance.
   #
   # @example Get an array of 5 users named Emily.
   #   users.find({:name => 'Emily'}).limit(5).to_a
@@ -28,20 +29,25 @@ module Mongo
   # @example Call a block on each user doc.
   #   users.find.each { |doc| puts doc }
   #
-  # @note The +Cursor+ API is semipublic.
-  # @api semipublic
+  # @api private
   class Cursor
     extend Forwardable
     include Enumerable
     include Retryable
 
-    def_delegators :@view, :collection, :limit
+    def_delegators :@view, :collection
     def_delegators :collection, :client, :database
     def_delegators :@server, :cluster
 
     # @return [ Collection::View ] view The collection view.
     attr_reader :view
 
+    # The resume token tracked by the cursor for change stream resuming
+    #
+    # @return [ BSON::Document | nil ] The cursor resume token.
+    # @api private
+    attr_reader :resume_token
+
     # Creates a +Cursor+ object.
     #
     # @example Instantiate the cursor.
@@ -50,21 +56,39 @@ module Mongo
     # @param [ CollectionView ] view The +CollectionView+ defining the query.
     # @param [ Operation::Result ] result The result of the first execution.
     # @param [ Server ] server The server this cursor is locked to.
+    # @param [ Hash ] options The cursor options.
+    #
+    # @option options [ true, false ] :disable_retry Whether to disable
+    #   retrying on error when sending getMore operations (deprecated, getMore
+    #   operations are no longer retried)
+    # @option options [ true, false ] :retry_reads Retry reads (following
+    #   the modern mechanism), default is true
     #
     # @since 2.0.0
-    def initialize(view, result, server)
+    def initialize(view, result, server, options = {})
       @view = view
       @server = server
       @initial_result = result
+      @namespace = result.namespace
       @remaining = limit if limited?
       @cursor_id = result.cursor_id
-      register
-      ObjectSpace.define_finalizer(self, self.class.finalize(result.cursor_id,
-                                                             cluster,
-                                                             kill_cursors_op_spec,
-                                                             server))
+      if @cursor_id.nil?
+        raise ArgumentError, 'Cursor id must be present in the result'
+      end
+      @options = options
+      @session = @options[:session]
+      unless closed?
+        register
+        ObjectSpace.define_finalizer(self, self.class.finalize(@cursor_id,
+          cluster,
+          kill_cursors_op_spec,
+          server,
+          @session))
+      end
     end
 
+    # @api private
+    attr_reader :server
 
     # Finalize the cursor for garbage collection. Schedules this cursor to be included
     # in a killCursors operation executed by the Cluster's CursorReaper.
@@ -80,8 +104,11 @@ module Mongo
     # @return [ Proc ] The Finalizer.
     #
     # @since 2.3.0
-    def self.finalize(cursor_id, cluster, op_spec, server)
-      proc { cluster.schedule_kill_cursor(cursor_id, op_spec, server) }
+    def self.finalize(cursor_id, cluster, op_spec, server, session)
+      proc do
+        cluster.schedule_kill_cursor(cursor_id, op_spec, server)
+        session.end_session if session && session.implicit?
+      end
     end
 
     # Get a human-readable string representation of +Cursor+.
@@ -98,6 +125,10 @@ module Mongo
 
     # Iterate through documents returned from the query.
     #
+    # A cursor may be iterated at most once. Incomplete iteration is also
+    # allowed. Attempting to iterate the cursor more than once raises
+    # InvalidCursorOperation.
+    #
     # @example Iterate over the documents in the cursor.
     #   cursor.each do |doc|
     #     ...
@@ -107,13 +138,102 @@ module Mongo
     #
     # @since 2.0.0
     def each
-      process(@initial_result).each { |doc| yield doc }
-      while more?
-        return kill_cursors if exhausted?
-        get_more.each { |doc| yield doc }
+
+      # If we already iterated past the first batch (i.e., called get_more
+      # at least once), the cursor on the server side has advanced past
+      # the first batch and restarting iteration from the beginning by
+      # returning initial result would miss documents in the second batch
+      # and subsequent batches up to wherever the cursor is. Detect this
+      # condition and abort the iteration.
+      #
+      # In a future driver version, each would either continue from the
+      # end of previous iteration or would always restart from the
+      # beginning.
+      if @get_more_called
+        raise Error::InvalidCursorOperation, 'Cannot restart iteration of a cursor which issued a getMore'
+      end
+
+      # To maintain compatibility with pre-2.10 driver versions, reset
+      # the documents array each time a new iteration is started.
+      @documents = nil
+
+      if block_given?
+        # StopIteration raised by try_next ends this loop.
+        loop do
+          document = try_next
+          yield document if document
+        end
+        self
+      else
+        documents = []
+        # StopIteration raised by try_next ends this loop.
+        loop do
+          document = try_next
+          documents << document if document
+        end
+        documents
       end
     end
 
+    # Return one document from the query, if one is available.
+    #
+    # This method will wait up to max_await_time_ms milliseconds
+    # for changes from the server, and if no changes are received
+    # it will return nil. If there are no more documents to return
+    # from the server, or if we have exhausted the cursor, it will
+    # raise a StopIteration exception.
+    #
+    # @note This method is experimental and subject to change.
+    #
+    # @return [ BSON::Document | nil ] A document.
+    #
+    # @raise [ StopIteration ] Raised on the calls after the cursor had been
+    #   completely iterated.
+    #
+    # @api private
+    def try_next
+      if @documents.nil?
+        # Since published versions of Mongoid have a copy of old driver cursor
+        # code, our dup call in #process isn't invoked when Mongoid query
+        # cache is active. Work around that by also calling dup here on
+        # the result of #process which might come out of Mongoid's code.
+        @documents = process(@initial_result).dup
+        # the documents here can be an empty array, hence
+        # we may end up issuing a getMore in the first try_next call
+      end
+
+      if @documents.empty?
+        # On empty batches, we cache the batch resume token
+        cache_batch_resume_token
+
+        unless closed?
+          if exhausted?
+            close
+            raise StopIteration
+          end
+          @documents = get_more
+        else
+          raise StopIteration
+        end
+      else
+        # cursor is closed here
+        # keep documents as an empty array
+      end
+
+      # If there is at least one document, cache its _id
+      if @documents[0]
+        cache_resume_token(@documents[0])
+      end
+
+      # Cache the batch resume token if we are iterating
+      # over the last document, or if the batch is empty
+      if @documents.size <= 1
+        cache_batch_resume_token
+      end
+
+      return @documents.shift
+    end
+
     # Get the batch size.
     #
     # @example Get the batch size.
@@ -135,7 +255,28 @@ module Mongo
     #
     # @since 2.2.0
     def closed?
-      !more?
+      # @cursor_id should in principle never be nil
+      @cursor_id.nil? || @cursor_id == 0
+    end
+
+    # Closes this cursor, freeing any associated resources on the client and
+    # the server.
+    #
+    # @return [ nil ] Always nil.
+    #
+    # @raise [ Error::OperationFailure ] If the server cursor close fails.
+    def close
+      return if closed?
+
+      unregister
+      read_with_one_retry do
+        kill_cursors_operation.execute(@server, client: client)
+      end
+
+      nil
+    ensure
+      end_session
+      @cursor_id = 0
     end
 
     # Get the parsed collection name.
@@ -147,7 +288,20 @@ module Mongo
     #
     # @since 2.2.0
     def collection_name
-      @coll_name || collection.name
+      # In most cases, this will be equivalent to the name of the collection
+      # object in the driver. However, in some cases (e.g. when connected
+      # to an Atlas Data Lake), the namespace returned by the find command
+      # may be different, which is why we want to use the collection name based
+      # on the namespace in the command result.
+      if @namespace
+        # Often, the namespace will be in the format "database.collection".
+        # However, sometimes the collection name will contain periods, which
+        # is why this method joins all the namespace components after the first.
+        ns_components = @namespace.split('.')
+        ns_components[1...ns_components.length].join('.')
+      else
+        collection.name
+      end
     end
 
     # Get the cursor id.
@@ -177,43 +331,58 @@ module Mongo
       use_limit? ? @remaining : (batch_size || 0)
     end
 
+    # Execute a getMore command and return the batch of documents
+    # obtained from the server.
+    #
+    # @return [ Array<BSON::Document> ] The batch of documents
+    #
+    # @api private
+    def get_more
+      @get_more_called = true
+
+      # Modern retryable reads specification prohibits retrying getMores.
+      # Legacy retryable read logic used to retry getMores, but since
+      # doing so may result in silent data loss, the driver no longer retries
+      # getMore operations in any circumstance.
+      # https://github.com/mongodb/specifications/blob/master/source/retryable-reads/retryable-reads.rst#qa
+      process(get_more_operation.execute(@server, client: client))
+    end
+
     private
 
     def exhausted?
       limited? ? @remaining <= 0 : false
     end
 
-    def get_more
-      read_with_retry do
-        process(get_more_operation.execute(@server))
+    def cache_resume_token(doc)
+      if doc[:_id] && doc[:_id].is_a?(Hash)
+        @resume_token = doc[:_id] && doc[:_id].dup.freeze
       end
     end
 
+    def cache_batch_resume_token
+      @resume_token = @post_batch_resume_token if @post_batch_resume_token
+    end
+
     def get_more_operation
-      if @server.features.find_command_enabled?
-        Operation::Commands::GetMore.new(Builder::GetMoreCommand.new(self).specification)
+      if @server.with_connection { |connection| connection.features }.find_command_enabled?
+        spec = Builder::GetMoreCommand.new(self, @session).specification
       else
-        Operation::Read::GetMore.new(Builder::OpGetMore.new(self).specification)
+        spec = Builder::OpGetMore.new(self).specification
       end
+      Operation::GetMore.new(spec)
     end
 
-    def kill_cursors
-      unregister
-      read_with_one_retry do
-        kill_cursors_operation.execute(@server)
-      end
+    def end_session
+      @session.end_session if @session && @session.implicit?
     end
 
     def kill_cursors_operation
-      if @server.features.find_command_enabled?
-        Operation::Commands::Command.new(kill_cursors_op_spec)
-      else
-        Operation::KillCursors.new(kill_cursors_op_spec)
-      end
+      Operation::KillCursors.new(kill_cursors_op_spec)
     end
 
     def kill_cursors_op_spec
-      if @server.features.find_command_enabled?
+      if @server.with_connection { |connection| connection.features }.find_command_enabled?
         Builder::KillCursorsCommand.new(self).specification
       else
         Builder::OpKillCursors.new(self).specification
@@ -224,22 +393,35 @@ module Mongo
       limit ? limit > 0 : false
     end
 
-    def more?
-      @cursor_id != 0
-    end
-
     def process(result)
       @remaining -= result.returned_count if limited?
-      @coll_name ||= result.namespace.sub("#{database.name}.", '') if result.namespace
-      unregister if result.cursor_id == 0
+      # #process is called for the first batch of results. In this case
+      # the @cursor_id may be zero (all results fit in the first batch).
+      # Thus we need to check both @cursor_id and the cursor_id of the result
+      # prior to calling unregister here.
+      unregister if !closed? && result.cursor_id == 0
       @cursor_id = result.cursor_id
-      result.documents
+
+      if result.respond_to?(:post_batch_resume_token)
+        @post_batch_resume_token = result.post_batch_resume_token
+      end
+
+      end_session if closed?
+
+      # Since our iteration code mutates the documents array by calling #shift
+      # on it, duplicate the documents here to permit restarting iteration
+      # from the beginning of the cursor as long as get_more was not called
+      result.documents.dup
     end
 
     def use_limit?
       limited? && batch_size >= @remaining
     end
 
+    def limit
+      @view.send(:limit)
+    end
+
     def register
       cluster.register_cursor(@cursor_id)
     end

data/lib/mongo/database/view.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2016 MongoDB, Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,8 +21,11 @@ module Mongo
     class View
       extend Forwardable
       include Enumerable
+      include Retryable
 
-      def_delegators :@database, :cluster, :read_preference
+      def_delegators :@database, :cluster, :read_preference, :client
+      # @api private
+      def_delegators :@database, :server_selector, :read_concern
       def_delegators :cluster, :next_primary
 
       # @return [ Integer ] batch_size The size of the batch of results
@@ -35,43 +38,72 @@ module Mongo
       # @return [ Collection ] collection The command collection.
       attr_reader :collection
 
-      # Get all the names of the non system collections in the database.
+      # Get all the names of the non-system collections in the database.
       #
-      # @example Get the collection names.
-      #   database.collection_names
+      # @note The set of returned collection names depends on the version of
+      #   MongoDB server that fulfills the request.
       #
       # @param [ Hash ] options Options for the listCollections command.
       #
       # @option options [ Integer ] :batch_size  The batch size for results
       #   returned from the listCollections command.
+      # @option options [ Hash ] :filter A filter on the collections returned.
+      # @option options [ true, false ] :authorized_collections A flag, when
+      #   set to true, that allows a user without the required privilege
+      #   to run the command when access control is enforced
+      #
+      #   See https://docs.mongodb.com/manual/reference/command/listCollections/
+      #   for more information and usage.
       #
       # @return [ Array<String> ] The names of all non-system collections.
       #
       # @since 2.0.0
       def collection_names(options = {})
         @batch_size = options[:batch_size]
-        server = next_primary(false)
-        @limit = -1 if server.features.list_collections_enabled?
-        collections_info(server).collect do |info|
-          if server.features.list_collections_enabled?
-            info[Database::NAME]
+        session = client.send(:get_session, options)
+        cursor = read_with_retry_cursor(session, ServerSelector.primary, self) do |server|
+          send_initial_query(server, session, options.merge(name_only: true))
+        end
+        cursor.map do |info|
+          if cursor.server.with_connection { |connection| connection.features }.list_collections_enabled?
+            info['name']
           else
-            (info[Database::NAME] &&
-              info[Database::NAME].sub("#{@database.name}.", ''))
+            (info['name'] &&
+              info['name'].sub("#{@database.name}.", ''))
           end
+        end.reject do |name|
+          name.start_with?('system.') || name.include?('$')
         end
       end
 
       # Get info on all the collections in the database.
       #
+      # @note The set of collections returned, and the schema of the
+      #   information hash per collection, depends on the MongoDB server
+      #   version that fulfills the request.
+      #
       # @example Get info on each collection.
       #   database.list_collections
       #
+      # @param [ Hash ] options
+      #
+      # @option options [ Hash ] :filter A filter on the collections returned.
+      # @option options [ true, false ] :name_only Indicates whether command
+      #   should return just collection/view names and type or return both the
+      #   name and other information
+      # @option options [ true, false ] :authorized_collections A flag, when
+      #   set to true and used with nameOnly: true, that allows a user without the
+      #   required privilege to run the command when access control is enforced
+      #
+      #   See https://docs.mongodb.com/manual/reference/command/listCollections/
+      #   for more information and usage.
+      #
       # @return [ Array<Hash> ] Info for each collection in the database.
       #
       # @since 2.0.5
-      def list_collections
-        collections_info(next_primary(false))
+      def list_collections(options = {})
+        session = client.send(:get_session)
+        collections_info(session, ServerSelector.primary, options)
       end
 
       # Create the new database view.
@@ -89,29 +121,75 @@ module Mongo
         @collection = @database[Database::COMMAND]
       end
 
+      # @api private
+      attr_reader :database
+
+      # Execute an aggregation on the database view.
+      #
+      # @example Aggregate documents.
+      #   view.aggregate([
+      #     { "$listLocalSessions" => {} }
+      #   ])
+      #
+      # @param [ Array<Hash> ] pipeline The aggregation pipeline.
+      # @param [ Hash ] options The aggregation options.
+      #
+      # @return [ Aggregation ] The aggregation object.
+      #
+      # @since 2.10.0
+      # @api private
+      def aggregate(pipeline, options = {})
+        Collection::View::Aggregation.new(self, pipeline, options)
+      end
+
       private
 
-      def collections_info(server, &block)
-        cursor = Cursor.new(self, send_initial_query(server), server).to_enum
-        cursor.each do |doc|
-          yield doc
-        end if block_given?
-        cursor
+      def collections_info(session, server_selector, options = {}, &block)
+        description = nil
+        cursor = read_with_retry_cursor(session, server_selector, self) do |server|
+          # TODO take description from the connection used to send the query
+          # once https://jira.mongodb.org/browse/RUBY-1601 is fixed.
+          description = server.description
+          send_initial_query(server, session, options)
+        end
+        # On 3.0+ servers, we get just the collection names.
+        # On 2.6 server, we get collection names prefixed with the database
+        # name. We need to filter system collections out here because
+        # in the caller we don't know which server version executed the
+        # command and thus what the proper filtering logic should be
+        # (it is valid for collection names to have dots, thus filtering out
+        # collections named system.* here for 2.6 servers would actually
+        # filter out collections in the system database).
+        if description.server_version_gte?('3.0')
+          cursor.reject do |doc|
+            doc['name'].start_with?('system.') || doc['name'].include?('$')
+          end
+        else
+          docs = cursor.reject do |doc|
+            doc['name'].start_with?("#{database.name}.system") || doc['name'].include?('$')
+          end
+        end
       end
 
-      def collections_info_spec
+      def collections_info_spec(session, options = {})
         { selector: {
             listCollections: 1,
             cursor: batch_size ? { batchSize: batch_size } : {} },
-          db_name: @database.name }
+          db_name: @database.name,
+          session: session
+        }.tap do |spec|
+          spec[:selector][:nameOnly] = true if options[:name_only]
+          spec[:selector][:filter] = options[:filter] if options[:filter]
+          spec[:selector][:authorizedCollections] = true if options[:authorized_collections]
+        end
       end
 
-      def initial_query_op
-        Operation::Commands::CollectionsInfo.new(collections_info_spec)
+      def initial_query_op(session, options = {})
+        Operation::CollectionsInfo.new(collections_info_spec(session, options))
       end
 
-      def send_initial_query(server)
-        initial_query_op.execute(server)
+      def send_initial_query(server, session, options = {})
+        initial_query_op(session, options).execute(server, client: client)
       end
     end
   end