mongo 2.6.2 → 2.16.0

data/lib/mongo/socket.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2014-2018 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,28 +15,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require 'socket'
-require 'timeout'
 require 'mongo/socket/ssl'
 require 'mongo/socket/tcp'
 require 'mongo/socket/unix'
+require 'mongo/socket/ocsp_verifier'
+require 'mongo/socket/ocsp_cache'
 
 module Mongo
 
   # Provides additional data around sockets for the driver's use.
   #
   # @since 2.0.0
+  # @api private
   class Socket
     include ::Socket::Constants
 
-    # Error message for SSL related exceptions.
+    # Error message for TLS related exceptions.
     #
     # @since 2.0.0
-    SSL_ERROR = 'MongoDB may not be configured with SSL support'.freeze
+    # @deprecated
+    SSL_ERROR = 'MongoDB may not be configured with TLS support'.freeze
 
     # Error message for timeouts on socket calls.
     #
     # @since 2.0.0
+    # @deprecated
     TIMEOUT_ERROR = 'Socket request timed out'.freeze
 
     # The pack directive for timeouts.
@@ -41,12 +47,82 @@ module Mongo
     # @since 2.0.0
     TIMEOUT_PACK = 'l_2'.freeze
 
+    # Write data to the socket in chunks of this size.
+    #
+    # @api private
+    WRITE_CHUNK_SIZE = 65536
+
+    # Initializes common socket attributes.
+    #
+    # @param [ Float ] timeout The socket timeout value.
+    # @param [ Hash ] options The options.
+    #
+    # @option options [ Float ] :connect_timeout Connect timeout.
+    # @option options [ Address ] :connection_address Address of the
+    #   connection that created this socket.
+    # @option options [ Integer ] :connection_generation Generation of the
+    #   connection (for non-monitoring connections) that created this socket.
+    # @option options [ true | false ] :monitor Whether this socket was
+    #   created by a monitoring connection.
+    #
+    # @api private
+    def initialize(timeout, options)
+      @timeout = timeout
+      @options = options
+    end
+
     # @return [ Integer ] family The type of host family.
     attr_reader :family
 
     # @return [ Socket ] socket The wrapped socket.
     attr_reader :socket
 
+    # @return [ Hash ] The options.
+    attr_reader :options
+
+    # @return [ Float ] timeout The socket timeout.
+    attr_reader :timeout
+
+    # @return [ Address ] Address of the connection that created this socket.
+    #
+    # @api private
+    def connection_address
+      options[:connection_address]
+    end
+
+    # @return [ Integer ] Generation of the connection (for non-monitoring
+    #   connections) that created this socket.
+    #
+    # @api private
+    def connection_generation
+      options[:connection_generation]
+    end
+
+    # @return [ true | false ] Whether this socket was created by a monitoring
+    #   connection.
+    #
+    # @api private
+    def monitor?
+      !!options[:monitor]
+    end
+
+    # @return [ String ] Human-readable summary of the socket for debugging.
+    #
+    # @api private
+    def summary
+      fileno = @socket&.fileno rescue '<no socket>' || '<no socket>'
+      if monitor?
+        indicator = if options[:push]
+          'pm'
+        else
+          'm'
+        end
+        "#{connection_address};#{indicator};fd=#{fileno}"
+      else
+        "#{connection_address};c:#{connection_generation};fd=#{fileno}"
+      end
+    end
+
     # Is the socket connection alive?
     #
     # @example Is the socket alive?
@@ -58,7 +134,16 @@ module Mongo
     def alive?
       sock_arr = [ @socket ]
       if Kernel::select(sock_arr, nil, sock_arr, 0)
-        eof?
+        # The eof? call is supposed to return immediately since select
+        # indicated the socket is readable. However, if @socket is a TLS
+        # socket, eof? can block anyway - see RUBY-2140.
+        begin
+          Timeout.timeout(0.1) do
+            eof?
+          end
+        rescue ::Timeout::Error
+          true
+        end
       else
         true
       end
@@ -73,7 +158,14 @@ module Mongo
     #
     # @since 2.0.0
     def close
-      @socket.close rescue true
+      begin
+        # Sometimes it seems the close call can hang for a long time
+        ::Timeout.timeout(5) do
+          @socket.close
+        end
+      rescue
+        # Silence all errors
+      end
       true
     end
 
@@ -88,21 +180,9 @@ module Mongo
     #
     # @since 2.0.0
     def gets(*args)
-      handle_errors { @socket.gets(*args) }
-    end
-
-    # Create the new socket for the provided family - ipv4, piv6, or unix.
-    #
-    # @example Create a new ipv4 socket.
-    #   Socket.new(Socket::PF_INET)
-    #
-    # @param [ Integer ] family The socket domain.
-    #
-    # @since 2.0.0
-    def initialize(family)
-      @family = family
-      @socket = ::Socket.new(family, SOCK_STREAM, 0)
-      set_socket_options(@socket)
+      map_exceptions do
+        @socket.gets(*args)
+      end
     end
 
     # Will read all data from the socket for the provided number of bytes.
@@ -112,19 +192,24 @@ module Mongo
     #   socket.read(4096)
     #
     # @param [ Integer ] length The number of bytes to read.
+    # @param [ Numeric ] timeout The timeout to use for each chunk read.
     #
     # @raise [ Mongo::SocketError ] If not all data is returned.
     #
     # @return [ Object ] The data from the socket.
     #
     # @since 2.0.0
-    def read(length)
-      handle_errors do
-        data = read_from_socket(length)
-        raise IOError unless (data.length > 0 || length == 0)
+    def read(length, timeout: nil)
+      map_exceptions do
+        data = read_from_socket(length, timeout: timeout)
+        unless (data.length > 0 || length == 0)
+          raise IOError, "Expected to read > 0 bytes but read 0 bytes"
+        end
         while data.length < length
-          chunk = read_from_socket(length - data.length)
-          raise IOError unless (chunk.length > 0 || length == 0)
+          chunk = read_from_socket(length - data.length, timeout: timeout)
+          unless (chunk.length > 0 || length == 0)
+            raise IOError, "Expected to read > 0 bytes but read 0 bytes"
+          end
           data << chunk
         end
         data
@@ -140,21 +225,24 @@ module Mongo
     #
     # @since 2.0.0
     def readbyte
-      handle_errors { @socket.readbyte }
+      map_exceptions do
+        @socket.readbyte
+      end
     end
 
     # Writes data to the socket instance.
     #
-    # @example Write to the socket.
-    #   socket.write(data)
-    #
     # @param [ Array<Object> ] args The data to be written.
     #
     # @return [ Integer ] The length of bytes written to the socket.
     #
+    # @raise [ Error::SocketError | Error::SocketTimeoutError ] When there is a network error during the write.
+    #
     # @since 2.0.0
     def write(*args)
-      handle_errors { @socket.write(*args) }
+      map_exceptions do
+        do_write(*args)
+      end
     end
 
     # Tests if this socket has reached EOF. Primarily used for liveness checks.
@@ -162,19 +250,35 @@ module Mongo
     # @since 2.0.5
     def eof?
       @socket.eof?
-    rescue IOError, SystemCallError => _
+    rescue IOError, SystemCallError
+      true
+    end
+
+    # For backwards compatibilty only, do not use.
+    #
+    # @return [ true ] Always true.
+    #
+    # @deprecated
+    def connectable?
       true
     end
 
     private
 
-    def read_from_socket(length)
+    def read_from_socket(length, timeout: nil)
       # Just in case
       if length == 0
         return ''.force_encoding('BINARY')
       end
 
-      deadline = (Time.now + timeout) if timeout
+      _timeout = timeout || self.timeout
+      if _timeout
+        if _timeout > 0
+          deadline = Utils.monotonic_time + _timeout
+        elsif _timeout < 0
+          raise Errno::ETIMEDOUT, "Negative timeout #{_timeout} given to socket"
+        end
+      end
 
       # We want to have a fixed and reasonably small size buffer for reads
       # because, for example, OpenSSL reads in 16 kb chunks max.
@@ -189,7 +293,7 @@ module Mongo
         buf_size = length
       end
 
-      # The binary encoding is important, otherwise ruby performs encoding
+      # The binary encoding is important, otherwise Ruby performs encoding
       # conversions of some sort during the write into the buffer which
       # kills performance
       buf = allocate_string(buf_size)
@@ -221,10 +325,35 @@ module Mongo
           data[retrieved, chunk.length] = chunk
           retrieved += chunk.length
         end
-      rescue IO::WaitReadable
-        select_timeout = (deadline - Time.now) if deadline
-        if (select_timeout && select_timeout <= 0) || !Kernel::select([@socket], nil, [@socket], select_timeout)
-          raise Timeout::Error.new("Took more than #{timeout} seconds to receive data.")
+      # As explained in https://ruby-doc.com/core-trunk/IO.html#method-c-select,
+      # reading from a TLS socket may require writing which may raise WaitWritable
+      rescue IO::WaitReadable, IO::WaitWritable => exc
+        if deadline
+          select_timeout = deadline - Utils.monotonic_time
+          if select_timeout <= 0
+            raise Errno::ETIMEDOUT, "Took more than #{_timeout} seconds to receive data"
+          end
+        end
+        if exc.is_a?(IO::WaitReadable)
+          select_args = [[@socket], nil, [@socket], select_timeout]
+        else
+          select_args = [nil, [@socket], [@socket], select_timeout]
+        end
+        rv = Kernel.select(*select_args)
+        if BSON::Environment.jruby?
+          # Ignore the return value of Kernel.select.
+          # On JRuby, select appears to return nil prior to timeout expiration
+          # (apparently due to a EAGAIN) which then causes us to fail the read
+          # even though we could have retried it.
+          # Check the deadline ourselves.
+          if deadline
+            select_timeout = deadline - Utils.monotonic_time
+            if select_timeout <= 0
+              raise Errno::ETIMEDOUT, "Took more than #{_timeout} seconds to receive data"
+            end
+          end
+        elsif rv.nil?
+          raise Errno::ETIMEDOUT, "Took more than #{_timeout} seconds to receive data (select call timed out)"
         end
         retry
       end
@@ -241,11 +370,44 @@ module Mongo
     end
 
     def read_buffer_size
-      # Buffer size for non-SSL reads
+      # Buffer size for non-TLS reads
       # 64kb
       65536
     end
 
+    # Writes data to the socket instance.
+    #
+    # This is a separate method from +write+ for ease of mocking in the tests.
+    # This method should not perform any exception mapping, upstream code
+    # sholud map exceptions.
+    #
+    # @param [ Array<Object> ] args The data to be written.
+    #
+    # @return [ Integer ] The length of bytes written to the socket.
+    def do_write(*args)
+      # This method used to forward arguments to @socket.write in a
+      # single call like so:
+      #
+      # @socket.write(*args)
+      #
+      # Turns out, when each buffer to be written is large (e.g. 32 MiB),
+      # this write call would take an extremely long time (20+ seconds)
+      # while using 100% CPU. Splitting the writes into chunks produced
+      # massively better performance (0.05 seconds to write the 32 MiB of
+      # data on the same hardware). Unfortunately splitting the data,
+      # one would assume, results in it being copied, but this seems to be
+      # a much more minor issue compared to CPU cost of writing large buffers.
+      args.each do |buf|
+        buf = buf.to_s
+        i = 0
+        while i < buf.length
+          chunk = buf[i...i+WRITE_CHUNK_SIZE]
+          @socket.write(chunk)
+          i += WRITE_CHUNK_SIZE
+        end
+      end
+    end
+
     def unix_socket?(sock)
       defined?(UNIXSocket) && sock.is_a?(UNIXSocket)
     end
@@ -254,7 +416,7 @@ module Mongo
 
     DEFAULT_TCP_KEEPCNT = 9
 
-    DEFAULT_TCP_KEEPIDLE = 300
+    DEFAULT_TCP_KEEPIDLE = 120
 
     def set_keepalive_opts(sock)
       sock.setsockopt(SOL_SOCKET, SO_KEEPALIVE, true)
@@ -262,6 +424,10 @@ module Mongo
       set_option(sock, :TCP_KEEPCNT, DEFAULT_TCP_KEEPCNT)
       set_option(sock, :TCP_KEEPIDLE, DEFAULT_TCP_KEEPIDLE)
     rescue
+      # JRuby 9.2.13.0 and lower do not define TCP_KEEPINTVL etc. constants.
+      # JRuby 9.2.14.0 defines the constants but does not allow to get or
+      # set them with this error:
+      # Errno::ENOPROTOOPT: Protocol not available - Protocol not available
     end
 
     def set_option(sock, option, default)
@@ -278,16 +444,20 @@ module Mongo
       set_keepalive_opts(sock)
     end
 
-    def handle_errors
+    def map_exceptions
       begin
         yield
-      rescue Errno::ETIMEDOUT
-        raise Error::SocketTimeoutError, TIMEOUT_ERROR
+      rescue Errno::ETIMEDOUT => e
+        raise Error::SocketTimeoutError, "#{e.class}: #{e} (for #{human_address})"
       rescue IOError, SystemCallError => e
-        raise Error::SocketError, "#{e.class}: #{e}"
+        raise Error::SocketError, "#{e.class}: #{e} (for #{human_address})"
       rescue OpenSSL::SSL::SSLError => e
-        raise Error::SocketError, "#{e.class}: #{e} (#{SSL_ERROR})"
+        raise Error::SocketError, "#{e.class}: #{e} (for #{human_address})"
       end
     end
+
+    def human_address
+      raise NotImplementedError
+    end
   end
 end

data/lib/mongo/srv/monitor.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2014-2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  module Srv
+
+    # Periodically retrieves SRV records for the cluster's SRV URI, and
+    # sets the cluster's server list to the SRV lookup result.
+    #
+    # If an error is encountered during SRV lookup or an SRV record is invalid
+    # or disallowed for security reasons, a warning is logged and monitoring
+    # continues.
+    #
+    # @api private
+    class Monitor
+      include Loggable
+      include BackgroundThread
+
+      MIN_SCAN_INTERVAL = 60
+
+      DEFAULT_TIMEOUT = 10
+
+      # Creates the SRV monitor.
+      #
+      # @param [ Cluster ] cluster The cluster.
+      #
+      # @option opts [ Float ] :timeout The timeout to use for DNS lookups.
+      # @option opts [ URI::SRVProtocol ] :srv_uri The SRV URI to monitor.
+      # @option opts [ Hash ] :resolv_options For internal driver use only.
+      #   Options to pass through to Resolv::DNS constructor for SRV lookups.
+      def initialize(cluster, **opts)
+        @cluster = cluster
+        unless @srv_uri = opts.delete(:srv_uri)
+          raise ArgumentError, 'SRV URI is required'
+        end
+        @options = opts.freeze
+        @resolver = Srv::Resolver.new(**opts)
+        @last_result = @srv_uri.srv_result
+        @stop_semaphore = Semaphore.new
+      end
+
+      attr_reader :options
+
+      attr_reader :cluster
+
+      # @return [ Srv::Result ] Last known SRV lookup result. Used for
+      #   determining intervals between SRV lookups, which depend on SRV DNS
+      #   records' TTL values.
+      attr_reader :last_result
+
+      private
+
+      def do_work
+        scan!
+        @stop_semaphore.wait(scan_interval)
+      end
+
+      def scan!
+        old_hosts = last_result.address_strs
+
+        begin
+          last_result = Timeout.timeout(timeout) do
+            @resolver.get_records(@srv_uri.query_hostname)
+          end
+        rescue Resolv::ResolvTimeout => e
+          log_warn("SRV monitor: timed out trying to resolve hostname #{@srv_uri.query_hostname}: #{e.class}: #{e}")
+          return
+        rescue ::Timeout::Error
+          log_warn("SRV monitor: timed out trying to resolve hostname #{@srv_uri.query_hostname} (timeout=#{timeout})")
+          return
+        rescue Resolv::ResolvError => e
+          log_warn("SRV monitor: unable to resolve hostname #{@srv_uri.query_hostname}: #{e.class}: #{e}")
+          return
+        end
+
+        if last_result.empty?
+          log_warn("SRV monitor: hostname #{@srv_uri.query_hostname} resolved to zero records")
+          return
+        end
+
+        @cluster.set_server_list(last_result.address_strs)
+      end
+
+      def scan_interval
+        if last_result.empty?
+          [cluster.heartbeat_interval, MIN_SCAN_INTERVAL].min
+        elsif last_result.min_ttl.nil?
+          MIN_SCAN_INTERVAL
+        else
+          [last_result.min_ttl, MIN_SCAN_INTERVAL].max
+        end
+      end
+
+      def timeout
+        options[:timeout] || DEFAULT_TIMEOUT
+      end
+    end
+  end
+end

data/lib/mongo/srv/resolver.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2017-2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  module Srv
+
+    # Encapsulates the necessary behavior for querying SRV records as
+    # required by the driver.
+    #
+    # @api private
+    class Resolver
+      include Loggable
+
+      # @return [ String ] RECORD_PREFIX The prefix prepended to each hostname
+      #   before querying SRV records.
+      RECORD_PREFIX = '_mongodb._tcp.'.freeze
+
+      # Creates a new Resolver.
+      #
+      # @option opts [ Float ] :timeout The timeout, in seconds, to use for
+      #   each DNS record resolution.
+      # @option opts [ Boolean ] :raise_on_invalid Whether or not to raise
+      #   an exception if either a record with a mismatched domain is found
+      #   or if no records are found. Defaults to true.
+      # @option opts [ Hash ] :resolv_options For internal driver use only.
+      #   Options to pass through to Resolv::DNS constructor for SRV lookups.
+      def initialize(**opts)
+        @options = opts.freeze
+        @resolver = Resolv::DNS.new(@options[:resolv_options])
+        @resolver.timeouts = timeout
+      end
+
+      # @return [ Hash ] Resolver options.
+      attr_reader :options
+
+      def timeout
+        options[:timeout] || Monitor::DEFAULT_TIMEOUT
+      end
+
+      # Obtains all of the SRV records for a given hostname.
+      #
+      # In the event that a record with a mismatched domain is found or no
+      # records are found, if the :raise_on_invalid option is true,
+      # an exception will be raised, otherwise a warning will be logged.
+      #
+      # @param [ String ] hostname The hostname whose records should be obtained.
+      #
+      # @raise [ Mongo::Error::MismatchedDomain ] If the :raise_in_invalid
+      #   Resolver option is true and a record with a domain name that does
+      #   not match the hostname's is found.
+      # @raise [ Mongo::Error::NoSRVRecords ] If the :raise_in_invalid Resolver
+      #   option is true and no records are found.
+      #
+      # @return [ Mongo::Srv::Result ] SRV lookup result.
+      def get_records(hostname)
+        query_name = RECORD_PREFIX + hostname
+        resources = @resolver.getresources(query_name, Resolv::DNS::Resource::IN::SRV)
+
+        # Collect all of the records into a Result object, raising an error
+        # or logging a warning if a record with a mismatched domain is found.
+        # Note that in the case a warning is raised, the record is _not_
+        # added to the Result object.
+        result = Srv::Result.new(hostname)
+        resources.each do |record|
+          begin
+            result.add_record(record)
+          rescue Error::MismatchedDomain => e
+            if raise_on_invalid?
+              raise
+            else
+              log_warn(e.message)
+            end
+          end
+        end
+
+        # If no records are found, either raise an error or log a warning
+        # based on the Resolver's :raise_on_invalid option.
+        if result.empty?
+          if raise_on_invalid?
+            raise Error::NoSRVRecords.new(URI::SRVProtocol::NO_SRV_RECORDS % hostname)
+          else
+            log_warn(URI::SRVProtocol::NO_SRV_RECORDS % hostname)
+          end
+        end
+
+        result
+      end
+
+      # Obtains the TXT records of a host.
+      #
+      # @param [ String ] hostname The host whose TXT records should be obtained.
+      #
+      # @return [ nil | String ] URI options string from TXT record
+      #   associated with the hostname, or nil if there is no such record.
+      #
+      # @raise [ Mongo::Error::InvalidTXTRecord ] If more than one TXT record is found.
+      def get_txt_options_string(hostname)
+        records = @resolver.getresources(hostname, Resolv::DNS::Resource::IN::TXT)
+        if records.empty?
+          return nil
+        end
+
+        if records.length > 1
+          msg = "Only one TXT record is allowed: querying hostname #{hostname} returned #{records.length} records"
+
+          raise Error::InvalidTXTRecord, msg
+        end
+
+        records[0].strings.join
+      end
+
+      private
+
+      # Checks whether an error should be raised due to either a record with
+      # a mismatched domain being found or no records being found.
+      #
+      # @return [ Boolean ] Whether an error should be raised.
+      def raise_on_invalid?
+        @raise_on_invalid ||= @options[:raise_on_invalid] || true
+      end
+    end
+  end
+end