mongo 2.13.3 → 2.14.0.rc1

data/lib/mongo/uri.rb

@@ -388,6 +388,12 @@ module Mongo
       [ creds_hosts, db_opts ].map { |s| s.reverse }
     end
 
+    def options_mapper
+      @options_mapper ||= OptionsMapper.new(
+        logger: @options[:logger],
+      )
+    end
+
     def parse_uri_options!(string)
       uri_options = {}
       unless string
@@ -403,7 +409,7 @@ module Mongo
         end
         key = decode(key)
         value = decode(value)
-        add_uri_option(key, value, uri_options)
+        options_mapper.add_uri_option(key, value, uri_options)
       end
       uri_options
     end
@@ -453,393 +459,6 @@ module Mongo
       CGI.escape(value).gsub('+', '%20')
     end
 
-    # Hash for storing map of URI option parameters to conversion strategies
-    URI_OPTION_MAP = {}
-
-    # Simple internal dsl to register a MongoDB URI option in the URI_OPTION_MAP.
-    #
-    # @param uri_key [String] The MongoDB URI option to register.
-    # @param name [Symbol] The name of the option in the driver.
-    # @param extra [Hash] Extra options.
-    #   * :group [Symbol] Nested hash where option will go.
-    #   * :type [Symbol] Name of function to transform value.
-    def self.uri_option(uri_key, name, extra = {})
-      URI_OPTION_MAP[uri_key] = { :name => name }.merge(extra)
-    end
-
-    # Replica Set Options
-    uri_option 'replicaset', :replica_set
-
-    # Timeout Options
-    uri_option 'connecttimeoutms', :connect_timeout, :type => :ms
-    uri_option 'sockettimeoutms', :socket_timeout, :type => :ms
-    uri_option 'serverselectiontimeoutms', :server_selection_timeout, :type => :ms
-    uri_option 'localthresholdms', :local_threshold, :type => :ms
-    uri_option 'heartbeatfrequencyms', :heartbeat_frequency, :type => :ms
-    uri_option 'maxidletimems', :max_idle_time, :type => :ms
-
-    # Write Options
-    uri_option 'w', :w, :group => :write_concern, type: :w
-    uri_option 'journal', :j, :group => :write_concern, :type => :bool
-    uri_option 'fsync', :fsync, :group => :write_concern, type: :bool
-    uri_option 'wtimeoutms', :wtimeout, :group => :write_concern, :type => :integer
-
-    # Read Options
-    uri_option 'readpreference', :mode, :group => :read, :type => :read_mode
-    uri_option 'readpreferencetags', :tag_sets, :group => :read, :type => :read_tags
-    uri_option 'maxstalenessseconds', :max_staleness, :group => :read, :type => :max_staleness
-
-    # Pool options
-    uri_option 'minpoolsize', :min_pool_size, :type => :integer
-    uri_option 'maxpoolsize', :max_pool_size, :type => :integer
-    uri_option 'waitqueuetimeoutms', :wait_queue_timeout, :type => :ms
-
-    # Security Options
-    uri_option 'ssl', :ssl, :type => :repeated_bool
-    uri_option 'tls', :ssl, :type => :repeated_bool
-    uri_option 'tlsallowinvalidcertificates', :ssl_verify_certificate,
-               :type => :inverse_bool
-    uri_option 'tlsallowinvalidhostnames', :ssl_verify_hostname,
-               :type => :inverse_bool
-    uri_option 'tlscafile', :ssl_ca_cert
-    uri_option 'tlscertificatekeyfile', :ssl_cert
-    uri_option 'tlscertificatekeyfilepassword', :ssl_key_pass_phrase
-    uri_option 'tlsinsecure', :ssl_verify, :type => :inverse_bool
-
-    # Topology options
-    uri_option 'directconnection', :direct_connection, type: :bool
-    uri_option 'connect', :connect, type: :symbol
-
-    # Auth Options
-    uri_option 'authsource', :auth_source
-    uri_option 'authmechanism', :auth_mech, :type => :auth_mech
-    uri_option 'authmechanismproperties', :auth_mech_properties, :type => :auth_mech_props
-
-    # Client Options
-    uri_option 'appname', :app_name
-    uri_option 'compressors', :compressors, :type => :array
-    uri_option 'readconcernlevel', :level, group: :read_concern, type: :symbol
-    uri_option 'retryreads', :retry_reads, :type => :bool
-    uri_option 'retrywrites', :retry_writes, :type => :bool
-    uri_option 'zlibcompressionlevel', :zlib_compression_level, :type => :zlib_compression_level
-
-    # Applies URI value transformation by either using the default cast
-    # or a transformation appropriate for the given type.
-    #
-    # @param key [String] URI option name.
-    # @param value [String] The value to be transformed.
-    # @param type [Symbol] The transform method.
-    def apply_transform(key, value, type)
-      if type
-        if respond_to?("convert_#{type}", true)
-          send("convert_#{type}", key, value)
-        else
-          send(type, value)
-        end
-      else
-        value
-      end
-    end
-
-    # Selects the output destination for an option.
-    #
-    # @param [Hash] uri_options The base target.
-    # @param [Symbol] group Group subtarget.
-    #
-    # @return [Hash] The target for the option.
-    def select_target(uri_options, group = nil)
-      if group
-        uri_options[group] ||= {}
-      else
-        uri_options
-      end
-    end
-
-    # Merges a new option into the target.
-    #
-    # If the option exists at the target destination the merge will
-    # be an addition.
-    #
-    # Specifically required to append an additional tag set
-    # to the array of tag sets without overwriting the original.
-    #
-    # @param target [Hash] The destination.
-    # @param value [Object] The value to be merged.
-    # @param name [Symbol] The name of the option.
-    def merge_uri_option(target, value, name)
-      if target.key?(name)
-        if REPEATABLE_OPTIONS.include?(name)
-          target[name] += value
-        else
-          log_warn("Repeated option key: #{name}.")
-        end
-      else
-        target.merge!(name => value)
-      end
-    end
-
-    # Adds an option to the uri options hash via the supplied strategy.
-    #
-    #   Acquires a target for the option based on group.
-    #   Transforms the value.
-    #   Merges the option into the target.
-    #
-    # @param key [String] URI option name.
-    # @param value [String] The value of the option.
-    # @param uri_options [Hash] The base option target.
-    def add_uri_option(key, value, uri_options)
-      strategy = URI_OPTION_MAP[key.downcase]
-      if strategy.nil?
-        log_warn("Unsupported URI option '#{key}' on URI '#{@string}'. It will be ignored.")
-        return
-      end
-
-      target = select_target(uri_options, strategy[:group])
-      value = apply_transform(key, value, strategy[:type])
-      merge_uri_option(target, value, strategy[:name])
-    end
-
-    # Authentication mechanism transformation.
-    #
-    # @param value [String] The authentication mechanism.
-    #
-    # @return [Symbol] The transformed authentication mechanism.
-    def auth_mech(value)
-      (AUTH_MECH_MAP[value.upcase] || value).tap do |mech|
-        log_warn("#{value} is not a valid auth mechanism") unless mech
-      end
-    end
-
-    # Read preference mode transformation.
-    #
-    # @param value [String] The read mode string value.
-    #
-    # @return [Symbol] The read mode symbol.
-    def read_mode(value)
-      READ_MODE_MAP[value.downcase] || value
-    end
-
-    # Read preference tags transformation.
-    #
-    # @param value [String] The string representing tag set.
-    #
-    # @return [Array<Hash>] Array with tag set.
-    def read_tags(value)
-      [read_set(value)]
-    end
-
-    # Read preference tag set extractor.
-    #
-    # @param value [String] The tag set string.
-    #
-    # @return [Hash] The tag set hash.
-    def read_set(value)
-      hash_extractor('readPreferenceTags', value)
-    end
-
-    # Auth mechanism properties extractor.
-    #
-    # @param value [ String ] The auth mechanism properties string.
-    #
-    # @return [ Hash ] The auth mechanism properties hash.
-    def auth_mech_props(value)
-      properties = hash_extractor('authMechanismProperties', value)
-      if properties && properties[:canonicalize_host_name]
-        properties.merge!(canonicalize_host_name:
-          properties[:canonicalize_host_name].downcase == 'true')
-      end
-      properties
-    end
-
-    # Parses the zlib compression level.
-    #
-    # @param value [ String ] The zlib compression level string.
-    #
-    # @return [ Integer | nil ] The compression level value if it is between -1 and 9 (inclusive),
-    #   otherwise nil (and a warning will be logged).
-    def zlib_compression_level(value)
-      if /\A-?\d+\z/ =~ value
-        i = value.to_i
-
-        if i >= -1 && i <= 9
-          return i
-        end
-      end
-
-      log_warn("#{value} is not a valid zlibCompressionLevel")
-      nil
-    end
-
-    # Converts the value into a boolean and returns it wrapped in an array.
-    #
-    # @param name [ String ] Name of the URI option being processed.
-    # @param value [ String ] URI option value.
-    #
-    # @return [ Array<true | false> ] The boolean value parsed and wraped
-    #   in an array.
-    def convert_repeated_bool(name, value)
-      [convert_bool(name, value)]
-    end
-
-    # Converts +value+ into an integer.
-    #
-    # If the value is not a valid integer, warns and returns nil.
-    #
-    # @param name [ String ] Name of the URI option being processed.
-    # @param value [ String ] URI option value.
-    #
-    # @return [ nil | Integer ] Converted value.
-    def convert_integer(name, value)
-      unless /\A\d+\z/ =~ value
-        log_warn("#{value} is not a valid integer for #{name}")
-        return nil
-      end
-
-      value.to_i
-    end
-
-    # Converts +value+ into a symbol.
-    #
-    # @param name [ String ] Name of the URI option being processed.
-    # @param value [ String ] URI option value.
-    #
-    # @return [ Symbol ] Converted value.
-    def convert_symbol(name, value)
-      value.to_sym
-    end
-
-    # Converts +value+ as a write concern.
-    #
-    # If +value+ is the word "majority", returns the symbol :majority.
-    # If +value+ is a number, returns the number as an integer.
-    # Otherwise returns the string +value+ unchanged.
-    #
-    # @param name [ String ] Name of the URI option being processed.
-    # @param value [ String ] URI option value.
-    #
-    # @return [ Integer | Symbol | String ] Converted value.
-    def convert_w(name, value)
-      case value
-      when 'majority'
-        :majority
-      when /\A[0-9]+\z/
-        value.to_i
-      else
-        value
-      end
-    end
-
-    # Converts +value+ to a boolean.
-    #
-    # Returns true for 'true', false for 'false', otherwise nil.
-    #
-    # @param name [ String ] Name of the URI option being processed.
-    # @param value [ String ] URI option value.
-    #
-    # @return [ true | false | nil ] Converted value.
-    def convert_bool(name, value)
-      case value
-      when "true", 'TRUE'
-        true
-      when "false", 'FALSE'
-        false
-      else
-        log_warn("invalid boolean option for #{name}: #{value}")
-        nil
-      end
-    end
-
-    # Parses a boolean value and returns its inverse.
-    #
-    # @param value [ String ] The URI option value.
-    #
-    # @return [ true | false | nil ] The inverse of the  boolean value parsed out, otherwise nil
-    #   (and a warning will be logged).
-    def convert_inverse_bool(name, value)
-      b = convert_bool(name, value)
-
-      if b.nil?
-        nil
-      else
-        !b
-      end
-    end
-
-    # Parses the max staleness value, which must be either "0" or an integer greater or equal to 90.
-    #
-    # @param value [ String ] The max staleness string.
-    #
-    # @return [ Integer | nil ] The max staleness integer parsed out if it is valid, otherwise nil
-    #   (and a warning will be logged).
-    def max_staleness(value)
-      if /\A-?\d+\z/ =~ value
-        int = value.to_i
-
-        if int == -1
-          int = nil
-        end
-
-        if int && (int >= 0 && int < 90 || int < 0)
-          log_warn("max staleness should be either 0 or greater than 90: #{value}")
-        end
-
-        return int
-      end
-
-      log_warn("Invalid max staleness value: #{value}")
-      nil
-    end
-
-    # Ruby's convention is to provide timeouts in seconds, not milliseconds and
-    # to use fractions where more precision is necessary. The connection string
-    # options are always in MS so we provide an easy conversion type.
-    #
-    # @param [ Integer ] value The millisecond value.
-    #
-    # @return [ Float ] The seconds value.
-    #
-    # @since 2.0.0
-    def convert_ms(name, value)
-      unless /\A-?\d+(\.\d+)?\z/ =~ value
-        log_warn("Invalid ms value for #{name}: #{value}")
-        return nil
-      end
-
-      if value[0] == '-'
-        log_warn("#{name} cannot be a negative number")
-        return nil
-      end
-
-      value.to_f / 1000
-    end
-
-    # Extract values from the string and put them into a nested hash.
-    #
-    # @param value [ String ] The string to build a hash from.
-    #
-    # @return [ Hash ] The hash built from the string.
-    def hash_extractor(name, value)
-      h = {}
-      value.split(',').each do |tag|
-        k, v = tag.split(':')
-        if v.nil?
-          log_warn("Invalid hash value for #{name}: key `#{k}` does not have a value: #{value}")
-        end
-
-        h[k.downcase.to_sym] = v
-      end
-      h
-    end
-
-    # Extract values from the string and put them into an array.
-    #
-    # @param [ String ] value The string to build an array from.
-    #
-    # @return [ Array ] The array built from the string.
-    def array(value)
-      value.split(',')
-    end
-
     def validate_uri_options!
       # The URI options spec requires that we raise an error if there are conflicting values of
       # 'tls' and 'ssl'. In order to fulfill this, we parse the values of each instance into an
@@ -861,10 +480,21 @@ module Mongo
         unless uri_options[:ssl_verify_hostname].nil?
           raise_invalid_error_no_fmt!("tlsInsecure' and 'tlsAllowInvalidHostnames' cannot both be specified")
         end
+
+        unless uri_options[:ssl_verify_ocsp_endpoint].nil?
+          raise_invalid_error_no_fmt!("tlsInsecure' and 'tlsDisableOCSPEndpointCheck' cannot both be specified")
+        end
+      end
+
+      unless uri_options[:ssl_verify_certificate].nil?
+        unless uri_options[:ssl_verify_ocsp_endpoint].nil?
+          raise_invalid_error_no_fmt!("tlsAllowInvalidCertificates' and 'tlsDisableOCSPEndpointCheck' cannot both be specified")
+        end
       end
 
-      # Since we know that the only URI option that sets :ssl_cert is "tlsCertificateKeyFile", any
-      # value set for :ssl_cert must also be set for :ssl_key.
+      # Since we know that the only URI option that sets :ssl_cert is
+      # "tlsCertificateKeyFile", any value set for :ssl_cert must also be set
+      # for :ssl_key.
       if uri_options[:ssl_cert]
         uri_options[:ssl_key] = uri_options[:ssl_cert]
       end
@@ -891,4 +521,5 @@ module Mongo
   end
 end
 
+require 'mongo/uri/options_mapper'
 require 'mongo/uri/srv_protocol'

data/lib/mongo/utils.rb

@@ -34,7 +34,7 @@ module Mongo
     # @option opts [ Logger ] :logger A custom logger to use.
     # @option opts [ String ] :log_prefix A custom log prefix to use when
     #   logging.
-    module_function def warn_monitor_exception(msg, exc, **opts)
+    module_function def warn_bg_exception(msg, exc, **opts)
       bt_excerpt = excerpt_backtrace(exc, **opts)
       logger = LocalLogger.new(**opts)
       logger.log_warn("#{msg}: #{exc.class}: #{exc}#{bt_excerpt}")
@@ -55,8 +55,19 @@ module Mongo
       end
     end
 
+    # Symbolizes the keys in the provided hash.
     module_function def shallow_symbolize_keys(hash)
       Hash[hash.map { |k, v| [k.to_sym, v] }]
     end
+
+    # Stringifies the keys in the provided hash and converts underscore
+    # style keys to camel case style keys.
+    module_function def shallow_camelize_keys(hash)
+      Hash[hash.map { |k, v| [camelize(k), v] }]
+    end
+
+    module_function def camelize(sym)
+      sym.to_s.gsub(/_(\w)/) { $1.upcase }
+    end
   end
 end

data/lib/mongo/version.rb

@@ -17,5 +17,5 @@ module Mongo
   # The current version of the driver.
   #
   # @since 2.0.0
-  VERSION = '2.13.3'.freeze
+  VERSION = '2.14.0.rc1'.freeze
 end

data/lib/mongo.rb

@@ -51,6 +51,7 @@ require 'mongo/protocol'
 require 'mongo/background_thread'
 require 'mongo/cluster'
 require 'mongo/cursor'
+require 'mongo/caching_cursor'
 require 'mongo/collection'
 require 'mongo/database'
 require 'mongo/crypt'
@@ -60,6 +61,7 @@ require 'mongo/dbref'
 require 'mongo/grid'
 require 'mongo/index'
 require 'mongo/lint'
+require 'mongo/query_cache'
 require 'mongo/server'
 require 'mongo/server_selector'
 require 'mongo/session'
@@ -70,3 +72,10 @@ require 'mongo/uri'
 require 'mongo/version'
 require 'mongo/write_concern'
 require 'mongo/utils'
+
+module Mongo
+  # Clears the driver's OCSP response cache.
+  module_function def clear_ocsp_cache
+    Socket::OcspCache.clear
+  end
+end

data/spec/NOTES.aws-auth.md

@@ -145,13 +145,16 @@ problem. Below are some of the puzzling responses I encountered:
   line) but the value is otherwise completely valid. This error has no relation
   to the "session token" or "security token" as used with temporary AWS
   credentials.
-- *The security token included in the request is invalid*: this error is
-  produced when the AWS access key id, as specified in the scope part of the
-  `Authorization` header, is not a valid access key id. In the case of
-  non-temporary credentials being used for authentication, the error refers to
-  a "security token" but the authentication process does not actually use a
-  security token as this term is used in the AWS documentation describing
-  temporary credentials.
+- *The security token included in the request is invalid*: this error can be
+  produced in several circumstances:
+  - When the AWS access key id, as specified in the scope part of the
+    `Authorization` header, is not a valid access key id. In the case of
+    non-temporary credentials being used for authentication, the error refers to
+    a "security token" but the authentication process does not actually use a
+    security token as this term is used in the AWS documentation describing
+    temporary credentials.
+  - When using temporary credentials and the security token is not provided
+    in the STS request at all (x-amz-security-token header).
 - *Signature expired: 20200317T000000Z is now earlier than 20200317T222541Z
   (20200317T224041Z - 15 min.)*: This error happens when `x-amz-date` header
   value is the formatted date (`YYYYMMDD`) rather than the ISO8601 formatted
@@ -168,6 +171,8 @@ problem. Below are some of the puzzling responses I encountered:
   produced when temporary credentials are used and the credentials have
   expired.
 
+See also [AWS documentation for STS error messages](https://docs.aws.amazon.com/STS/latest/APIReference/CommonErrors.html).
+
 ### Resources
 
 Generally I found Amazon's own documentation to be the best for implementing

data/spec/README.md

@@ -107,7 +107,7 @@ other tests require a sharded cluster with more than one shard. Tests requiring
 a single shard can be run against a deployment with multiple shards by
 specifying only one mongos address in MONGODB_URI.
 
-## Note Regarding SSL/TLS Arguments
+## Note Regarding TLS/SSL Arguments
 
 MongoDB 4.2 (server and shell) added new command line options for setting TLS
 parameters. These options follow the naming of URI options used by both the
@@ -185,6 +185,51 @@ verification, run:
 Note that there are tests in the test suite that cover TLS verification, and
 they may fail if the test suite is run in this way.
 
+## OCSP
+
+There are several types of OCSP tests implemented in the test suite.
+
+OCSP unit tests are in `spec/integration/ocsp_verifier_spec.rb`. To run
+these, set `OCSP_VERIFIER=1` in the environment. There must NOT be a process
+running on the host port 8100 as that port will be used by the OCSP responder
+launched by the tests.
+
+For the remaining OCSP tests, the following environment variables must be set
+to the possible values indicated below:
+
+    OCSP_ALGORITHM=rsa|ecdsa
+    OCSP_STATUS=valid|revoked|unknown
+    OCSP_DELEGATE=0|1
+    OCSP_MUST_STAPLE=0|1
+
+These tests also require the mock OCSP responder running on the host machine
+on port 8100 with the configuration that matches the environment variables
+just described. Please refer to the Docker and Evergreen scripts in the
+driver repository for further details.
+
+Additionally, the server must be configured to use the appropriate server
+certificate and CA certificate from the respective subdirectory of
+`spec/support/ocsp`. This is easiest to achieve by using the Docker tooling
+described in `.evergreen/README.md`.
+
+OCSP connectivity tests are in `spec/integration/ocsp_connectivity.rb`.
+These test the combinations described
+[here](https://github.com/mongodb/specifications/blob/master/source/ocsp-support/tests/README.rst#integration-tests-permutations-to-be-tested).
+To run these tests, set `OCSP_CONNECTIVITY=pass` environment variable if
+the tests are expected to connect successfully or `OCSP_CONNECTIVITY=fail` if
+the tests are expected to not connect.
+Note that some of these configurations require OCSP responder to return
+the failure response; in such configurations, ONLY the OCSP connectivity tests
+may pass (since the driver may reject connections to servers when OCSP
+responder returns the failure response, or OCSP verification otherwise
+definitively fails).
+
+When not running either OCSP verifier tests or OCSP connectivity tests but
+when OCSP algorithm is configured, the test suite will execute normally
+using the provided `MONGO_URI`. This configuration may be used to exercise
+OCSP while running the full test suite. In this case, setting `OCSP_STATUS`
+to `revoked` will generally cause the test suite to fail.
+
 ## Authentication
 
 mlaunch can configure authentication on the server:
@@ -511,6 +556,16 @@ following environment variable:
 
     FORK=1
 
+OCSP tests require Python 3 with asn1crypto, oscrypto and flask packages
+installed, and they require the drivers-evergreen-tools submodule to be
+checked out. To run these tests, set the following environment variable:
+
+    OCSP=1
+
+To check out the submodule, run:
+
+    git submodule update --init --recursive
+
 ## Debug Logging
 
 The test suite is run with the driver log level set to `WARN` by default.

data/spec/integration/bson_symbol_spec.rb

@@ -25,10 +25,8 @@ describe 'Symbol encoding to BSON' do
   end
 
   it 'round-trips symbol values using the same byte buffer' do
-    if BSON::Environment.jruby? && (BSON::VERSION.split('.').map(&:to_i) <=> [4, 11, 0]) < 0
-      skip 'This test is only relevant to bson versions that increment ByteBuffer '\
-       'read and write positions separately in JRuby, as implemented in ' \
-       'bson version 4.11.0. For more information, see https://jira.mongodb.org/browse/RUBY-2128'
+    if BSON::Environment.jruby?
+      pending 'https://jira.mongodb.org/browse/RUBY-2128'
     end
 
     Hash.from_bson(hash.to_bson).should == hash

data/spec/integration/bulk_write_spec.rb

@@ -16,4 +16,52 @@ describe 'Bulk writes' do
       end.not_to raise_error
     end
   end
+
+  context 'when bulk write needs to be split' do
+    let(:subscriber) { EventSubscriber.new }
+
+    let(:max_bson_size) { Mongo::Server::ConnectionBase::DEFAULT_MAX_BSON_OBJECT_SIZE }
+
+    let(:insert_events) do
+      subscriber.command_started_events('insert')
+    end
+
+    let(:failed_events) do
+      subscriber.failed_events
+    end
+
+    let(:operations) do
+      [{ insert_one: { text: 'a' * (max_bson_size/2) } }] * 6
+    end
+
+    before do
+      authorized_client.subscribe(Mongo::Monitoring::COMMAND, subscriber)
+      authorized_collection.bulk_write(operations)
+    end
+
+    context '3.6+ server' do
+      min_server_fcv '3.6'
+
+      it 'splits the operations' do
+        # 3.6+ servers can send multiple bulk operations in one message,
+        # with the whole message being limited to 48m.
+        expect(insert_events.length).to eq(2)
+      end
+    end
+
+    context 'pre-3.6 server' do
+      max_server_version '3.4'
+
+      it 'splits the operations' do
+        # Pre-3.6 servers limit the entire message payload to the size of
+        # a single document which is 16m. Given our test data this means
+        # twice as many messages are sent.
+        expect(insert_events.length).to eq(4)
+      end
+    end
+
+    it 'does not have a command failed event' do
+      expect(failed_events).to be_empty
+    end
+  end
 end