mongo 2.13.0.beta1 → 2.14.0

data/lib/mongo/uri/srv_protocol.rb

@@ -97,7 +97,7 @@ module Mongo
 
       # @return [ String ] NO_SRV_RECORDS Error message format string indicating that no SRV records
       #   were found.
-      NO_SRV_RECORDS = "The DNS query returned no SRV records at hostname (%s)".freeze
+      NO_SRV_RECORDS = "The DNS query returned no SRV records for '%s'".freeze
 
       # @return [ String ] INVALID_TXT_RECORD_OPTION Error message format string indicating that an
       #   unexpected TXT record option was found.
@@ -130,6 +130,7 @@ module Mongo
         @resolver ||= Srv::Resolver.new(
           raise_on_invalid: true,
           resolv_options: options[:resolv_options],
+          timeout: options[:connect_timeout],
         )
       end
 
@@ -220,7 +221,7 @@ module Mongo
           raise Error::InvalidTXTRecord.new(INVALID_OPTS_VALUE_DELIM) unless opt.index(URI_OPTS_VALUE_DELIM)
           key, value = opt.split(URI_OPTS_VALUE_DELIM)
           raise Error::InvalidTXTRecord.new(INVALID_TXT_RECORD_OPTION) unless VALID_TXT_OPTIONS.include?(key.downcase)
-          add_uri_option(key, value, txt_options)
+          options_mapper.add_uri_option(key, value, txt_options)
           txt_options
         end
       end

data/lib/mongo/utils.rb

@@ -0,0 +1,73 @@
+# Copyright (C) 2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+
+  # @api private
+  module Utils
+
+    class LocalLogger
+      include Loggable
+
+      def initialize(**opts)
+        @options = opts
+      end
+
+      attr_reader :options
+    end
+
+    # @option opts [ true | false | nil | Integer ] :bg_error_backtrace
+    #   Experimental. Set to true to log complete backtraces for errors in
+    #   background threads. Set to false or nil to not log backtraces. Provide
+    #   a positive integer to log up to that many backtrace lines.
+    # @option opts [ Logger ] :logger A custom logger to use.
+    # @option opts [ String ] :log_prefix A custom log prefix to use when
+    #   logging.
+    module_function def warn_bg_exception(msg, exc, **opts)
+      bt_excerpt = excerpt_backtrace(exc, **opts)
+      logger = LocalLogger.new(**opts)
+      logger.log_warn("#{msg}: #{exc.class}: #{exc}#{bt_excerpt}")
+    end
+
+    # @option opts [ true | false | nil | Integer ] :bg_error_backtrace
+    #   Experimental. Set to true to log complete backtraces for errors in
+    #   background threads. Set to false or nil to not log backtraces. Provide
+    #   a positive integer to log up to that many backtrace lines.
+    module_function def excerpt_backtrace(exc, **opts)
+      case lines = opts[:bg_error_backtrace]
+      when Integer
+        ":\n#{exc.backtrace[0..lines].join("\n")}"
+      when false, nil
+        nil
+      else
+        ":\n#{exc.backtrace.join("\n")}"
+      end
+    end
+
+    # Symbolizes the keys in the provided hash.
+    module_function def shallow_symbolize_keys(hash)
+      Hash[hash.map { |k, v| [k.to_sym, v] }]
+    end
+
+    # Stringifies the keys in the provided hash and converts underscore
+    # style keys to camel case style keys.
+    module_function def shallow_camelize_keys(hash)
+      Hash[hash.map { |k, v| [camelize(k), v] }]
+    end
+
+    module_function def camelize(sym)
+      sym.to_s.gsub(/_(\w)/) { $1.upcase }
+    end
+  end
+end

data/lib/mongo/version.rb

@@ -17,5 +17,5 @@ module Mongo
   # The current version of the driver.
   #
   # @since 2.0.0
-  VERSION = '2.13.0.beta1'
+  VERSION = '2.14.0'.freeze
 end

data/spec/NOTES.aws-auth.md

@@ -145,13 +145,16 @@ problem. Below are some of the puzzling responses I encountered:
   line) but the value is otherwise completely valid. This error has no relation
   to the "session token" or "security token" as used with temporary AWS
   credentials.
-- *The security token included in the request is invalid*: this error is
-  produced when the AWS access key id, as specified in the scope part of the
-  `Authorization` header, is not a valid access key id. In the case of
-  non-temporary credentials being used for authentication, the error refers to
-  a "security token" but the authentication process does not actually use a
-  security token as this term is used in the AWS documentation describing
-  temporary credentials.
+- *The security token included in the request is invalid*: this error can be
+  produced in several circumstances:
+  - When the AWS access key id, as specified in the scope part of the
+    `Authorization` header, is not a valid access key id. In the case of
+    non-temporary credentials being used for authentication, the error refers to
+    a "security token" but the authentication process does not actually use a
+    security token as this term is used in the AWS documentation describing
+    temporary credentials.
+  - When using temporary credentials and the security token is not provided
+    in the STS request at all (x-amz-security-token header).
 - *Signature expired: 20200317T000000Z is now earlier than 20200317T222541Z
   (20200317T224041Z - 15 min.)*: This error happens when `x-amz-date` header
   value is the formatted date (`YYYYMMDD`) rather than the ISO8601 formatted
@@ -168,6 +171,8 @@ problem. Below are some of the puzzling responses I encountered:
   produced when temporary credentials are used and the credentials have
   expired.
 
+See also [AWS documentation for STS error messages](https://docs.aws.amazon.com/STS/latest/APIReference/CommonErrors.html).
+
 ### Resources
 
 Generally I found Amazon's own documentation to be the best for implementing

data/spec/README.aws-auth.md

@@ -233,7 +233,7 @@ To provision the instance for running the driver's test suite via Docker, run:
 To run the AWS auth tests using the EC2 instance role credentials, run:
 
     ./.evergreen/test-docker-remote ubuntu@$ip \
-      MONGODB_VERSION=4.3 AUTH=aws-ec2 \
+      MONGODB_VERSION=4.4 AUTH=aws-ec2 \
       -s .evergreen/run-tests-aws-auth.sh \
       -a .env.private
 
@@ -299,7 +299,7 @@ If the public IP address is in the `IP` shell variable, provision the task:
 
 To run the credentials retrieval test on the ECS task, execute:
 
-    ./.evergreen/test-remote root@$IP env AUTH=aws-ecs RVM_RUBY=ruby-2.7 MONGODB_VERSION=4.3 TEST_CMD='rspec spec/integration/aws*spec.rb' .evergreen/run-tests.sh
+    ./.evergreen/test-remote root@$IP env AUTH=aws-ecs RVM_RUBY=ruby-2.7 MONGODB_VERSION=4.4 TEST_CMD='rspec spec/integration/aws*spec.rb' .evergreen/run-tests.sh
 
 To run the test again without rebuilding the remote environment, execute:
 

data/spec/README.md

@@ -2,6 +2,13 @@
 
 ## Quick Start
 
+The test suite requires shared tooling that is stored in a separate repository
+and is referenced as a submodule. After checking out the desired driver
+branch, check out the matching submodules:
+
+    git submodule init
+    git submodule update
+
 To run the test suite against a local MongoDB deployment listening on port
 27017, run:
 
@@ -107,7 +114,7 @@ other tests require a sharded cluster with more than one shard. Tests requiring
 a single shard can be run against a deployment with multiple shards by
 specifying only one mongos address in MONGODB_URI.
 
-## Note Regarding SSL/TLS Arguments
+## Note Regarding TLS/SSL Arguments
 
 MongoDB 4.2 (server and shell) added new command line options for setting TLS
 parameters. These options follow the naming of URI options used by both the
@@ -185,6 +192,51 @@ verification, run:
 Note that there are tests in the test suite that cover TLS verification, and
 they may fail if the test suite is run in this way.
 
+## OCSP
+
+There are several types of OCSP tests implemented in the test suite.
+
+OCSP unit tests are in `spec/integration/ocsp_verifier_spec.rb`. To run
+these, set `OCSP_VERIFIER=1` in the environment. There must NOT be a process
+running on the host port 8100 as that port will be used by the OCSP responder
+launched by the tests.
+
+For the remaining OCSP tests, the following environment variables must be set
+to the possible values indicated below:
+
+    OCSP_ALGORITHM=rsa|ecdsa
+    OCSP_STATUS=valid|revoked|unknown
+    OCSP_DELEGATE=0|1
+    OCSP_MUST_STAPLE=0|1
+
+These tests also require the mock OCSP responder running on the host machine
+on port 8100 with the configuration that matches the environment variables
+just described. Please refer to the Docker and Evergreen scripts in the
+driver repository for further details.
+
+Additionally, the server must be configured to use the appropriate server
+certificate and CA certificate from the respective subdirectory of
+`spec/support/ocsp`. This is easiest to achieve by using the Docker tooling
+described in `.evergreen/README.md`.
+
+OCSP connectivity tests are in `spec/integration/ocsp_connectivity.rb`.
+These test the combinations described
+[here](https://github.com/mongodb/specifications/blob/master/source/ocsp-support/tests/README.rst#integration-tests-permutations-to-be-tested).
+To run these tests, set `OCSP_CONNECTIVITY=pass` environment variable if
+the tests are expected to connect successfully or `OCSP_CONNECTIVITY=fail` if
+the tests are expected to not connect.
+Note that some of these configurations require OCSP responder to return
+the failure response; in such configurations, ONLY the OCSP connectivity tests
+may pass (since the driver may reject connections to servers when OCSP
+responder returns the failure response, or OCSP verification otherwise
+definitively fails).
+
+When not running either OCSP verifier tests or OCSP connectivity tests but
+when OCSP algorithm is configured, the test suite will execute normally
+using the provided `MONGO_URI`. This configuration may be used to exercise
+OCSP while running the full test suite. In this case, setting `OCSP_STATUS`
+to `revoked` will generally cause the test suite to fail.
+
 ## Authentication
 
 mlaunch can configure authentication on the server:
@@ -511,6 +563,16 @@ following environment variable:
 
     FORK=1
 
+OCSP tests require Python 3 with asn1crypto, oscrypto and flask packages
+installed, and they require the drivers-evergreen-tools submodule to be
+checked out. To run these tests, set the following environment variable:
+
+    OCSP=1
+
+To check out the submodule, run:
+
+    git submodule update --init --recursive
+
 ## Debug Logging
 
 The test suite is run with the driver log level set to `WARN` by default.

data/spec/integration/awaited_ismaster_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+describe 'awaited ismaster' do
+  min_server_fcv '4.4'
+
+  # If we send the consecutive ismasters to different mongoses,
+  # they have different process ids, and so the awaited one would return
+  # immediately.
+  require_no_multi_shard
+
+  let(:client) { authorized_client }
+
+  it 'waits' do
+    # Perform a regular ismaster to get topology version
+    resp = client.database.command(ismaster: 1)
+    doc = resp.replies.first.documents.first
+    tv = Mongo::TopologyVersion.new(doc['topologyVersion'])
+    tv.should be_a(BSON::Document)
+
+    elapsed_time = Benchmark.realtime do
+      resp = client.database.command(ismaster: 1,
+        topologyVersion: tv.to_doc, maxAwaitTimeMS: 500)
+    end
+    doc = resp.replies.first.documents.first
+
+    elapsed_time.should > 0.5
+  end
+end

data/spec/integration/bson_symbol_spec.rb

@@ -25,8 +25,10 @@ describe 'Symbol encoding to BSON' do
   end
 
   it 'round-trips symbol values using the same byte buffer' do
-    if BSON::Environment.jruby?
-      pending 'https://jira.mongodb.org/browse/RUBY-2128'
+    if BSON::Environment.jruby? && (BSON::VERSION.split('.').map(&:to_i) <=> [4, 11, 0]) < 0
+      skip 'This test is only relevant to bson versions that increment ByteBuffer '\
+       'read and write positions separately in JRuby, as implemented in ' \
+       'bson version 4.11.0. For more information, see https://jira.mongodb.org/browse/RUBY-2128'
     end
 
     Hash.from_bson(hash.to_bson).should == hash

data/spec/integration/bulk_write_spec.rb

@@ -0,0 +1,67 @@
+require 'spec_helper'
+
+describe 'Bulk writes' do
+  before do
+    authorized_collection.drop
+  end
+
+  context 'when bulk write is larger than 48MB' do
+    let(:operations) do
+      [ { insert_one: { text: 'a' * 1000 * 1000 } } ] * 48
+    end
+
+    it 'succeeds' do
+      expect do
+        authorized_collection.bulk_write(operations)
+      end.not_to raise_error
+    end
+  end
+
+  context 'when bulk write needs to be split' do
+    let(:subscriber) { EventSubscriber.new }
+
+    let(:max_bson_size) { Mongo::Server::ConnectionBase::DEFAULT_MAX_BSON_OBJECT_SIZE }
+
+    let(:insert_events) do
+      subscriber.command_started_events('insert')
+    end
+
+    let(:failed_events) do
+      subscriber.failed_events
+    end
+
+    let(:operations) do
+      [{ insert_one: { text: 'a' * (max_bson_size/2) } }] * 6
+    end
+
+    before do
+      authorized_client.subscribe(Mongo::Monitoring::COMMAND, subscriber)
+      authorized_collection.bulk_write(operations)
+    end
+
+    context '3.6+ server' do
+      min_server_fcv '3.6'
+
+      it 'splits the operations' do
+        # 3.6+ servers can send multiple bulk operations in one message,
+        # with the whole message being limited to 48m.
+        expect(insert_events.length).to eq(2)
+      end
+    end
+
+    context 'pre-3.6 server' do
+      max_server_version '3.4'
+
+      it 'splits the operations' do
+        # Pre-3.6 servers limit the entire message payload to the size of
+        # a single document which is 16m. Given our test data this means
+        # twice as many messages are sent.
+        expect(insert_events.length).to eq(4)
+      end
+    end
+
+    it 'does not have a command failed event' do
+      expect(failed_events).to be_empty
+    end
+  end
+end

data/spec/integration/change_stream_examples_spec.rb

@@ -174,8 +174,10 @@ describe 'change streams examples in Ruby' do
 
         # Start Changestream Example 4
 
-        pipeline = [ {'$match' => { '$or' => [{ 'fullDocument.username' => 'alice' },
-                                              { 'operationType' => 'delete' }] } }]
+        pipeline = [
+          { "$match" => { 'fullDocument.username' => 'alice' } },
+          { "$addFields" => { 'newField' => 'this is an added field!' } }
+        ];
         cursor = inventory.watch(pipeline).to_enum
         cursor.next
 
@@ -191,6 +193,8 @@ describe 'change streams examples in Ruby' do
       expect(change['fullDocument']).not_to be_nil
       expect(change['fullDocument']['_id']).not_to be_nil
       expect(change['fullDocument']['username']).to eq('alice')
+      expect(change['newField']).not_to be_nil
+      expect(change['newField']).to eq('this is an added field!')
       expect(change['ns']).not_to be_nil
       expect(change['ns']['db']).to eq(SpecConfig.instance.test_db)
       expect(change['ns']['coll']).to eq(inventory.name)

data/spec/integration/change_stream_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 describe 'Change stream integration', retry: 4 do
-  only_mri
+  require_mri
   max_example_run_time 7
   min_server_fcv '3.6'
   require_topology :replica_set

data/spec/integration/check_clean_slate_spec.rb

@@ -0,0 +1,16 @@
+require 'lite_spec_helper'
+
+# This test can be used to manually verify that there are no leaked
+# background threads - execute it after executing another test (in the same
+# rspec run) that is suspected to leak background threads, such as by
+# running:
+#
+# rspec your_spec.rb spec/integration/check_clean_slate_spec.rb
+
+describe 'Check clean slate' do
+  clean_slate_for_all_if_possible
+
+  it 'checks' do
+    # Nothing
+  end
+end

data/spec/integration/client_authentication_options_spec.rb

@@ -131,7 +131,7 @@ describe 'Client authentication options' do
     end
   end
 
-  shared_examples_for 'an auth mechanism that doesn\'t support auth_mech_properties' do
+  shared_examples_for 'an auth mechanism that does not support auth_mech_properties' do
     context 'with URI options' do
       let(:credentials) { "#{user}:#{pwd}@" }
       let(:options) { "?authMechanism=#{auth_mech_string}&authMechanismProperties=CANONICALIZE_HOST_NAME:true" }
@@ -163,7 +163,7 @@ describe 'Client authentication options' do
     end
   end
 
-  shared_examples_for 'an auth mechanism that doesn\'t support invalid auth sources' do
+  shared_examples_for 'an auth mechanism that does not support invalid auth sources' do
     context 'with URI options' do
       let(:credentials) { "#{user}:#{pwd}@" }
       let(:options) { "?authMechanism=#{auth_mech_string}&authSource=foo" }
@@ -199,7 +199,7 @@ describe 'Client authentication options' do
 
     it_behaves_like 'a supported auth mechanism'
     it_behaves_like 'auth mechanism that uses database or default auth source', 'admin'
-    it_behaves_like 'an auth mechanism that doesn\'t support auth_mech_properties'
+    it_behaves_like 'an auth mechanism that does not support auth_mech_properties'
   end
 
   context 'with SCRAM-SHA-1 auth mechanism' do
@@ -208,7 +208,7 @@ describe 'Client authentication options' do
 
     it_behaves_like 'a supported auth mechanism'
     it_behaves_like 'auth mechanism that uses database or default auth source', 'admin'
-    it_behaves_like 'an auth mechanism that doesn\'t support auth_mech_properties'
+    it_behaves_like 'an auth mechanism that does not support auth_mech_properties'
   end
 
   context 'with SCRAM-SHA-256 auth mechanism' do
@@ -217,7 +217,7 @@ describe 'Client authentication options' do
 
     it_behaves_like 'a supported auth mechanism'
     it_behaves_like 'auth mechanism that uses database or default auth source', 'admin'
-    it_behaves_like 'an auth mechanism that doesn\'t support auth_mech_properties'
+    it_behaves_like 'an auth mechanism that does not support auth_mech_properties'
   end
 
   context 'with GSSAPI auth mechanism' do
@@ -227,7 +227,7 @@ describe 'Client authentication options' do
     let(:auth_mech_sym) { :gssapi }
 
     it_behaves_like 'a supported auth mechanism'
-    it_behaves_like 'an auth mechanism that doesn\'t support invalid auth sources'
+    it_behaves_like 'an auth mechanism that does not support invalid auth sources'
 
     let(:auth_mech_properties) { { canonicalize_host_name: true, service_name: 'other'} }
 
@@ -256,6 +256,43 @@ describe 'Client authentication options' do
         expect(client.options[:auth_mech_properties]).to eq({ 'service_name' => 'mongodb' })
       end
     end
+
+    context 'when properties are given but not service name' do
+      context 'with URI options' do
+        let(:credentials) { "#{user}:#{pwd}@" }
+
+        context 'with default auth mech properties' do
+          let(:options) { '?authMechanism=GSSAPI&authMechanismProperties=service_realm:foo' }
+
+          it 'sets service name to mongodb' do
+            expect(client.options[:auth_mech_properties]).to eq(
+              'service_name' => 'mongodb',
+              'service_realm' => 'foo',
+            )
+          end
+        end
+      end
+
+      context 'with client options' do
+        let(:client_opts) do
+          {
+            auth_mech: :gssapi,
+            user: user,
+            password: pwd,
+            auth_mech_properties: {
+              service_realm: 'foo',
+            }.freeze,
+          }.freeze
+        end
+
+        it 'sets default auth mech properties' do
+          expect(client.options[:auth_mech_properties]).to eq(
+            'service_name' => 'mongodb',
+            'service_realm' => 'foo',
+          )
+        end
+      end
+    end
   end
 
   context 'with PLAIN auth mechanism' do
@@ -265,7 +302,7 @@ describe 'Client authentication options' do
     it_behaves_like 'a supported auth mechanism'
     it_behaves_like 'auth mechanism that uses database or default auth source', '$external'
     it_behaves_like 'an auth mechanism with ssl'
-    it_behaves_like 'an auth mechanism that doesn\'t support auth_mech_properties'
+    it_behaves_like 'an auth mechanism that does not support auth_mech_properties'
   end
 
   context 'with MONGODB-X509 auth mechanism' do
@@ -276,8 +313,8 @@ describe 'Client authentication options' do
 
     it_behaves_like 'a supported auth mechanism'
     it_behaves_like 'an auth mechanism with ssl'
-    it_behaves_like 'an auth mechanism that doesn\'t support auth_mech_properties'
-    it_behaves_like 'an auth mechanism that doesn\'t support invalid auth sources'
+    it_behaves_like 'an auth mechanism that does not support auth_mech_properties'
+    it_behaves_like 'an auth mechanism that does not support invalid auth sources'
 
     context 'with URI options' do
       let(:credentials) { "#{user}@" }
@@ -402,35 +439,62 @@ describe 'Client authentication options' do
       {
         service_name: service_name,
         canonicalize_host_name: canonicalize_host_name,
-        service_realm: service_realm
-      }
+        service_realm: service_realm,
+      }.freeze
     end
 
-    context 'with URI options' do
-      let(:options) do
-        "?authMechanismProperties=SERVICE_NAME:#{service_name}," +
-          "CANONICALIZE_HOST_NAME:#{canonicalize_host_name}," +
-          "SERVICE_REALM:#{service_realm}"
-      end
-
+    shared_examples 'correctly sets auth mechanism properties on the client' do
       it 'correctly sets auth mechanism properties on the client' do
-        expect(client.options[:auth_mech_properties]).to eq({
+        expect(client.options[:auth_mech_properties]).to eq(
           'service_name' => service_name,
           'canonicalize_host_name' => canonicalize_host_name,
-          'service_realm' => service_realm
-        })
+          'service_realm' => service_realm,
+        )
+      end
+    end
+
+    context 'with URI options' do
+      let(:options) do
+        "?authMechanismProperties=SERVICE_name:#{service_name}," +
+          "CANONICALIZE_HOST_name:#{canonicalize_host_name}," +
+          "SERVICE_realm:#{service_realm}"
       end
+
+      include_examples 'correctly sets auth mechanism properties on the client'
     end
 
     context 'with client options' do
-      let(:client_opts) { { auth_mech_properties: auth_mechanism_properties } }
+      [:auth_mech_properties, 'auth_mech_properties'].each do |key|
 
-      it 'correctly sets auth mechanism properties on the client' do
-        expect(client.options[:auth_mech_properties]).to eq({
-          'service_name' => service_name,
-          'canonicalize_host_name' => canonicalize_host_name,
-          'service_realm' => service_realm
-        })
+        context "using #{key.class} keys" do
+          let(:client_opts) { { key => auth_mechanism_properties } }
+
+          include_examples 'correctly sets auth mechanism properties on the client'
+
+          context 'when options are given in mixed case' do
+            let(:auth_mechanism_properties) do
+              {
+                service_NAME: service_name,
+                canonicalize_host_NAME: canonicalize_host_name,
+                service_REALM: service_realm,
+              }.freeze
+            end
+
+            context 'using URI and options' do
+
+              let(:client) { new_local_client_nmio(uri, client_opts) }
+
+              include_examples 'correctly sets auth mechanism properties on the client'
+            end
+
+            context 'using host and options' do
+
+              let(:client) { new_local_client_nmio(['localhost'], client_opts) }
+
+              include_examples 'correctly sets auth mechanism properties on the client'
+            end
+          end
+        end
       end
     end
   end