mongo 2.11.6 → 2.12.0.rc0

data/lib/mongo/server/monitor.rb

@@ -94,6 +94,8 @@ module Mongo
 
       # The compressor is determined during the handshake, so it must be an attribute
       # of the connection.
+      #
+      # @deprecated
       def_delegators :connection, :compressor
 
       # @return [ Monitoring ] monitoring The monitoring.

data/lib/mongo/server/monitor/connection.rb

@@ -19,7 +19,7 @@ module Mongo
       # This class models the monitor connections and their behavior.
       #
       # @since 2.0.0
-      class Connection
+      class Connection < Server::ConnectionCommon
         include Retryable
         include Connectable
         include Loggable
@@ -64,12 +64,14 @@ module Mongo
         # Key for compression algorithms in the response from the server during handshake.
         #
         # @since 2.5.0
+        # @deprecated
         COMPRESSION = 'compression'.freeze
 
         # Warning message that the server has no compression algorithms in common with those requested
         #   by the client.
         #
         # @since 2.5.0
+        # @deprecated
         COMPRESSION_WARNING = 'The server has no compression algorithms in common with those requested. ' +
                                 'Compression will not be used.'.freeze
 
@@ -125,11 +127,6 @@ module Mongo
         # @return [ Mongo::Address ] address The address to connect to.
         attr_reader :address
 
-        # The compressor, which is determined during the handshake.
-        #
-        # @since 2.5.0
-        attr_reader :compressor
-
         # Send the preserialized ismaster call.
         #
         # @example Send a preserialized ismaster message.
@@ -209,18 +206,6 @@ module Mongo
 
         private
 
-        def set_compressor!(reply)
-          server_compressors = reply[COMPRESSION]
-
-          if options[:compressors]
-            if intersection = (server_compressors & options[:compressors])
-              @compressor = intersection[0]
-            else
-              log_warn(COMPRESSION_WARNING)
-            end
-          end
-        end
-
         def handshake!(socket)
           if @app_metadata
             socket.write(@app_metadata.ismaster_bytes)

data/lib/mongo/server/pending_connection.rb

@@ -21,8 +21,9 @@ module Mongo
     class PendingConnection < ConnectionBase
       extend Forwardable
 
-      def initialize(socket, server, monitoring, options = {})
+      def initialize(socket, description, server, monitoring, options = {})
         @socket = socket
+        @description = description
         @options = options
         @server = server
         @monitoring = monitoring

data/lib/mongo/session.rb

@@ -578,7 +578,7 @@ module Mongo
               txn_num: txn_num,
               write_concern: write_concern,
             }
-            Operation::Command.new(spec).execute(server)
+            Operation::Command.new(spec).execute(server, client: @client)
           end
         end
       ensure
@@ -622,7 +622,7 @@ module Mongo
               db_name: 'admin',
               session: self,
               txn_num: txn_num
-            ).execute(server)
+            ).execute(server, client: @client)
           end
         end
 

data/lib/mongo/session/session_pool.rb

@@ -113,9 +113,14 @@ module Mongo
       def end_sessions
         while !@queue.empty?
           server = ServerSelector.get(mode: :primary_preferred).select_server(@cluster)
-          Operation::Command.new(
-              :selector => {endSessions: @queue.shift(10_000).collect { |s| s.session_id }},
-              :db_name => Database::ADMIN).execute(server)
+          op = Operation::Command.new(
+            selector: {
+              endSessions: @queue.shift(10_000).map(&:session_id),
+            },
+            db_name: Database::ADMIN,
+          )
+          # end_sessions does not take a client as an argument
+          op.execute(server, client: nil)
         end
       rescue Mongo::Error, Error::AuthError
       end

data/lib/mongo/socket.rb

@@ -40,6 +40,11 @@ module Mongo
     # @since 2.0.0
     TIMEOUT_PACK = 'l_2'.freeze
 
+    # Write data to the socket in chunks of this size.
+    #
+    # @api private
+    WRITE_CHUNK_SIZE = 65536
+
     # @return [ Integer ] family The type of host family.
     attr_reader :family
 
@@ -87,7 +92,7 @@ module Mongo
     #
     # @since 2.0.0
     def close
-      @socket.close rescue true
+      @socket.close rescue nil
       true
     end
 
@@ -105,20 +110,6 @@ module Mongo
       handle_errors { @socket.gets(*args) }
     end
 
-    # Create the new socket for the provided family - ipv4, piv6, or unix.
-    #
-    # @example Create a new ipv4 socket.
-    #   Socket.new(Socket::PF_INET)
-    #
-    # @param [ Integer ] family The socket domain.
-    #
-    # @since 2.0.0
-    def initialize(family)
-      @family = family
-      @socket = ::Socket.new(family, SOCK_STREAM, 0)
-      set_socket_options(@socket)
-    end
-
     # Will read all data from the socket for the provided number of bytes.
     # If no data is returned, an exception will be raised.
     #
@@ -172,7 +163,29 @@ module Mongo
     #
     # @since 2.0.0
     def write(*args)
-      handle_errors { @socket.write(*args) }
+      handle_errors do
+        # This method used to forward arguments to @socket.write in a
+        # single call like so:
+        #
+        # @socket.write(*args)
+        #
+        # Turns out, when each buffer to be written is large (e.g. 32 MiB),
+        # this write call would take an extremely long time (20+ seconds)
+        # while using 100% CPU. Splitting the writes into chunks produced
+        # massively better performance (0.05 seconds to write the 32 MiB of
+        # data on the same hardware). Unfortunately splitting the data,
+        # one would assume, results in it being copied, but this seems to be
+        # a much more minor issue compared to CPU cost of writing large buffers.
+        args.each do |buf|
+          buf = buf.to_s
+          i = 0
+          while i < buf.length
+            chunk = buf[i...i+WRITE_CHUNK_SIZE]
+            @socket.write(chunk)
+            i += WRITE_CHUNK_SIZE
+          end
+        end
+      end
     end
 
     # Tests if this socket has reached EOF. Primarily used for liveness checks.

data/lib/mongo/socket/ssl.rb

@@ -46,12 +46,22 @@ module Mongo
       # @since 2.0.0
       def connect!
         Timeout.timeout(options[:connect_timeout], Error::SocketTimeoutError) do
-          handle_errors { @tcp_socket.connect(::Socket.pack_sockaddr_in(port, host)) }
+          handle_errors do
+            @tcp_socket.connect(::Socket.pack_sockaddr_in(port, host))
+          end
           @socket = OpenSSL::SSL::SSLSocket.new(@tcp_socket, context)
-          @socket.hostname = @host_name
-          @socket.sync_close = true
-          handle_errors { @socket.connect }
-          verify_certificate!(@socket)
+          begin
+            @socket.hostname = @host_name
+            @socket.sync_close = true
+            handle_errors do
+              @socket.connect
+            end
+            verify_certificate!(@socket)
+          rescue
+            @socket.close
+            @socket = nil
+            raise
+          end
           self
         end
       end
@@ -76,9 +86,14 @@ module Mongo
         @context = create_context(options)
         @family = family
         @tcp_socket = ::Socket.new(family, SOCK_STREAM, 0)
-        @tcp_socket.setsockopt(IPPROTO_TCP, TCP_NODELAY, 1)
-        set_socket_options(@tcp_socket)
-        connect!
+        begin
+          @tcp_socket.setsockopt(IPPROTO_TCP, TCP_NODELAY, 1)
+          set_socket_options(@tcp_socket)
+          connect!
+        rescue
+          @tcp_socket.close
+          raise
+        end
       end
 
       # Read a single byte from the socket.

data/lib/mongo/socket/tcp.rb

@@ -40,7 +40,9 @@ module Mongo
       def connect!
         Timeout.timeout(options[:connect_timeout], Error::SocketTimeoutError) do
           socket.setsockopt(IPPROTO_TCP, TCP_NODELAY, 1)
-          handle_errors { socket.connect(::Socket.pack_sockaddr_in(port, host)) }
+          handle_errors do
+            socket.connect(::Socket.pack_sockaddr_in(port, host))
+          end
           self
         end
       end
@@ -63,8 +65,15 @@ module Mongo
       # @since 2.0.0
       def initialize(host, port, timeout, family, options = {})
         @host, @port, @timeout, @options = host, port, timeout, options
-        super(family)
-        connect!
+        @family = family
+        @socket = ::Socket.new(family, SOCK_STREAM, 0)
+        begin
+          set_socket_options(@socket)
+          connect!
+        rescue
+          @socket.close
+          raise
+        end
       end
 
       private

data/lib/mongo/timeout.rb

@@ -0,0 +1,49 @@
+# Copyright (C) 2020 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  module Timeout
+
+    # A wrapper around Ruby core's Timeout::timeout method that provides
+    # a standardized API for Ruby versions older and newer than 2.4.0,
+    # which is when the third argument was introduced.
+    #
+    # @param [ Numeric ] sec The number of seconds before timeout.
+    # @param [ Class ] klass The exception class to raise on timeout, optional.
+    #   When no error exception is provided, Timeout::Error is raised.
+    # @param [ String ] message The error message passed to the exception raised
+    #   on timeout, optional. When no error message is provided, the default
+    #   error message is "execution expired".
+    #
+    # @note Ruby versions older than 2.4.0 do not support specifying a custom
+    #   error message, and any error message passed in as an argument will be
+    #   ignored.
+    def timeout(sec, klass=nil, message=nil)
+      if RUBY_VERSION < '2.4.0'
+        ::Timeout.timeout(sec, klass) do
+          yield
+        end
+      else
+        # Jruby Timeout::timeout method does not support passing nil arguments.
+        # Remove the nil arguments before passing them along to the core
+        # Timeout::timeout method.
+        optional_args = [klass, message].compact
+        ::Timeout.timeout(sec, *optional_args) do
+          yield
+        end
+      end
+    end
+    module_function :timeout
+  end
+end

data/lib/mongo/uri.rb

@@ -215,10 +215,22 @@ module Mongo
     # @example Get the uri object.
     #   URI.get(string)
     #
+    # @param [ String ] string The URI to parse.
+    # @param [ Hash ] options The options.
+    #
+    # @option options [ Logger ] :logger A custom logger to use.
+    #
     # @return [URI, URI::SRVProtocol] The uri object.
     #
     # @since 2.5.0
     def self.get(string, opts = {})
+      unless string
+        raise Error::InvalidURI.new(string, 'URI must be a string, not nil.')
+      end
+      if string.empty?
+        raise Error::InvalidURI.new(string, 'Cannot parse an empty URI.')
+      end
+
       scheme, _, remaining = string.partition(SCHEME_DELIM)
       case scheme
         when MONGODB_SCHEME
@@ -257,13 +269,22 @@ module Mongo
     # @example Create the new URI.
     #   URI.new('mongodb://localhost:27017')
     #
-    # @param [ String ] string The uri string.
+    # @param [ String ] string The URI to parse.
     # @param [ Hash ] options The options.
     #
+    # @option options [ Logger ] :logger A custom logger to use.
+    #
     # @raise [ Error::InvalidURI ] If the uri does not match the spec.
     #
     # @since 2.0.0
     def initialize(string, options = {})
+      unless string
+        raise Error::InvalidURI.new(string, 'URI must be a string, not nil.')
+      end
+      if string.empty?
+        raise Error::InvalidURI.new(string, 'Cannot parse an empty URI.')
+      end
+
       @string = string
       @options = options
       parsed_scheme, _, remaining = string.partition(SCHEME_DELIM)
@@ -302,6 +323,14 @@ module Mongo
       if @uri_options[:ssl_cert]
         @uri_options[:ssl_key] = @uri_options[:ssl_cert]
       end
+
+      if uri_options[:write_concern] && !uri_options[:write_concern].empty?
+        begin
+          WriteConcern.get(uri_options[:write_concern])
+        rescue Error::InvalidWriteConcern => e
+          raise_invalid_error_no_fmt!("#{e.class}: #{e}")
+        end
+      end
     end
 
     # Get the credentials provided in the URI.

data/lib/mongo/version.rb

@@ -17,5 +17,5 @@ module Mongo
   # The current version of the driver.
   #
   # @since 2.0.0
-  VERSION = '2.11.6'.freeze
+  VERSION = '2.12.0.rc0'.freeze
 end

data/mongo.gemspec

@@ -40,5 +40,5 @@ Gem::Specification.new do |s|
 
   s.required_ruby_version = ">= 2.3"
 
-  s.add_dependency 'bson', '>=4.4.2', '<5.0.0'
+  s.add_dependency 'bson', '>=4.8.2', '<5.0.0'
 end

data/spec/README.md

@@ -20,8 +20,13 @@ configuration are given later in this document.
 ## MongoDB Server Deployment
 
 The tests require a running MongoDB deployment, configured and started
-externally to the test suite. Tests that are not appropriate for the running
-deployment will be skipped.
+externally to the test suite.
+
+Tests that are not appropriate for the running deployment will be skipped,
+with one exception: the test suite assumes that fail points are enabled in
+the deployment (see the Fail Points section below). Not every test uses fail
+points, therefore it is possible to launch the server without fail points
+being enabled and still pass many of the tests in the test suite.
 
 ## Starting MongoDB Deployment
 
@@ -41,7 +46,7 @@ launched as follows:
 
     # Launch mongod in one terminal
     mkdir /tmp/mdb
-    mongod --dbpath /tmp/mdb
+    mongod --dbpath /tmp/mdb --setParameter enableTestCommands=1
 
     # Run tests in another terminal
     rake
@@ -133,7 +138,7 @@ the top of the driver source tree:
 
     mlaunch init --single --dir /tmp/mdb-ssl --sslMode requireSSL \
       --sslPEMKeyFile `pwd`/spec/support/certificates/server.pem \
-      --sslCAFile `pwd`/spec/support/certificates/ca.pem \
+      --sslCAFile `pwd`/spec/support/certificates/ca.crt \
       --sslClientCertificate `pwd`/spec/support/certificates/client.pem
 
 To test that the driver works when the server's certificate is signed by an
@@ -142,7 +147,7 @@ server certificate bundle:
 
     mlaunch init --single --dir /tmp/mdb-ssl --sslMode requireSSL \
       --sslPEMKeyFile `pwd`/spec/support/certificates/server-second-level-bundle.pem \
-      --sslCAFile `pwd`/spec/support/certificates/ca.pem \
+      --sslCAFile `pwd`/spec/support/certificates/ca.crt \
       --sslClientCertificate `pwd`/spec/support/certificates/client.pem
 
 The driver's test suite is configured to verify certificates by default.
@@ -168,7 +173,7 @@ case a standalone server can be started as follows:
 
     mlaunch init --single --dir /tmp/mdb-ssl --sslMode requireSSL \
       --sslPEMKeyFile `pwd`/spec/support/certificates/server.pem \
-      --sslCAFile `pwd`/spec/support/certificates/ca.pem \
+      --sslCAFile `pwd`/spec/support/certificates/ca.crt \
       --sslAllowConnectionsWithoutCertificates \
       --sslAllowInvalidCertificates
 
@@ -264,9 +269,123 @@ To run the test suite against such a server, run:
 
     MONGODB_URI="mongodb://localhost:27017/?authMechanism=MONGODB-X509&tls=true&tlsCAFile=spec/support/certificates/ca.crt&tlsCertificateKeyFile=spec/support/certificates/client-x509.pem" rake
 
+## Kerberos
+
+The Kerberos-related functionality is packaged in a separate gem,
+`mongo_kerberos`. The driver test suite includes a number of unit tests that
+are skipped by default. To run them as part of the test suite, use the
+provided gemfile as follows:
+
+    export BUNDLE_GEMFILE=gemfiles/mongo_kerberos.gemfile
+    bundle install
+    export KERBEROS=1
+    rake
+
+Note that running the full test suite requires a MongoDB deployment. It is
+possible to run just the Kerberos-related unit tests without provisioning a
+MongoDB deployment; consult .evergreen/run-tests-kerberos-unit.sh for the
+full list of relevant test files.
+
+## Client-Side Encryption
+
+Install libmongocrypt on your machine:
+
+Option 1: Download a pre-built binary
+- Download a tarball of all libmongocrypt variations from this link:
+    https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz
+- Unzip the file you downloaded. You will see a list of folders, each
+    corresponding to an operating system. Find the folder that matches
+    your operating system and open it.
+- Inside that folder, open the folder called "nocrypto." In either the
+    lib or lb64 folder, you will find the libmongocrypt.so or
+    libmongocrypt.dylib or libmongocrypt.dll file, depending on your OS.
+- Move that file to wherever you want to keep it on your machine.
+
+Option 2: Build from source
+- To build libmongocrypt from source, follow the instructions in the README on the libmongocrypt GitHub repo: https://github.com/mongodb/libmongocrypt
+
+Create AWS KMS keys
+Many of the Client-Side Encryption tests require that you have an encryption
+master key hosted on AWS's Key Management Service. Set up a master key by following
+these steps:
+
+1. Sign up for an AWS account at this link if you don't already have one: https://aws.amazon.com/resources/create-account/
+
+2. Create a new IAM user that you want to have permissions to access your new
+master key by following this guide: the "Creating an Administrator IAM User and Group (Console)"
+section of this guide: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user.html
+
+3. Create an access key for your new IAM user and store the access key credentials
+in environment variables on your local machine. Create an access key by following the
+"Managing Access Keys (Console)" instructions in this guide:
+https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey
+Once an access key has been created, store the access key id and the access key
+secret in environment variables. If you plan to frequently run Client-Side
+Encryption tests, it may be a good idea to put these lines in your .bash_profile
+or .bashrc file. Otherwise, you can run them in the terminal window where you
+plan to run your tests.
+
+    ```
+    export MONGO_RUBY_DRIVER_AWS_KEY="YOUR-ACCESS-KEY-ID"
+    export MONGO_RUBY_DRIVER_AWS_SECRET="YOUR-ACCESS-KEY-SECRET"
+    ```
+
+4. Create a new symmetric Customer Master Key (CMK) by following the "Creating Symmetric CMKs (Console)"
+section of this guide: https://docs.aws.amazon.com/kms/latest/developerguide/create-keys.html
+
+5. Store information about your CMK in the following environment variables:
+
+    a. **Region:** Find your AWS region by following this guide: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#using-regions-availability-zones-describe
+    (for example, your region might be "us-east-1" or "ap-south-2").
+
+    b. **Amazon Resource Name (ARN):** Read the following guide to learn more about ARNs
+        and how to view your key's ARN: https://docs.aws.amazon.com/kms/latest/developerguide/viewing-keys-console.html
+
+    Store these two pieces of information in environment variables. If you plan
+    to frequently run Client-Side Encryption tests, it may be a good idea to put
+    these lines in your .bash_profile or .bashrc file. Otherwise, you can run
+    them in the terminal window where you plan to run your tests.
+
+    ```
+    export MONGO_RUBY_DRIVER_AWS_REGION="YOUR-AWS-REGION"
+    export MONGO_RUBY_DRIVER_AWS_ARN="YOUR-AWS-ARN"
+    ```
+
+6. Give your IAM user "Key administrator" and "Key user" privileges on your new CMK
+by following the "Using the AWS Management Console Default View" section of this guide:
+https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-modifying.html
+
+In one terminal, launch MongoDB:
+
+NOTE: You must be running MongoDB 4.2 or higher. All auto-encryption features
+require an enterprise build of MongoDB, but you can still run
+explicit encryption tests using the community edition of MongoDB.
+
+Download different versions of MongoDB here: https://www.mongodb.com/download-center/enterprise
+
+```
+mkdir /tmp/mdb
+mongod --dbpath /tmp/mdb --setParameter enableTestCommands=1
+```
+
+In another terminal run the tests, making sure to set the `LIBMONGOCRYPT_PATH`
+environment variable to the full path to the .so/.dll/.dylib
+```
+LIBMONGOCRYPT_PATH=/path/to/your/libmongocrypt/nocrypto/libmongocrypt.so bundle exec rake
+```
+
 ## Compression
 
-To be written.
+To test compression, set the `compressors` URI option:
+
+    MONGODB_URI="mongodb://localhost:27017/?compressors=zlib" rake
+
+Note that as of this writing, the driver only supports zlib compression.
+Servers 4.2+ enable zlib by default; to test older servers, explicitly enable
+zlib compression when launching the server:
+
+    mongod --dbpath /tmp/mdb --setParameter enableTestCommands=1 \
+      --networkMessageCompressors snappy,zlib
 
 ## Other Options
 
@@ -360,3 +479,11 @@ Then, any of the standard RSpec invocations will work:
 To have the test suite report its current configuration, run:
 
     rake spec:config
+
+## Color Output
+
+The test suite uses color output by default. To view the output in `less`
+with color, use the `-R` option:
+
+    rake 2>&1 | tee rake.log
+    less -R rake.log