mongo 2.14.1 → 2.15.0.alpha

data/lib/mongo/server/monitor.rb

@@ -63,6 +63,10 @@ module Mongo
       # @option options [ Float ] :heartbeat_interval The interval between
       #   regular server checks.
       # @option options [ Logger ] :logger A custom logger to use.
+      # @option options [ Mongo::Server::Monitor::AppMetadata ] :monitor_app_metadata
+      #   The metadata to use for regular monitoring connection.
+      # @option options [ Mongo::Server::Monitor::AppMetadata ] :push_monitor_app_metadata
+      #   The metadata to use for push monitor's connection.
       # @option options [ Float ] :socket_timeout The timeout, in seconds, to
       #   execute operations on the monitoring connection.
       #
@@ -72,6 +76,12 @@ module Mongo
         unless monitoring.is_a?(Monitoring)
           raise ArgumentError, "Wrong monitoring type: #{monitoring.inspect}"
         end
+        unless options[:app_metadata]
+          raise ArgumentError, 'App metadata is required'
+        end
+        unless options[:push_monitor_app_metadata]
+          raise ArgumentError, 'Push monitor app metadata is required'
+        end
         @server = server
         @event_listeners = event_listeners
         @monitoring = monitoring
@@ -273,7 +283,15 @@ module Mongo
         if @connection
           result = server.round_trip_time_averager.measure do
             begin
-              message = @connection.dispatch_bytes(Monitor::Connection::ISMASTER_BYTES)
+              ismaster_bytes = if server_api = options[:server_api]
+                ismaster_doc = Monitor::Connection::ISMASTER.merge(
+                  Utils.transform_server_api(server_api)
+                )
+                Protocol::Query.new(Database::ADMIN, Database::COMMAND, ismaster_doc, limit: -1).serialize.to_s
+              else
+                Monitor::Connection::ISMASTER_BYTES
+              end
+              message = @connection.dispatch_bytes(ismaster_bytes)
               message.documents.first
             rescue Mongo::Error
               @connection.disconnect!
@@ -297,6 +315,7 @@ module Mongo
                 monitoring,
                 **Utils.shallow_symbolize_keys(options.merge(
                   socket_timeout: heartbeat_interval + connection.socket_timeout,
+                  app_metadata: options[:push_monitor_app_metadata],
                 )),
               )
             end

data/lib/mongo/server/pending_connection.rb

@@ -63,6 +63,11 @@ module Mongo
         end
 
         result = handshake!(speculative_auth_doc: speculative_auth_doc)
+
+        if description.unknown?
+          raise Error::InternalDriverError, "Connection description cannot be unknown after successful handshake: #{description.inspect}"
+        end
+
         if speculative_auth_doc && (speculative_auth_result = result['speculativeAuthenticate'])
           unless description.features.scram_sha_1_enabled?
             raise Error::InvalidServerAuthResponse, "Speculative auth succeeded on a pre-3.0 server"
@@ -80,11 +85,15 @@ module Mongo
               speculative_auth_result: speculative_auth_result,
             )
           else
-            raise NotImplementedError, "Speculative auth unexpectedly succeeded for mechanism #{speculative_auth_user.mechanism.inspect}"
+            raise Error::InternalDriverError, "Speculative auth unexpectedly succeeded for mechanism #{speculative_auth_user.mechanism.inspect}"
           end
         elsif !description.arbiter?
           authenticate!
         end
+
+        if description.unknown?
+          raise Error::InternalDriverError, "Connection description cannot be unknown after successful authentication: #{description.inspect}"
+        end
       end
 
       private
@@ -96,7 +105,7 @@ module Mongo
       #   this particular connection.
       def handshake!(speculative_auth_doc: nil)
         unless socket
-          raise Error::HandshakeError, "Cannot handshake because there is no usable socket (for #{address})"
+          raise Error::InternalDriverError, "Cannot handshake because there is no usable socket (for #{address})"
         end
 
         ismaster_doc = app_metadata.validated_document
@@ -104,18 +113,27 @@ module Mongo
           ismaster_doc = ismaster_doc.merge(speculativeAuthenticate: speculative_auth_doc)
         end
 
+        if server_api = options[:server_api]
+          ismaster_doc = ismaster_doc.merge(
+            Utils.transform_server_api(server_api)
+          )
+        end
+
         ismaster_command = Protocol::Query.new(Database::ADMIN, Database::COMMAND,
           ismaster_doc, :limit => -1)
 
-        response = nil
+        doc = nil
         @server.handle_handshake_failure! do
           begin
             response = @server.round_trip_time_averager.measure do
               add_server_diagnostics do
                 socket.write(ismaster_command.serialize.to_s)
-                Protocol::Message.deserialize(socket, Protocol::Message::MAX_MESSAGE_SIZE).documents.first
+                Protocol::Message.deserialize(socket, Protocol::Message::MAX_MESSAGE_SIZE)
               end
             end
+            result = Operation::Result.new([response])
+            result.validate!
+            doc = result.documents.first
           rescue => exc
             msg = "Failed to handshake with #{address}"
             Utils.warn_bg_exception(msg, exc,
@@ -127,9 +145,9 @@ module Mongo
           end
         end
 
-        post_handshake(response, @server.round_trip_time_averager.average_round_trip_time)
+        post_handshake(doc, @server.round_trip_time_averager.average_round_trip_time)
 
-        response
+        doc
       end
 
       # @param [ String | nil ] speculative_auth_client_nonce The client

data/lib/mongo/server/push_monitor.rb

@@ -33,6 +33,9 @@ module Mongo
         if topology_version.nil?
           raise ArgumentError, 'Topology version must be provided but it was nil'
         end
+        unless options[:app_metadata]
+          raise ArgumentError, 'App metadata is required'
+        end
         @monitor = monitor
         @topology_version = topology_version
         @monitoring = monitoring
@@ -139,7 +142,9 @@ module Mongo
           raise
         end
         @server_pushing = resp_msg.flags.include?(:more_to_come)
-        result = resp_msg.documents.first
+        result = Operation::Result.new(resp_msg)
+        result.validate!
+        result.documents.first
       end
 
       def write_ismaster
@@ -147,6 +152,11 @@ module Mongo
           topologyVersion: topology_version.to_doc,
           maxAwaitTimeMS: monitor.heartbeat_interval * 1000,
         )
+        if server_api = options[:server_api]
+          payload.update(
+            Utils.transform_server_api(server_api)
+          )
+        end
 
         req_msg = Protocol::Msg.new([:exhaust_allowed], {}, payload)
         @lock.synchronize { @connection }.write_bytes(req_msg.serialize.to_s)

data/lib/mongo/server.rb

@@ -71,7 +71,8 @@ module Mongo
       unless options[:monitoring_io] == false
         @monitor = Monitor.new(self, event_listeners, monitoring,
           options.merge(
-            app_metadata: Monitor::AppMetadata.new(cluster.options),
+            app_metadata: cluster.monitor_app_metadata,
+            push_monitor_app_metadata: cluster.push_monitor_app_metadata,
             heartbeat_interval: cluster.heartbeat_interval,
         ))
         unless _monitor == false
@@ -172,6 +173,11 @@ module Mongo
                    :cluster_time,
                    :update_cluster_time
 
+    # @api private
+    def_delegators :cluster,
+                   :monitor_app_metadata,
+                   :push_monitor_app_metadata
+
     def_delegators :features,
                    :check_driver_support!
 

data/lib/mongo/server_selector/secondary_preferred.rb

@@ -86,8 +86,13 @@ module Mongo
       #
       # @since 2.0.0
       def to_mongos
-        # Always send the read preference to mongos: DRIVERS-1642.
-        to_doc
+        if tag_sets.empty? && max_staleness.nil? && hedge.nil?
+          # The server preference is not sent to mongos as part of the query
+          # selector if there are no tag sets, for maximum backwards compatibility.
+          nil
+        else
+          to_doc
+        end
       end
 
       private

data/lib/mongo/session/session_pool.rb

@@ -119,8 +119,10 @@ module Mongo
             },
             db_name: Database::ADMIN,
           )
-          # end_sessions does not take a client as an argument
-          op.execute(server, client: nil)
+          context = Operation::Context.new(options: {
+            server_api: server.options[:server_api],
+          })
+          op.execute(server, context: context)
         end
       rescue Mongo::Error, Error::AuthError
       end

data/lib/mongo/session.rb

@@ -582,7 +582,7 @@ module Mongo
               txn_num: txn_num,
               write_concern: write_concern,
             }
-            Operation::Command.new(spec).execute(server, client: @client)
+            Operation::Command.new(spec).execute(server, context: Operation::Context.new(client: @client, session: self))
           end
         end
       ensure
@@ -629,7 +629,7 @@ module Mongo
               db_name: 'admin',
               session: self,
               txn_num: txn_num
-            ).execute(server, client: @client)
+            ).execute(server, context: Operation::Context.new(client: @client, session: self))
           end
         end
 

data/lib/mongo/socket/ssl.rb

@@ -106,6 +106,8 @@ module Mongo
         begin
           @tcp_socket.setsockopt(IPPROTO_TCP, TCP_NODELAY, 1)
           set_socket_options(@tcp_socket)
+          run_tls_context_hooks
+
           connect!
         rescue
           @tcp_socket.close
@@ -379,6 +381,12 @@ module Mongo
       def human_address
         "#{host}:#{port} (#{host_name}:#{port}, TLS)"
       end
+
+      def run_tls_context_hooks
+        Mongo.tls_context_hooks.each do |hook|
+          hook.call(@context)
+        end
+      end
     end
   end
 end

data/lib/mongo/socket.rb

@@ -30,6 +30,7 @@ module Mongo
     # Error message for TLS related exceptions.
     #
     # @since 2.0.0
+    # @deprecated
     SSL_ERROR = 'MongoDB may not be configured with TLS support'.freeze
 
     # Error message for timeouts on socket calls.
@@ -154,7 +155,14 @@ module Mongo
     #
     # @since 2.0.0
     def close
-      @socket.close rescue nil
+      begin
+        # Sometimes it seems the close call can hang for a long time
+        ::Timeout.timeout(5) do
+          @socket.close
+        end
+      rescue
+        # Silence all errors
+      end
       true
     end
 
@@ -326,8 +334,21 @@ module Mongo
         else
           select_args = [nil, [@socket], [@socket], select_timeout]
         end
-        unless Kernel::select(*select_args)
-          raise Errno::ETIMEDOUT, "Took more than #{_timeout} seconds to receive data"
+        rv = Kernel.select(*select_args)
+        if BSON::Environment.jruby?
+          # Ignore the return value of Kernel.select.
+          # On JRuby, select appears to return nil prior to timeout expiration
+          # (apparently due to a EAGAIN) which then causes us to fail the read
+          # even though we could have retried it.
+          # Check the deadline ourselves.
+          if deadline
+            select_timeout = deadline - Time.now
+            if select_timeout <= 0
+              raise Errno::ETIMEDOUT, "Took more than #{_timeout} seconds to receive data"
+            end
+          end
+        elsif rv.nil?
+          raise Errno::ETIMEDOUT, "Took more than #{_timeout} seconds to receive data (select call timed out)"
         end
         retry
       end
@@ -398,6 +419,10 @@ module Mongo
       set_option(sock, :TCP_KEEPCNT, DEFAULT_TCP_KEEPCNT)
       set_option(sock, :TCP_KEEPIDLE, DEFAULT_TCP_KEEPIDLE)
     rescue
+      # JRuby 9.2.13.0 and lower do not define TCP_KEEPINTVL etc. constants.
+      # JRuby 9.2.14.0 defines the constants but does not allow to get or
+      # set them with this error:
+      # Errno::ENOPROTOOPT: Protocol not available - Protocol not available
     end
 
     def set_option(sock, option, default)
@@ -422,7 +447,7 @@ module Mongo
       rescue IOError, SystemCallError => e
         raise Error::SocketError, "#{e.class}: #{e} (for #{human_address})"
       rescue OpenSSL::SSL::SSLError => e
-        raise Error::SocketError, "#{e.class}: #{e} (for #{human_address}) (#{SSL_ERROR})"
+        raise Error::SocketError, "#{e.class}: #{e} (for #{human_address})"
       end
     end
 

data/lib/mongo/uri/options_mapper.rb

@@ -82,6 +82,44 @@ module Mongo
         end
       end
 
+      def smc_to_ruby(opts)
+        uri_options = {}
+
+        opts.each do |key, value|
+          strategy = URI_OPTION_MAP[key.downcase]
+          if strategy.nil?
+            log_warn("Unsupported URI option '#{key}' on URI '#{@string}'. It will be ignored.")
+            return
+          end
+
+          group = strategy[:group]
+          target = if group
+            uri_options[group] || {}
+          else
+            uri_options
+          end
+
+          if key == 'readConcernLevel'
+            value = value.to_sym
+          end
+
+          #value = apply_transform(key, value, strategy[:type])
+          # Sometimes the value here would be nil, for example if we are processing
+          # read preference tags or auth mechanism properties and all of the
+          # data within is invalid. Ignore such options.
+          unless value.nil?
+            merge_uri_option(target, value, strategy[:name])
+          end
+
+          if group && !target.empty? && !uri_options.key?(group)
+            uri_options[group] = target
+          end
+        end
+
+        #p uri_options
+        uri_options
+      end
+
       # Converts Ruby options provided to "standardized MongoClient options".
       #
       # @param [ Hash ] opts Ruby options to convert.

data/lib/mongo/utils.rb

@@ -69,5 +69,20 @@ module Mongo
     module_function def camelize(sym)
       sym.to_s.gsub(/_(\w)/) { $1.upcase }
     end
+
+    # @note server_api must have symbol keys or be a BSON::Document.
+    module_function def transform_server_api(server_api)
+      {}.tap do |doc|
+        if version = server_api[:version]
+          doc['apiVersion'] = version
+        end
+        unless server_api[:strict].nil?
+          doc['apiStrict'] = server_api[:strict]
+        end
+        unless server_api[:deprecation_errors].nil?
+          doc['apiDeprecationErrors'] = server_api[:deprecation_errors]
+        end
+      end
+    end
   end
 end

data/lib/mongo/version.rb

@@ -17,5 +17,5 @@ module Mongo
   # The current version of the driver.
   #
   # @since 2.0.0
-  VERSION = '2.14.1'.freeze
+  VERSION = '2.15.0.alpha'.freeze
 end

data/lib/mongo.rb

@@ -78,4 +78,27 @@ module Mongo
   module_function def clear_ocsp_cache
     Socket::OcspCache.clear
   end
+
+  # This is a user-settable list of hooks that will be invoked when any new
+  # TLS socket is connected. Each hook should be a Proc that takes
+  # an OpenSSL::SSL::SSLContext object as an argument. These hooks can be used
+  # to modify the TLS context (for example to disallow certain ciphers).
+  #
+  # @return [ Array<Proc> ] The list of procs to be invoked when a TLS socket
+  #   is connected (may be an empty Array).
+  module_function def tls_context_hooks
+    @tls_context_hooks ||= []
+  end
+
+  # Set the TLS context hooks.
+  #
+  # @param [ Array<Proc> ] hooks An Array of Procs, each of which should take
+  #   an OpenSSL::SSL::SSLContext object as an argument.
+  module_function def tls_context_hooks=(hooks)
+    unless hooks.is_a?(Array) && hooks.all? { |hook| hook.is_a?(Proc) }
+      raise ArgumentError, "TLS context hooks must be an array of Procs"
+    end
+
+    @tls_context_hooks = hooks
+  end
 end

data/spec/README.md

@@ -512,13 +512,36 @@ To test compression, set the `compressors` URI option:
 
     MONGODB_URI="mongodb://localhost:27017/?compressors=zlib" rake
 
-Note that as of this writing, the driver only supports zlib compression.
+Note that as of this writing, the driver supports
+[ztsd](https://docs.mongodb.com/manual/reference/glossary/#term-zstd),
+[snappy](https://docs.mongodb.com/manual/reference/glossary/#term-snappy)
+and [zlib](https://docs.mongodb.com/manual/reference/glossary/#term-zlib)
+compression.
+
 Servers 4.2+ enable zlib by default; to test older servers, explicitly enable
 zlib compression when launching the server:
 
     mongod --dbpath /tmp/mdb --setParameter enableTestCommands=1 \
       --networkMessageCompressors snappy,zlib
 
+## Server API
+
+To specify server API parameters, use the `SERVER_API` environment variable.
+The server API parameters cannot be specified via URI options.
+
+Both YAML and JSON syntaxes are accepted:
+    
+    SERVER_API='{version: "1", strict: true}' rake
+
+    SERVER_API='{"version":"1","strict":true}' rake
+
+Note that the input must be valid YAML or JSON and the version number must
+be a string, therefore all of the following specifications are invalid:
+
+    SERVER_API='{version:"1",strict:true}' rake
+    SERVER_API='{version: 1}' rake
+    SERVER_API='{"version":1,"strict":true}' rake
+
 ## Other Options
 
 Generally, all URI options recognized by the driver may be set for a test run,

data/spec/integration/auth_spec.rb

@@ -10,8 +10,12 @@ describe 'Auth' do
       authorized_client.cluster.next_primary
     end
 
+    let(:base_options) do
+      SpecConfig.instance.monitoring_options
+    end
+
     let(:connection) do
-      Mongo::Server::Connection.new(server, options)
+      Mongo::Server::Connection.new(server, base_options.merge(options))
     end
 
     before(:all) do
@@ -31,8 +35,9 @@ describe 'Auth' do
     context 'user mechanism not provided' do
 
       context 'user does not exist' do
-        let(:options) { SpecConfig.instance.ssl_options.merge(
-          user: 'nonexistent_user') }
+        let(:options) do
+          {user: 'nonexistent_user' }
+        end
 
         before do
           expect(connection.app_metadata.send(:document)[:saslSupportedMechs]).to eq('admin.nonexistent_user')
@@ -63,8 +68,9 @@ describe 'Auth' do
       end
 
       context 'user exists' do
-        let(:options) { SpecConfig.instance.ssl_options.merge(
-          user: 'existing_user', password: 'bogus') }
+        let(:options) do
+          {user: 'existing_user', password: 'bogus'}
+        end
 
         before do
           expect(connection.app_metadata.send(:document)[:saslSupportedMechs]).to eq("admin.existing_user")
@@ -99,8 +105,9 @@ describe 'Auth' do
       min_server_fcv '3.0'
 
       context 'scram-sha-1 requested' do
-        let(:options) { SpecConfig.instance.ssl_options.merge(
-          user: 'nonexistent_user', auth_mech: :scram) }
+        let(:options) do
+          {user: 'nonexistent_user', auth_mech: :scram}
+        end
 
         it 'indicates scram-sha-1 was requested and used' do
           expect do
@@ -112,8 +119,9 @@ describe 'Auth' do
       context 'scram-sha-256 requested' do
         min_server_fcv '4.0'
 
-        let(:options) { SpecConfig.instance.ssl_options.merge(
-          user: 'nonexistent_user', auth_mech: :scram256) }
+        let(:options) do
+          {user: 'nonexistent_user', auth_mech: :scram256}
+        end
 
         it 'indicates scram-sha-256 was requested and used' do
           expect do
@@ -124,8 +132,9 @@ describe 'Auth' do
     end
 
     context 'when authentication fails' do
-      let(:options) { SpecConfig.instance.ssl_options.merge(
-        user: 'nonexistent_user', password: 'foo') }
+      let(:options) do
+        {user: 'nonexistent_user', password: 'foo'}
+      end
 
       it 'reports which server authentication was attempted against' do
         expect do
@@ -142,8 +151,9 @@ describe 'Auth' do
       end
 
       context 'with custom auth source' do
-        let(:options) { SpecConfig.instance.ssl_options.merge(
-          user: 'nonexistent_user', password: 'foo', auth_source: 'authdb') }
+        let(:options) do
+          {user: 'nonexistent_user', password: 'foo', auth_source: 'authdb'}
+        end
 
         it 'reports auth source used' do
           expect do
@@ -240,7 +250,7 @@ describe 'Auth' do
     require_no_auth
 
     let(:client) do
-      new_local_client(SpecConfig.instance.addresses, SpecConfig.instance.ssl_options.merge(
+      new_local_client(SpecConfig.instance.addresses, SpecConfig.instance.monitoring_options.merge(
         auth_source: 'foo'))
     end
 
@@ -254,7 +264,7 @@ describe 'Auth' do
     require_x509_auth
 
     let(:client) do
-      new_local_client(SpecConfig.instance.addresses, SpecConfig.instance.ssl_options.merge(
+      new_local_client(SpecConfig.instance.addresses, base_options.merge(
         auth_mech: :mongodb_x509))
     end
 

data/spec/integration/bulk_write_error_message_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+
+describe 'BulkWriteError message' do
+  let(:client) { authorized_client }
+  let(:collection_name) { 'bulk_write_error_message_spec' }
+  let(:collection) { client[collection_name] }
+
+  before do
+    collection.delete_many
+  end
+
+  context 'a bulk write with one error' do
+    it 'reports code name, code and message' do
+      begin
+        collection.insert_many([
+          {_id: 1},
+          {_id: 1},
+          {_id: 1},
+        ], ordered: true)
+        fail('Should have raised')
+      rescue Mongo::Error::BulkWriteError => e
+        e.message.should =~ %r,\A\[11000\]: (insertDocument :: caused by :: 11000 )?E11000 duplicate key error (collection|index):,
+      end
+    end
+  end
+
+  context 'a bulk write with multiple errors' do
+    it 'reports code name, code and message' do
+      begin
+        collection.insert_many([
+          {_id: 1},
+          {_id: 1},
+          {_id: 1},
+        ], ordered: false)
+        fail('Should have raised')
+      rescue Mongo::Error::BulkWriteError => e
+        e.message.should =~ %r,\AMultiple errors: \[11000\]: (insertDocument :: caused by :: 11000 )?E11000 duplicate key error (collection|index):.*\[11000\]: (insertDocument :: caused by :: 11000 )?E11000 duplicate key error (collection|index):,
+      end
+    end
+  end
+end

data/spec/integration/change_stream_spec.rb

@@ -98,7 +98,7 @@ describe 'Change stream integration', retry: 4 do
       it 'watch raises error' do
         expect do
           client['change-stream'].watch
-        end.to raise_error(Mongo::Error::OperationFailure, /Failing command due to 'failCommand' failpoint \(10107\)/)
+        end.to raise_error(Mongo::Error::OperationFailure, /10107\b.*Failing command due to 'failCommand' failpoint/)
       end
     end
 
@@ -283,7 +283,7 @@ describe 'Change stream integration', retry: 4 do
 
         expect do
           enum.next
-        end.to raise_error(Mongo::Error::OperationFailure, /Failing command due to 'failCommand' failpoint \(101\)/)
+        end.to raise_error(Mongo::Error::OperationFailure, /101\b.*Failing command due to 'failCommand' failpoint/)
       end
     end
   end
@@ -414,7 +414,7 @@ describe 'Change stream integration', retry: 4 do
 
           expect do
             enum.try_next
-          end.to raise_error(Mongo::Error::OperationFailure, /Failing command due to 'failCommand' failpoint \(10107\)/)
+          end.to raise_error(Mongo::Error::OperationFailure, /10107\b.*Failing command due to 'failCommand' failpoint/)
         end
       end
 
@@ -441,7 +441,7 @@ describe 'Change stream integration', retry: 4 do
 
           expect do
             enum.try_next
-          end.to raise_error(Mongo::Error::OperationFailure, /Failing command due to 'failCommand' failpoint \(10107\)/)
+          end.to raise_error(Mongo::Error::OperationFailure, /10107\b.*Failing command due to 'failCommand' failpoint/)
         end
       end
     end

data/spec/integration/command_monitoring_spec.rb

@@ -135,8 +135,8 @@ describe 'Command monitoring' do
 
       subscriber.clear_events!
       expect do
-        command.execute(server, client: nil)
-      end.to raise_error(Mongo::Error::OperationFailure, /Not enough data-bearing nodes \(100\)/)
+        command.execute(server, context: Mongo::Operation::Context.new(session: session))
+      end.to raise_error(Mongo::Error::OperationFailure, /100\b.*Not enough data-bearing nodes/)
 
       expect(subscriber.started_events.length).to eq(1)
       event = subscriber.started_events.first

data/spec/integration/connection_spec.rb

@@ -232,6 +232,8 @@ describe 'Connections' do
     end
 
     describe 'wire protocol version range update' do
+      require_no_required_api_version
+
       # 3.2 wire protocol is 4.
       # Wire protocol < 2 means only scram auth is available,
       # which is not supported by modern mongos.