mongo 2.15.0 → 2.16.1

data/spec/shared/lib/mrss/event_subscriber.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+module Mrss
+  # Test event subscriber.
+  class EventSubscriber
+
+    # The mappings of event names to types.
+    MAPPINGS = {
+      'topology_opening_event' => Mongo::Monitoring::Event::TopologyOpening,
+      'topology_description_changed_event' => Mongo::Monitoring::Event::TopologyChanged,
+      'topology_closed_event' => Mongo::Monitoring::Event::TopologyClosed,
+      'server_opening_event' => Mongo::Monitoring::Event::ServerOpening,
+      'server_description_changed_event' => Mongo::Monitoring::Event::ServerDescriptionChanged,
+      'server_closed_event' => Mongo::Monitoring::Event::ServerClosed
+    }.freeze
+
+    attr_reader :all_events
+
+    attr_reader :started_events
+
+    attr_reader :succeeded_events
+
+    attr_reader :failed_events
+
+    attr_reader :published_events
+
+    # @param [ String ] name Optional name for the event subscriber.
+    def initialize(name: nil)
+      @mutex = Mutex.new
+      clear_events!
+      @name = name
+    end
+
+    def to_s
+      %Q`#<EventSubscriber:#{@name ? "\"#{@name}\"" : '%x' % object_id} \
+  started=#{started_events.length} \
+  succeeded=#{succeeded_events.length} \
+  failed=#{failed_events.length} \
+  published=#{published_events.length}>`
+    end
+
+    alias :inspect :to_s
+
+    # Event retrieval
+
+    def select_started_events(cls)
+      started_events.select do |event|
+        event.is_a?(cls)
+      end
+    end
+
+    def select_succeeded_events(cls)
+      succeeded_events.select do |event|
+        event.is_a?(cls)
+      end
+    end
+
+    def select_completed_events(*classes)
+      (succeeded_events + failed_events).select do |event|
+        classes.any? { |c| c === event }
+      end
+    end
+
+    def select_published_events(cls)
+      published_events.select do |event|
+        event.is_a?(cls)
+      end
+    end
+
+    # Filters command started events for the specified command name.
+    def command_started_events(command_name)
+      started_events.select do |event|
+        event.command[command_name]
+      end
+    end
+
+    def non_auth_command_started_events
+      started_events.reject do |event|
+        %w(authenticate getnonce saslSstart saslContinue).any? do |cmd|
+          event.command[cmd]
+        end
+      end
+    end
+
+    # Locates command stated events for the specified command name,
+    # asserts that there is exactly one such event, and returns it.
+    def single_command_started_event(command_name, include_auth: false)
+      events = if include_auth
+                 started_events
+               else
+                 non_auth_command_started_events
+               end
+      events.select! do |event|
+        event.command[command_name]
+      end
+      if events.length != 1
+        raise "Expected a single #{command_name} event but we have #{events.length}"
+      end
+      events.first
+    end
+
+
+    # Get the first succeeded event published for the name, and then delete it.
+    #
+    # @param [ String ] name The event name.
+    #
+    # @return [ Event ] The matching event.
+    def first_event(name)
+      cls = MAPPINGS[name]
+      if cls.nil?
+        raise ArgumentError, "Bogus event name #{name}"
+      end
+      matching = succeeded_events.find do |event|
+        cls === event
+      end
+      succeeded_events.delete(matching)
+      matching
+    end
+
+    # Event recording
+
+    # Cache the started event.
+    #
+    # @param [ Event ] event The event.
+    def started(event)
+      @mutex.synchronize do
+        started_events << event
+        all_events << event
+      end
+    end
+
+    # Cache the succeeded event.
+    #
+    # @param [ Event ] event The event.
+    def succeeded(event)
+      @mutex.synchronize do
+        succeeded_events << event
+        all_events << event
+      end
+    end
+
+    # Cache the failed event.
+    #
+    # @param [ Event ] event The event.
+    def failed(event)
+      @mutex.synchronize do
+        failed_events << event
+        all_events << event
+      end
+    end
+
+    def published(event)
+      @mutex.synchronize do
+        published_events << event
+        all_events << event
+      end
+    end
+
+    # Clear all cached events.
+    def clear_events!
+      @all_events = []
+      @started_events = []
+      @succeeded_events = []
+      @failed_events = []
+      @published_events = []
+      self
+    end
+  end
+  # Only handles succeeded events correctly.
+  class PhasedEventSubscriber < EventSubscriber
+    def initialize
+      super
+      @phase_events = {}
+    end
+
+    def phase_finished(phase_index)
+      @phase_events[phase_index] = succeeded_events
+      @succeeded_events = []
+    end
+
+    def phase_events(phase_index)
+      @phase_events[phase_index]
+    end
+
+    def event_count
+      @phase_events.inject(0) do |sum, event|
+        sum + event.length
+      end
+    end
+  end
+
+  class VerboseEventSubscriber < EventSubscriber
+    %w(started succeeded failed published).each do |meth|
+      define_method(meth) do |event|
+        puts event.summary
+        super(event)
+      end
+    end
+  end
+end

data/spec/shared/lib/mrss/server_version_registry.rb

@@ -48,19 +48,24 @@ module Mrss
         url = dl['archive']['url']
       end
     rescue MissingDownloadUrl
-      if %w(4.7 4.7.0).include?(desired_version)
-        # 4.7.0 has no advertised downloads but it is downloadable and
-        # we do need it. Dirty hack below.
-        registry = self.class.new('4.4.3', arch)
-        registry.download_url.sub('4.4.3', '4.7.0').tap do |url|
-          # Sanity check - ensure the URL we hacked up is a valid one
-          io = uri_open(url)
-          begin
-            io.read(1)
-          ensure
-            io.close
-          end
+      if %w(2.6 3.0).include?(desired_version) && arch == 'ubuntu1604'
+        # 2.6 and 3.0 are only available for ubuntu1204 and ubuntu1404.
+        # Those ubuntus have ancient Pythons that don't work due to not
+        # implementing recent TLS protocols.
+        # Because of this we test on ubuntu1604 which has a newer Python.
+        # But we still need to retrieve ubuntu1404-targeting builds.
+        url = self.class.new('3.2', arch).download_url
+        unless url.include?('3.2.')
+          raise 'URL not in expected format'
         end
+        url = case desired_version
+        when '2.6'
+          url.sub(/\b3\.2\.\d+/, '2.6.12')
+        when '3.0'
+          url.sub(/\b3\.2\.\d+/, '3.0.15')
+        else
+          raise NotImplementedError
+        end.sub('ubuntu1604', 'ubuntu1404')
       else
         raise
       end

data/spec/shared/share/Dockerfile.erb

@@ -55,7 +55,7 @@ FROM <%= base_image %>
   # Ruby runtime dependencies: libyaml-0-2
   # Compiling ruby libraries: gcc make
   # Compiling pyhton packages: python2.7-dev
-  # JRuby: openjdk-8-jre-headless
+  # JRuby: openjdk-8-jdk-headless
   # Server dependencies: libsnmp30 libcurl3/libcurl4
   # Determining OS we are running on: lsb-release
   # Load balancer testing: haproxy
@@ -68,6 +68,7 @@ FROM <%= base_image %>
   # nokogiri: zlib1g-dev
   # Mongoid testing: tzdata shared-mime-info
   # Mongoid application testing: nodejs (8.x or newer)
+  # Test suite: procps for ps (to kill JRubies)
   #
   # We currently use Python 2-compatible version of mtools, which
   # is installable via pip (which uses Python 2). All of the MongoDB
@@ -75,7 +76,7 @@ FROM <%= base_image %>
   # therefore install python-pip in all configurations here.
 
   <% packages = %w(
-    lsb-release bzip2 curl zsh
+    procps lsb-release bzip2 curl zsh
     git make gcc libyaml-0-2 libgmp-dev zlib1g-dev libsnappy-dev
     krb5-user krb5-kdc krb5-admin-server libsasl2-dev libsasl2-modules-gssapi-mit
     haproxy
@@ -103,11 +104,11 @@ FROM <%= base_image %>
   <% end %>
 
   <% if distro =~ /debian10/ %>
-    <% packages << 'openjdk-11-jre-headless' %>
+    <% packages << 'openjdk-11-jdk-headless' %>
   <% elsif distro =~ /ubuntu1404/ %>
     # Ubuntu 14.04 only has openjdk 7, this is too old to be useful
   <% else %>
-    <% packages << 'openjdk-8-jre-headless' %>
+    <% packages << 'openjdk-8-jdk-headless' %>
   <% end %>
 
   # ubuntu1404, ubuntu1604: libcurl3

data/spec/shared/shlib/server.sh

@@ -10,21 +10,21 @@ set_fcv() {
 
 add_uri_option() {
   opt=$1
-  
+
   if ! echo $MONGODB_URI |sed -e s,//,, |grep -q /; then
     MONGODB_URI="$MONGODB_URI/"
   fi
-  
+
   if ! echo $MONGODB_URI |grep -q '?'; then
     MONGODB_URI="$MONGODB_URI?"
   fi
-  
+
   MONGODB_URI="$MONGODB_URI&$opt"
 }
 
 prepare_server() {
   arch=$1
-  
+
   if test -n "$USE_OPT_MONGODB"; then
     export BINDIR=/opt/mongodb/bin
     export PATH=$BINDIR:$PATH
@@ -39,7 +39,7 @@ prepare_server() {
   else
     download_version="$MONGODB_VERSION"
   fi
-  
+
   url=`$(dirname $0)/get-mongodb-download-url $download_version $arch`
 
   prepare_server_from_url $url
@@ -80,7 +80,7 @@ install_mlaunch_pip() {
     # mlaunch is preinstalled in the docker image, do not install it here
     return
   fi
-  
+
   python -V || true
   python3 -V || true
   pythonpath="$MONGO_ORCHESTRATION_HOME"/python
@@ -96,20 +96,20 @@ install_mlaunch_git() {
   python3 -V || true
   which pip || true
   which pip3 || true
-  
+
   if false; then
     if ! virtualenv --version; then
       python3 `which pip3` install --user virtualenv
       export PATH=$HOME/.local/bin:$PATH
       virtualenv --version
     fi
-    
+
     venvpath="$MONGO_ORCHESTRATION_HOME"/venv
     virtualenv -p python3 $venvpath
     . $venvpath/bin/activate
-    
+
     pip3 install psutil pymongo
-    
+
     git clone $repo mlaunch
     cd mlaunch
     git checkout origin/$branch
@@ -118,13 +118,13 @@ install_mlaunch_git() {
   else
     pip install --user 'virtualenv==13'
     export PATH=$HOME/.local/bin:$PATH
-    
+
     venvpath="$MONGO_ORCHESTRATION_HOME"/venv
     virtualenv $venvpath
     . $venvpath/bin/activate
-  
+
     pip install psutil pymongo
-    
+
     git clone $repo mlaunch
     (cd mlaunch &&
       git checkout origin/$branch &&
@@ -133,26 +133,32 @@ install_mlaunch_git() {
   fi
 }
 
+# This function sets followong global variables:
+#   server_cert_path
+#   server_ca_path
+#   server_client_cert_path
+#
+# These variables are used later to connect to processes via mongo client.
 calculate_server_args() {
   local mongo_version=`echo $MONGODB_VERSION |tr -d .`
-  
+
   if test -z "$mongo_version"; then
     echo "$MONGODB_VERSION must be set and not contain only dots" 1>&2
     exit 3
   fi
-  
+
   if test $mongo_version = latest; then
     mongo_version=49
   fi
 
   local args="--setParameter enableTestCommands=1"
-  
+
   if test $mongo_version -ge 50; then
     args="$args --setParameter acceptApiVersion2=1"
   elif test $mongo_version -ge 47; then
     args="$args --setParameter acceptAPIVersion2=1"
   fi
-  
+
   # diagnosticDataCollectionEnabled is a mongod-only parameter on server 3.2,
   # and mlaunch does not support specifying mongod-only parameters:
   # https://github.com/rueckstiess/mtools/issues/696
@@ -209,7 +215,6 @@ calculate_server_args() {
     fi
   fi
 
-  local server_cert_path server_ca_path server_client_cert_path
   if test "$SSL" = ssl || test -n "$OCSP_ALGORITHM"; then
     if test -n "$OCSP_ALGORITHM"; then
       if test "$OCSP_MUST_STAPLE" = 1; then
@@ -288,7 +293,7 @@ calculate_server_args() {
       ocsp_args="$ocsp_args --fault $OCSP_STATUS"
     fi
   fi
-  
+
   OCSP_ARGS="$ocsp_args"
   SERVER_ARGS="$args"
   URI_OPTIONS="$uri_options"
@@ -306,12 +311,57 @@ launch_server() {
   local dbdir="$1"
   python -m mtools.mlaunch.mlaunch --dir "$dbdir" --binarypath "$BINDIR" $SERVER_ARGS
 
+  if test "$TOPOLOGY" = sharded-cluster && test $MONGODB_VERSION = 3.6; then
+    # On 3.6 server the sessions collection is not immediately available,
+    # so we run the refreshLogicalSessionCacheNow command on the config server
+    # and again on each mongos in order for the mongoses
+    # to correctly report logicalSessionTimeoutMinutes.
+    mongos_regex="\s*mongos\s+([0-9]+)\s+running\s+[0-9]+"
+    config_server_regex="\s*config\sserver\s+([0-9]+)\s+running\s+[0-9]+"
+    config_server=""
+    mongoses=()
+    if test "$AUTH" = auth
+    then
+      base_url="mongodb://bob:pwd123@localhost"
+    else
+      base_url="mongodb://localhost"
+    fi
+    if test "$SSL" = "ssl"
+    then
+      mongo_command="${BINDIR}/mongo --ssl --sslPEMKeyFile $server_cert_path --sslCAFile $server_ca_path"
+    else
+      mongo_command="${BINDIR}/mongo"
+    fi
+
+    while read -r line
+    do
+        if [[ $line =~ $config_server_regex ]]
+        then
+            port="${BASH_REMATCH[1]}"
+            config_server="${base_url}:${port}"
+        fi
+        if [[ $line =~ $mongos_regex ]]
+        then
+            port="${BASH_REMATCH[1]}"
+            mongoses+=("${base_url}:${port}")
+        fi
+    done < <(python -m mtools.mlaunch.mlaunch list --dir "$dbdir" --binarypath "$BINDIR")
+
+    if [ -n "$config_server" ]; then
+      ${mongo_command} "$config_server" --eval 'db.adminCommand("refreshLogicalSessionCacheNow")'
+      for mongos in ${mongoses[*]}
+      do
+        ${mongo_command} "$mongos" --eval 'db.adminCommand("refreshLogicalSessionCacheNow")'
+      done
+    fi
+  fi
+
   if test "$TOPOLOGY" = load-balanced; then
     if test -z "$haproxy_config"; then
       echo haproxy_config should have been set 1>&2
       exit 3
     fi
-    
+
     haproxy -D -f $haproxy_config -p $mongodb_dir/haproxy.pid
   fi
 }

data/spec/spec_tests/change_streams_spec.rb

@@ -8,7 +8,7 @@ require 'runners/change_streams/spec'
 
 describe 'ChangeStreams' do
   require_wired_tiger
-  require_no_multi_shard
+  require_no_multi_mongos
 
   CHANGE_STREAMS_TESTS.each do |file|
 

data/spec/spec_tests/cmap_spec.rb

@@ -40,7 +40,7 @@ describe 'Cmap' do
     context("#{spec.description} (#{file.sub(%r'.*/data/cmap/', '')})") do
 
       before do
-        subscriber = EventSubscriber.new
+        subscriber = Mrss::EventSubscriber.new
 
         monitoring = Mongo::Monitoring.new(monitoring: false)
         monitoring.subscribe(Mongo::Monitoring::CONNECTION_POOL, subscriber)
@@ -76,6 +76,9 @@ describe 'Cmap' do
         allow(socket).to receive(:alive?).and_return(true)
 
         allow_any_instance_of(Mongo::Server::Connection).to receive(:do_connect).and_return(socket)
+        if @server.load_balancer?
+          allow_any_instance_of(Mongo::Server::Connection).to receive(:service_id).and_return('very fake')
+        end
         spec.run
       end
 

data/spec/spec_tests/command_monitoring_spec.rb

@@ -24,7 +24,7 @@ describe 'Command Monitoring Events' do
           end
 
           let(:subscriber) do
-            EventSubscriber.new
+            Mrss::EventSubscriber.new
           end
 
           let(:monitoring) do
@@ -54,7 +54,7 @@ describe 'Command Monitoring Events' do
 
   def check_event(subscriber, index, expectation)
     subscriber.all_events.length.should > index
-    # TODO move this filtering into EventSubscriber
+    # TODO move this filtering into Mrss::EventSubscriber
     events = subscriber.all_events.reject do |event|
       (
         event.is_a?(Mongo::Monitoring::Event::CommandStarted) ||

data/spec/spec_tests/data/command_monitoring/find.yml

@@ -43,14 +43,13 @@ tests:
         filter: { _id: { $gt: 1 } }
         sort: { _id: 1 }
         skip: {"$numberLong": "2"}
-        modifiers:
-          $comment: "test"
-          $hint: { _id: 1 }
-          $max: { _id: 6 }
-          $maxTimeMS: 6000
-          $min: { _id: 0 }
-          $returnKey: false
-          $showDiskLoc: false
+        comment: "test"
+        hint: { _id: 1 }
+        max: { _id: 6 }
+        maxTimeMS: 6000
+        min: { _id: 0 }
+        returnKey: false
+        showRecordId: false
     expectations:
       -
         command_started_event:
@@ -195,8 +194,9 @@ tests:
               - { $numberLong: "42" }
           command_name: "killCursors"
   -
-    description: "A successful find event with a getmore and the server kills the cursor"
+    description: "A successful find event with a getmore and the server kills the cursor (<= 4.4)"
     ignore_if_server_version_less_than: "3.1"
+    ignore_if_server_version_greater_than: "4.4"
     ignore_if_topology_type:
       - "sharded"
     operation:

data/spec/spec_tests/data/crud/read/aggregate-collation.yml

@@ -1,6 +1,7 @@
 data:
     - {_id: 1, x: 'ping'}
 minServerVersion: '3.4'
+serverless: 'forbid'
 
 tests:
     -
@@ -11,7 +12,7 @@ tests:
                 pipeline:
                     - $match:
                         x: 'PING'
-                collation: { locale: 'en_US', strength: 2 } # https://docs.mongodb.com/master/reference/collation/#collation-document
+                collation: { locale: 'en_US', strength: 2 } # https://docs.mongodb.com/manual/reference/collation/#collation-document
         outcome:
             result:
                 - {_id: 1, x: 'ping'}

data/spec/spec_tests/data/crud/read/aggregate-out.yml

@@ -3,6 +3,7 @@ data:
     - {_id: 2, x: 22}
     - {_id: 3, x: 33}
 minServerVersion: '2.6'
+serverless: 'forbid'
 
 tests:
     -

data/spec/spec_tests/data/crud/read/count-collation.yml

@@ -1,6 +1,7 @@
 data:
     - {_id: 1, x: 'PING'}
 minServerVersion: '3.4'
+serverless: 'forbid'
 
 tests:
     -
@@ -9,7 +10,7 @@ tests:
             name: countDocuments
             arguments:
                 filter: { x: 'ping' }
-                collation: { locale: 'en_US', strength: 2 } # https://docs.mongodb.com/master/reference/collation/#collation-document
+                collation: { locale: 'en_US', strength: 2 } # https://docs.mongodb.com/manual/reference/collation/#collation-document
 
         outcome:
             result: 1

data/spec/spec_tests/data/crud/read/distinct-collation.yml

@@ -2,6 +2,7 @@ data:
     - {_id: 1, string: 'PING'}
     - {_id: 2, string: 'ping'}
 minServerVersion: '3.4'
+serverless: 'forbid'
 
 tests:
     -
@@ -10,7 +11,7 @@ tests:
             name: distinct
             arguments:
                 fieldName: "string"
-                collation: { locale: 'en_US', strength: 2 } # https://docs.mongodb.com/master/reference/collation/#collation-document
+                collation: { locale: 'en_US', strength: 2 } # https://docs.mongodb.com/manual/reference/collation/#collation-document
 
         outcome:
             result:

data/spec/spec_tests/data/crud/read/find-collation.yml

@@ -1,6 +1,7 @@
 data:
     - {_id: 1, x: 'ping'}
 minServerVersion: '3.4'
+serverless: 'forbid'
 
 tests:
     -
@@ -9,7 +10,7 @@ tests:
             name: "find"
             arguments:
                 filter: {x: 'PING'}
-                collation: { locale: 'en_US', strength: 2 } # https://docs.mongodb.com/master/reference/collation/#collation-document
+                collation: { locale: 'en_US', strength: 2 } # https://docs.mongodb.com/manual/reference/collation/#collation-document
         outcome:
             result:
                 - {_id: 1, x: 'ping'}

data/spec/spec_tests/data/crud/write/bulkWrite-collation.yml

@@ -6,8 +6,9 @@ data:
     - {_id: 5, x: 'pONg'}
 
 minServerVersion: '3.4'
+serverless: 'forbid'
 
-# See: https://docs.mongodb.com/master/reference/collation/#collation-document
+# See: https://docs.mongodb.com/manual/reference/collation/#collation-document
 tests:
     -
         description: "BulkWrite with delete operations and collation"

data/spec/spec_tests/data/crud/write/deleteMany-collation.yml

@@ -3,6 +3,7 @@ data:
     - {_id: 2, x: 'ping'}
     - {_id: 3, x: 'pINg'}
 minServerVersion: '3.4'
+serverless: 'forbid'
 
 tests:
     -
@@ -12,7 +13,7 @@ tests:
             arguments:
                 filter:
                     x: 'PING'
-                collation: { locale: 'en_US', strength: 2 } # https://docs.mongodb.com/master/reference/collation/#collation-document
+                collation: { locale: 'en_US', strength: 2 } # https://docs.mongodb.com/manual/reference/collation/#collation-document
 
         outcome:
             result:

data/spec/spec_tests/data/crud/write/deleteOne-collation.yml

@@ -3,6 +3,7 @@ data:
     - {_id: 2, x: 'ping'}
     - {_id: 3, x: 'pINg'}
 minServerVersion: '3.4'
+serverless: 'forbid'
 
 tests:
     -
@@ -11,7 +12,7 @@ tests:
             name: "deleteOne"
             arguments:
                 filter: {x: 'PING'}
-                collation: { locale: 'en_US', strength: 2 } # https://docs.mongodb.com/master/reference/collation/#collation-document
+                collation: { locale: 'en_US', strength: 2 } # https://docs.mongodb.com/manual/reference/collation/#collation-document
 
         outcome:
             result: