mongo 2.14.1 → 2.15.0.alpha

data/spec/spec_tests/data/versioned_api/transaction-handling.yml

@@ -0,0 +1,180 @@
+description: "Transaction handling"
+
+schemaVersion: "1.1"
+
+runOnRequirements:
+  - minServerVersion: "4.9"
+    topologies: [ replicaset, sharded-replicaset ]
+
+createEntities:
+  - client:
+      id: &client client
+      observeEvents:
+        - commandStartedEvent
+      serverApi:
+        version: "1"
+  - database:
+      id: &database database
+      client: *client
+      databaseName: &databaseName versioned-api-tests
+  - collection:
+      id: &collection collection
+      database: *database
+      collectionName: &collectionName test
+  - session:
+      id: &session session
+      client: *client
+
+_yamlAnchors:
+  versions:
+    - &expectedApiVersion
+      apiVersion: "1"
+      apiStrict: { $$unsetOrMatches: false }
+      apiDeprecationErrors: { $$unsetOrMatches: false }
+    - &noApiVersion
+      apiVersion: { $$exists: false }
+      apiStrict: { $$exists: false }
+      apiDeprecationErrors: { $$exists: false }
+
+
+initialData:
+  - collectionName: *collectionName
+    databaseName: *databaseName
+    documents:
+      - { _id: 1, x: 11 }
+      - { _id: 2, x: 22 }
+      - { _id: 3, x: 33 }
+      - { _id: 4, x: 44 }
+      - { _id: 5, x: 55 }
+
+tests:
+  - description: "Only the first command in a transaction declares an API version"
+    runOnRequirements:
+      - topologies: [ replicaset, sharded-replicaset ]
+    operations:
+      - name: startTransaction
+        object: *session
+      - name: insertOne
+        object: *collection
+        arguments:
+          session: *session
+          document: { _id: 6, x: 66 }
+        expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 6 } } }
+      - name: insertOne
+        object: *collection
+        arguments:
+          session: *session
+          document: { _id: 7, x: 77 }
+        expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 7 } } }
+      - name: commitTransaction
+        object: *session
+    expectEvents:
+      - client: *client
+        events:
+          - commandStartedEvent:
+              command:
+                insert: *collectionName
+                documents: [ { _id: 6, x: 66 } ]
+                lsid: { $$sessionLsid: *session }
+                startTransaction: true
+                <<: *expectedApiVersion
+          - commandStartedEvent:
+              command:
+                insert: *collectionName
+                documents: [ { _id: 7, x: 77 } ]
+                lsid: { $$sessionLsid: *session }
+                <<: *noApiVersion
+          - commandStartedEvent:
+              command:
+                commitTransaction: 1
+                lsid: { $$sessionLsid: *session }
+                <<: *noApiVersion
+  - description: "Committing a transaction twice does not append server API options"
+    runOnRequirements:
+      - topologies: [ replicaset, sharded-replicaset ]
+    operations:
+      - name: startTransaction
+        object: *session
+      - name: insertOne
+        object: *collection
+        arguments:
+          session: *session
+          document: { _id: 6, x: 66 }
+        expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 6 } } }
+      - name: insertOne
+        object: *collection
+        arguments:
+          session: *session
+          document: { _id: 7, x: 77 }
+        expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 7 } } }
+      - name: commitTransaction
+        object: *session
+      - name: commitTransaction
+        object: *session
+    expectEvents:
+      - client: *client
+        events:
+          - commandStartedEvent:
+              command:
+                insert: *collectionName
+                documents: [ { _id: 6, x: 66 } ]
+                lsid: { $$sessionLsid: *session }
+                startTransaction: true
+                <<: *expectedApiVersion
+          - commandStartedEvent:
+              command:
+                insert: *collectionName
+                documents: [ { _id: 7, x: 77 } ]
+                lsid: { $$sessionLsid: *session }
+                <<: *noApiVersion
+          - commandStartedEvent:
+              command:
+                commitTransaction: 1
+                lsid: { $$sessionLsid: *session }
+                <<: *noApiVersion
+          - commandStartedEvent:
+              command:
+                commitTransaction: 1
+                lsid: { $$sessionLsid: *session }
+                <<: *noApiVersion
+  - description: "abortTransaction does not include an API version"
+    runOnRequirements:
+      - topologies: [ replicaset, sharded-replicaset ]
+    operations:
+      - name: startTransaction
+        object: *session
+      - name: insertOne
+        object: *collection
+        arguments:
+          session: *session
+          document: { _id: 6, x: 66 }
+        expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 6 } } }
+      - name: insertOne
+        object: *collection
+        arguments:
+          session: *session
+          document: { _id: 7, x: 77 }
+        expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 7 } } }
+      - name: abortTransaction
+        object: *session
+    expectEvents:
+      - client: *client
+        events:
+          - commandStartedEvent:
+              command:
+                insert: *collectionName
+                documents: [ { _id: 6, x: 66 } ]
+                lsid: { $$sessionLsid: *session }
+                startTransaction: true
+                <<: *expectedApiVersion
+          - commandStartedEvent:
+              command:
+                insert: *collectionName
+                documents: [ { _id: 7, x: 77 } ]
+                lsid: { $$sessionLsid: *session }
+                <<: *noApiVersion
+          - commandStartedEvent:
+              command:
+                abortTransaction: 1
+                lsid: { $$sessionLsid: *session }
+                <<: *noApiVersion

data/spec/spec_tests/unified_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+require 'runners/unified'
+
+base = "#{CURRENT_PATH}/spec_tests/data/unified"
+PASS_UNIFIED_TESTS = Dir.glob("#{base}/valid-pass/**/*.yml").sort
+FAIL_UNIFIED_TESTS = Dir.glob("#{base}/valid-fail/**/*.yml").sort
+
+describe 'Unified spec tests' do
+  define_unified_spec_tests(base, PASS_UNIFIED_TESTS)
+end
+
+describe 'Unified spec tests - expected failures' do
+  define_unified_spec_tests(base, FAIL_UNIFIED_TESTS, expect_failure: true)
+end

data/spec/spec_tests/uri_options_spec.rb

@@ -50,6 +50,22 @@ describe 'URI options' do
             end
 
             if test.options
+              if test.options['compressors'] && test.options['compressors'].include?('snappy')
+                before do
+                  unless ENV.fetch('BUNDLE_GEMFILE', '') =~ /snappy/
+                    skip "This test requires snappy compression"
+                  end
+                end
+              end
+
+              if test.options['compressors'] && test.options['compressors'].include?('zstd')
+                before do
+                  unless ENV.fetch('BUNDLE_GEMFILE', '') =~ /zstd/
+                    skip "This test requires zstd compression"
+                  end
+                end
+              end
+
               it 'creates a client with the correct options' do
                 mapped = Mongo::URI::OptionsMapper.new.ruby_to_smc(test.client.options)
                 expected = Mongo::ConnectionString.adjust_expected_mongo_client_options(

data/spec/spec_tests/versioned_api_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+
+require 'runners/unified'
+
+base = "#{CURRENT_PATH}/spec_tests/data/versioned_api"
+UNIFIED_TESTS = Dir.glob("#{base}/**/*.yml").sort
+
+describe 'Versioned API spec tests' do
+  define_unified_spec_tests(base, UNIFIED_TESTS)
+end

data/spec/support/client_registry.rb

@@ -188,14 +188,10 @@ class ClientRegistry
   end
   private :new_global_client
 
-  def new_local_client(*args, &block)
-    if block_given?
-      Mongo::Client.new(*args, &block)
-    else
-      Mongo::Client.new(*args).tap do |client|
-        @lock.synchronize do
-          @local_clients << client
-        end
+  def new_local_client(*args)
+    Mongo::Client.new(*args).tap do |client|
+      @lock.synchronize do
+        @local_clients << client
       end
     end
   end

data/spec/support/client_registry_macros.rb

@@ -1,9 +1,9 @@
 module ClientRegistryMacros
-  def new_local_client(address, options=nil, &block)
-    ClientRegistry.instance.new_local_client(address, options, &block)
+  def new_local_client(address, options=nil)
+    ClientRegistry.instance.new_local_client(address, options)
   end
 
-  def new_local_client_nmio(address, options=nil, &block)
+  def new_local_client_nmio(address, options=nil)
     # Avoid type converting options.
     base_options = {monitoring_io: false}
     if BSON::Document === options || options&.keys&.any? { |key| String === key }
@@ -14,7 +14,7 @@ module ClientRegistryMacros
     else
       base_options
     end
-    new_local_client(address, options, &block)
+    new_local_client(address, options)
   end
 
   def close_local_clients

data/spec/support/common_shortcuts.rb

@@ -126,7 +126,19 @@ module CommonShortcuts
         process = ChildProcess.new(*args)
 
         process.io.inherit!
-        process.start
+
+        retried = false
+        begin
+          process.start
+        rescue
+          if retried
+            raise
+          else
+            sleep 1
+            retried = true
+            retry
+          end
+        end
 
         begin
           sleep 0.4
@@ -210,6 +222,8 @@ module CommonShortcuts
       allow(cluster).to receive(:app_metadata)
       allow(cluster).to receive(:options).and_return({})
       allow(cluster).to receive(:run_sdam_flow)
+      allow(cluster).to receive(:monitor_app_metadata)
+      allow(cluster).to receive(:push_monitor_app_metadata)
       allow(cluster).to receive(:heartbeat_interval).and_return(10)
       server = Mongo::Server.new(address, cluster, monitoring, listeners,
         SpecConfig.instance.test_options.merge(monitoring_io: false))

data/spec/support/shared/session.rb

@@ -54,9 +54,9 @@ shared_examples 'an operation using a session' do
       end
 
       it 'raises an exception' do
-        expect {
+        expect do
           operation_result
-        }.to raise_exception(Mongo::Error::InvalidSession)
+        end.to raise_exception(Mongo::Error::InvalidSession)
       end
     end
 

data/spec/support/spec_config.rb

@@ -8,6 +8,7 @@ class SpecConfig
   # only loading the lite spec helper. Do I/O eagerly in accessor methods.
   def initialize
     @uri_options = {}
+    @ruby_options = {}
     if ENV['MONGODB_URI']
       @mongodb_uri = Mongo::URI.new(ENV['MONGODB_URI'])
       @uri_options = Mongo::Options::Mapper.transform_keys_to_symbols(@mongodb_uri.uri_options)
@@ -46,9 +47,19 @@ class SpecConfig
     end
 
     @ssl ||= false
+
+    if (server_api = ENV['SERVER_API']) && !server_api.empty?
+      @ruby_options[:server_api] = BSON::Document.new(YAML.load(server_api))
+      # Since the tests pass options provided by SpecConfig directly to
+      # internal driver objects (e.g. connections), transform server api
+      # parameters here as they would be transformed by Client constructor.
+      if (v = @ruby_options[:server_api][:version]).is_a?(Integer)
+        @ruby_options[:server_api][:version] = v.to_s
+      end
+    end
   end
 
-  attr_reader :uri_options, :connect_options
+  attr_reader :uri_options, :ruby_options, :connect_options
 
   def addresses
     @addresses ||= begin
@@ -75,7 +86,7 @@ class SpecConfig
       # https://github.com/10gen/mongo-orchestration/issues/268
       client = Mongo::Client.new(addresses, Mongo::Options::Redacted.new(
         server_selection_timeout: 5.03,
-      ).merge(ssl_options))
+      ).merge(ssl_options).merge(ruby_options))
 
       begin
         case client.cluster.topology.class.name
@@ -110,14 +121,6 @@ class SpecConfig
     !!(RUBY_PLATFORM =~ /\bjava\b/)
   end
 
-  def linux?
-    !!(RbConfig::CONFIG['host_os'].downcase =~ /\blinux/)
-  end
-
-  def macos?
-    !!(RbConfig::CONFIG['host_os'].downcase =~ /\bdarwin/)
-  end
-
   def platform
     RUBY_PLATFORM
   end
@@ -427,6 +430,20 @@ EOT
     {retry_writes: retry_writes?}
   end
 
+  # The options needed for a successful socket connection to the server(s).
+  # These exclude options needed to handshake (e.g. server api parameters).
+  def connection_options
+    ssl_options
+  end
+
+  # The options needed for successful monitoring of the server(s).
+  # These exclude options needed to perform operations (e.g. credentials).
+  def monitoring_options
+    ssl_options.merge(
+      server_api: ruby_options[:server_api],
+    )
+  end
+
   # Base test options.
   def base_test_options
     {
@@ -461,7 +478,9 @@ EOT
       # Uncomment to have exceptions in background threads log complete
       # backtraces.
       #bg_error_backtrace: true,
-   }
+    }.merge(ruby_options).merge(
+      server_api: ruby_options[:server_api] && ::Utils.underscore_hash(ruby_options[:server_api])
+    )
   end
 
   # Options for test suite clients.
@@ -535,6 +554,18 @@ EOT
         { role: Mongo::Auth::Roles::DATABASE_ADMIN, db: 'withTransaction-tests' },
         { role: Mongo::Auth::Roles::READ_WRITE, db: 'admin' },
         { role: Mongo::Auth::Roles::DATABASE_ADMIN, db: 'admin' },
+        { role: Mongo::Auth::Roles::READ_WRITE, db: 'command-monitoring-tests' },
+        { role: Mongo::Auth::Roles::DATABASE_ADMIN, db: 'command-monitoring-tests' },
+        { role: Mongo::Auth::Roles::READ_WRITE, db: 'session-tests' },
+        { role: Mongo::Auth::Roles::DATABASE_ADMIN, db: 'session-tests' },
+        { role: Mongo::Auth::Roles::READ_WRITE, db: 'gridfs-tests' },
+        { role: Mongo::Auth::Roles::DATABASE_ADMIN, db: 'gridfs-tests' },
+        { role: Mongo::Auth::Roles::READ_WRITE, db: 'change-stream-tests' },
+        { role: Mongo::Auth::Roles::DATABASE_ADMIN, db: 'change-stream-tests' },
+        { role: Mongo::Auth::Roles::READ_WRITE, db: 'change-stream-tests-2' },
+        { role: Mongo::Auth::Roles::DATABASE_ADMIN, db: 'change-stream-tests-2' },
+        { role: Mongo::Auth::Roles::READ_WRITE, db: 'retryable-writes-tests' },
+        { role: Mongo::Auth::Roles::DATABASE_ADMIN, db: 'retryable-writes-tests' },
       ]
     )
   end

data/spec/support/utils.rb

@@ -34,16 +34,43 @@ module Utils
   #
   # For example, { 'fooBar' => { 'baz' => 'bingBing', :x => 1 } } converts to
   # { :foo_bar => { :baz => :bing_bing, :x => 1 } }.
-  def snakeize_hash(value)
-    return underscore(value) if value.is_a?(String)
+  module_function def shallow_underscore_hash(value)
     return value unless value.is_a?(Hash)
 
     value.reduce({}) do |hash, (k, v)|
       hash.tap do |h|
-        h[underscore(k)] = snakeize_hash(v)
+        h[underscore(k)] = v
       end
     end
   end
+
+  # Creates a copy of a hash where all keys and string values are converted to
+  # snake-case symbols.
+  #
+  # For example, { 'fooBar' => { 'baz' => 'bingBing', :x => 1 } } converts to
+  # { :foo_bar => { :baz => :bing_bing, :x => 1 } }.
+  def snakeize_hash(value)
+    return underscore(value) if value.is_a?(String)
+    case value
+    when Array
+      value.map do |sub|
+        case sub
+        when Hash
+          snakeize_hash(sub)
+        else
+          sub
+        end
+      end
+    when Hash
+      value.reduce({}) do |hash, (k, v)|
+        hash.tap do |h|
+          h[underscore(k)] = snakeize_hash(v)
+        end
+      end
+    else
+      value
+    end
+  end
   module_function :snakeize_hash
 
   # Like snakeize_hash but does not recurse.
@@ -380,6 +407,40 @@ module Utils
   end
   module_function :match_with_type?
 
+  # Takes a timeout and a block. Waits up to the specified timeout until
+  # the value of the block is true. If timeout is reached, this method
+  # returns normally and does not raise an exception. The block is invoked
+  # every second or so.
+  module_function def wait_for_condition(timeout)
+    deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
+    loop do
+      if yield
+        break
+      end
+      if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
+        break
+      end
+      sleep 1
+    end
+  end
+
+  module_function def ensure_port_free(port)
+    TCPServer.open(port) do
+      # Nothing
+    end
+  end
+
+  module_function def wait_for_port_free(port, timeout)
+    wait_for_condition(timeout) do
+      begin
+        ensure_port_free(port)
+        true
+      rescue Errno::EADDRINUSE
+        false
+      end
+    end
+  end
+
   module_function def get_ec2_metadata_token(ttl: 30, http: nil)
     http ||= Net::HTTP.new('169.254.169.254')
     req = Net::HTTP::Put.new('/latest/api/token',