mongo 2.4.1 → 2.14.0

data/spec/support/authorization.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2009-2016 MongoDB, Inc.
+# Copyright (C) 2009-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,158 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# The default test database for all specs.
-#
-# @since 2.0.0
-TEST_DB = 'ruby-driver'.freeze
-
 # The default test collection.
 #
 # @since 2.0.0
 TEST_COLL = 'test'.freeze
 
-# The seed addresses to be used when creating a client.
-#
-# @since 2.0.0
-ADDRESSES = ENV['MONGODB_ADDRESSES'] ? ENV['MONGODB_ADDRESSES'].split(',').freeze :
-              [ '127.0.0.1:27017' ].freeze
-
-# The topology type.
-#
-# @since 2.0.0
-CONNECT = ENV['RS_ENABLED'] == 'true' ? { connect: :replica_set, replica_set: ENV['RS_NAME'] } :
-          ENV['SHARDED_ENABLED'] == 'true' ? { connect: :sharded } :
-          { connect: :direct }
-
-# The write concern to use in the tests.
-#
-# @since 2.0.0
-WRITE_CONCERN = CONNECT[:connect] == :replica_set ? { w: ADDRESSES.size } : { w: 1 }
-
-# Whether to use SSL.
-#
-# @since 2.0.3
-SSL = ENV['SSL_ENABLED'] == 'true'
-
-# SSL options.
-#
-# @since 2.1.0
-SSL_OPTIONS = {
-                  ssl: SSL,
-                  ssl_verify: false,
-                  ssl_cert:  CLIENT_CERT_PEM,
-                  ssl_key:  CLIENT_KEY_PEM
-                }
-
-# Base test options.
-#
-# @since 2.1.0
-BASE_OPTIONS = {
-                  max_pool_size: 1,
-                  write: WRITE_CONCERN,
-                  heartbeat_frequency: 20,
-                  max_read_retries: 5
-               }
-
-# Options for test suite clients.
-#
-# @since 2.0.3
-TEST_OPTIONS = BASE_OPTIONS.merge(CONNECT).merge(SSL_OPTIONS)
-
-# The root user name.
-#
-# @since 2.0.0
-ROOT_USER_NAME = ENV['ROOT_USER_NAME'] || 'root-user'
-
-# The root user password.
+# An invalid write concern.
 #
-# @since 2.0.0
-ROOT_USER_PWD = ENV['ROOT_USER_PWD'] || 'password'
-
-# Gets the root system administrator user.
-#
-# @since 2.0.0
-ROOT_USER = Mongo::Auth::User.new(
-  user: ROOT_USER_NAME,
-  password: ROOT_USER_PWD,
-  roles: [
-    Mongo::Auth::Roles::USER_ADMIN_ANY_DATABASE,
-    Mongo::Auth::Roles::DATABASE_ADMIN_ANY_DATABASE,
-    Mongo::Auth::Roles::READ_WRITE_ANY_DATABASE,
-    Mongo::Auth::Roles::HOST_MANAGER,
-    Mongo::Auth::Roles::CLUSTER_ADMIN
-  ]
-)
-
-# Get the default test user for the suite on versions 2.6 and higher.
-#
-# @since 2.0.0
-TEST_USER = Mongo::Auth::User.new(
-  database: Mongo::Database::ADMIN,
-  user: 'test-user',
-  password: 'password',
-  roles: [
-    { role: Mongo::Auth::Roles::READ_WRITE, db: TEST_DB },
-    { role: Mongo::Auth::Roles::DATABASE_ADMIN, db: TEST_DB },
-    { role: Mongo::Auth::Roles::READ_WRITE, db: 'invalid_database' },
-    { role: Mongo::Auth::Roles::DATABASE_ADMIN, db: 'invalid_database' }
-  ]
-)
-
-# MongoDB 2.4 and lower does not allow hashes as roles, so we need to create a
-# user on those versions for each database permission in order to ensure the
-# legacy roles work with users. The following users are those.
-
-# Gets the default test user for the suite on 2.4 and lower.
-#
-# @since 2.0.
-TEST_READ_WRITE_USER = Mongo::Auth::User.new(
-  database: TEST_DB,
-  user: TEST_USER.name,
-  password: TEST_USER.password,
-  roles: [ Mongo::Auth::Roles::READ_WRITE, Mongo::Auth::Roles::DATABASE_ADMIN ]
-)
-
-# Provides an authorized mongo client on the default test database for the
-# default test user.
-#
-# @since 2.0.0
-AUTHORIZED_CLIENT = Mongo::Client.new(
-  ADDRESSES,
-  TEST_OPTIONS.merge(
-    database: TEST_DB,
-    user: TEST_USER.name,
-    password: TEST_USER.password)
-)
-
-# Provides an unauthorized mongo client on the default test database.
-#
-# @since 2.0.0
-UNAUTHORIZED_CLIENT = Mongo::Client.new(
-  ADDRESSES,
-  TEST_OPTIONS.merge(database: TEST_DB, monitoring: false)
-)
-
-# Provides an unauthorized mongo client on the admin database, for use in
-# setting up the first admin root user.
-#
-# @since 2.0.0
-ADMIN_UNAUTHORIZED_CLIENT = Mongo::Client.new(
-  ADDRESSES,
-  TEST_OPTIONS.merge(database: Mongo::Database::ADMIN, monitoring: false)
-)
-
-# Get an authorized client on the test database logged in as the admin
-# root user.
-#
-# @since 2.0.0
-ADMIN_AUTHORIZED_TEST_CLIENT = ADMIN_UNAUTHORIZED_CLIENT.with(
-  user: ROOT_USER.name,
-  password: ROOT_USER.password,
-  database: TEST_DB,
-  auth_source: Mongo::Database::ADMIN,
-  monitoring: false
-)
+# @since 2.4.2
+INVALID_WRITE_CONCERN = { w: 4 }
 
 module Authorization
 
@@ -177,35 +34,76 @@ module Authorization
     # Gets the root system administrator user.
     #
     # @since 2.0.0
-    context.let(:root_user) { ROOT_USER }
+    context.let(:root_user) { SpecConfig.instance.root_user }
 
     # Get the default test user for the suite.
     #
     # @since 2.0.0
-    context.let(:test_user) { TEST_USER }
+    context.let(:test_user) { SpecConfig.instance.test_user }
 
     # Provides an authorized mongo client on the default test database for the
     # default test user.
     #
     # @since 2.0.0
-    context.let(:authorized_client) { AUTHORIZED_CLIENT }
+    context.let(:authorized_client) { ClientRegistry.instance.global_client('authorized') }
+
+    # A client with a different cluster, for testing session use across
+    # clients
+    context.let(:another_authorized_client) do
+      new_local_client(
+        SpecConfig.instance.addresses,
+        SpecConfig.instance.test_options.merge(
+          database: SpecConfig.instance.test_db,
+          user: SpecConfig.instance.test_user.name,
+          password: SpecConfig.instance.test_user.password,
+          heartbeat_frequency: 10,
+        ),
+      )
+    end
+
+    # Provides an authorized mongo client on the default test database that retries writes.
+    #
+    # @since 2.5.1
+    context.let(:authorized_client_with_retry_writes) do
+      ClientRegistry.instance.global_client('authorized_with_retry_writes')
+    end
+
+    context.let(:authorized_client_without_retry_writes) do
+      ClientRegistry.instance.global_client('authorized_without_retry_writes')
+    end
+
+    context.let(:authorized_client_without_retry_reads) do
+      ClientRegistry.instance.global_client('authorized_without_retry_reads')
+    end
+
+    context.let(:authorized_client_without_any_retry_reads) do
+      ClientRegistry.instance.global_client('authorized_without_any_retry_reads')
+    end
+
+    context.let(:authorized_client_without_any_retries) do
+      ClientRegistry.instance.global_client('authorized_without_any_retries')
+    end
 
     # Provides an unauthorized mongo client on the default test database.
     #
     # @since 2.0.0
-    context.let!(:unauthorized_client) { UNAUTHORIZED_CLIENT }
+    context.let(:unauthorized_client) { ClientRegistry.instance.global_client('unauthorized') }
 
     # Provides an unauthorized mongo client on the admin database, for use in
     # setting up the first admin root user.
     #
     # @since 2.0.0
-    context.let!(:admin_unauthorized_client) { ADMIN_UNAUTHORIZED_CLIENT }
+    context.let(:admin_unauthorized_client) { ClientRegistry.instance.global_client('admin_unauthorized') }
 
     # Get an authorized client on the test database logged in as the admin
     # root user.
     #
     # @since 2.0.0
-    context.let!(:root_authorized_client) { ADMIN_AUTHORIZED_TEST_CLIENT }
+    context.let(:root_authorized_client) { ClientRegistry.instance.global_client('root_authorized') }
+
+    context.let(:root_authorized_admin_client) do
+      ClientRegistry.instance.global_client('root_authorized').use(:admin)
+    end
 
     # Gets the default test collection from the authorized client.
     #

data/spec/support/aws_utils/base.rb

@@ -0,0 +1,134 @@
+module AwsUtils
+  class Base
+
+    def initialize(access_key_id: nil, secret_access_key: nil, region: nil, **options)
+      @access_key_id = access_key_id || ENV['AWS_ACCESS_KEY_ID']
+      @secret_access_key = secret_access_key || ENV['AWS_SECRET_ACCESS_KEY']
+      @region = region || ENV['AWS_REGION']
+      @options = options
+    end
+
+    attr_reader :access_key_id, :secret_access_key, :region, :options
+
+    private
+
+    def detect_object(resp, resp_attr, object_attr, value)
+      resp.each do |batch|
+        batch.send(resp_attr).each do |object|
+          if object.send(object_attr) == value
+            return object
+          end
+        end
+      end
+      nil
+    end
+
+    def ssh_security_group_id
+      begin
+        sg = ec2_client.describe_security_groups(
+          group_names: [AWS_AUTH_SECURITY_GROUP_NAME],
+        ).security_groups.first
+        sg&.group_id
+      rescue Aws::EC2::Errors::InvalidGroupNotFound
+        # Unlike almost all other describe calls, this one raises an exception
+        # if there isn't a security group matching the criteria.
+        nil
+      end
+    end
+
+    def ssh_security_group_id!
+      ssh_security_group_id.tap do |security_group_id|
+        if security_group_id.nil?
+          raise 'Security group does not exist, please provision'
+        end
+      end
+    end
+
+    def ssh_vpc_security_group_id
+      begin
+        # If the top-level group_name parameter is used, only non-VPC
+        # security groups are returned which does not find the VPC group
+        # we are looking for here.
+        sg = ec2_client.describe_security_groups(
+          filters: [{
+            name: 'group-name',
+            values: [AWS_AUTH_VPC_SECURITY_GROUP_NAME],
+          }],
+        ).security_groups.first
+        sg&.group_id
+      rescue Aws::EC2::Errors::InvalidGroupNotFound
+        # Unlike almost all other describe calls, this one raises an exception
+        # if there isn't a security group matching the criteria.
+        nil
+      end
+    end
+
+    def ssh_vpc_security_group_id!
+      ssh_vpc_security_group_id.tap do |security_group_id|
+        if security_group_id.nil?
+          raise 'Security group does not exist, please provision'
+        end
+      end
+    end
+
+    def subnet_id
+      # This directly queries the subnets for the one with the expected
+      # CIDR block, to save on the number of requests made to AWS.
+      ec2_client.describe_subnets(
+        filters: [{
+          name: 'cidr-block',
+          values: [AWS_AUTH_VPC_CIDR],
+        }],
+      ).subnets.first&.subnet_id
+    end
+
+    def subnet_id!
+      subnet_id.tap do |subnet_id|
+        if subnet_id.nil?
+          raise 'Subnet does not exist, please provision'
+        end
+      end
+    end
+
+    def credentials
+      Aws::Credentials.new(access_key_id, secret_access_key)
+    end
+
+    public
+
+    def ec2_client
+      @ec2_client ||= Aws::EC2::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def iam_client
+      iam_client = Aws::IAM::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def ecs_client
+      @ecs_client ||= Aws::ECS::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def logs_client
+      @logs_client ||= Aws::CloudWatchLogs::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def sts_client
+      @sts_client ||= Aws::STS::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+  end
+end

data/spec/support/aws_utils/inspector.rb

@@ -0,0 +1,224 @@
+module AwsUtils
+  class Inspector < Base
+
+    def list_key_pairs
+      ec2_client.describe_key_pairs.key_pairs.each do |key_pair|
+        puts key_pair.key_name
+      end
+    end
+
+    def assume_role_arn
+      assume_role = detect_object(iam_client.list_roles, :roles, :role_name, AWS_AUTH_ASSUME_ROLE_NAME)
+      if assume_role.nil?
+        raise 'No user found, please run `aws setup-resources`'
+      end
+      assume_role.arn
+    end
+
+    def ecs_status(cluster_name: AWS_AUTH_ECS_CLUSTER_NAME,
+      service_name: AWS_AUTH_ECS_SERVICE_NAME,
+      get_public_ip: true, get_logs: true
+    )
+      service = ecs_client.describe_services(
+        cluster: cluster_name,
+        services: [service_name],
+      ).services.first
+      if service.nil?
+        raise "No service #{service_name} in cluster #{cluster_name} - provision first"
+      end
+
+      # When Ruby driver tooling is used, task definition generation is
+      # going up on each service launch, and service name is the fixed.
+      # When testing in Evergreen, generation is fixed because we do not
+      # change the task definition, but service name is different for
+      # each test run.
+      if service.task_definition =~ /:(\d+)$/
+        generation = $1
+        puts "Current task definition generation: #{generation} for service: #{service_name}"
+      else
+        raise 'Could not determine task definition generation'
+      end
+
+      colors = {
+        'running' => :green,
+        'pending' => :yellow,
+        'stopped' => :red,
+      }
+      # Pending status in the API includes tasks in provisioning status as
+      # show in the AWS console.
+      #
+      # The API returns the tasks unordered, in particular the latest task
+      # may be in the middle of the list following relatively ancient tasks.
+      # Collect all tasks in a single list and order them by generation.
+      # We expect to have a single task per generation.
+      tasks = []
+      %w(running pending stopped).each do |status|
+        resp = ecs_client.list_tasks(
+          cluster: cluster_name,
+          service_name: service_name,
+          desired_status: status,
+        )
+        task_arns = resp.map(&:task_arns).flatten
+        if task_arns.empty?
+          next
+        end
+        ecs_client.describe_tasks(
+          cluster: cluster_name,
+          tasks: task_arns,
+        ).each do |tbatch|
+          unless tbatch.failures.empty?
+            # The task list endpoint does not raise an exception if it can't
+            # find the tasks, but reports "failures".
+            puts "Failures for #{task_arns.join(', ')}:"
+            tbatch.failures.each do |failure|
+              puts "#{failure.arn}: #{failure.reason}"
+              next
+            end
+          end
+          tbatch.tasks.each do |task|
+            tasks << task
+          end
+        end
+      end
+
+      tasks.each do |task|
+        class << task
+          def generation
+            @generation ||= if task_definition_arn =~ /:(\d+)$/
+              $1.to_i
+            else
+              raise 'Could not determine generation'
+            end
+          end
+
+          def task_uuid
+            @uuid ||= task_arn.split('/').last
+          end
+        end
+      end
+
+      tasks = tasks.sort_by do |task|
+        -task.generation
+      end.first(3)
+
+      running_task = nil
+      running_private_ip = nil
+      running_public_ip = nil
+
+      if tasks.empty?
+        puts 'No tasks in the cluster'
+      end
+      tasks.each do |task|
+        status = task.last_status.downcase
+
+        status_ext = case status
+        when 'stopped'
+          ": #{task.stopped_reason}"
+        else
+          ''
+        end
+        decorated_status = Paint[status.upcase, colors[status]]
+        puts "Task for generation #{task.generation}: #{decorated_status}#{status_ext} (uuid: #{task.task_uuid})"
+        if status == 'running'
+          puts "Task ARN: #{task.task_arn}"
+          running_task ||= task
+        end
+        task.containers.each do |container|
+          if container.reason
+            puts container.reason
+          end
+        end
+
+        if status == 'running'
+          attachment = detect_object([task], :attachments, :type, 'ElasticNetworkInterface')
+          ip = detect_object([attachment], :details, :name, 'privateIPv4Address')
+          if ip
+            private_ip = ip.value
+            running_private_ip ||= private_ip
+          end
+          msg = "Private IP: #{private_ip}"
+          if get_public_ip
+            niid = detect_object([attachment], :details, :name, 'networkInterfaceId')
+            network_interface = ec2_client.describe_network_interfaces(
+              network_interface_ids: [niid.value],
+            ).network_interfaces.first
+            public_ip =  network_interface&.association&.public_ip
+            running_public_ip ||= public_ip
+            msg += ", public IP: #{public_ip}"
+          end
+          puts msg
+        end
+        puts
+      end
+
+      puts
+      task_ids = []
+      max_event_count = 5
+      event_count = 0
+      service = ecs_client.describe_services(
+        cluster: cluster_name,
+        services: [service_name],
+      ).services.first
+      if service.nil?
+        puts 'Service is missing'
+      else
+        if service.events.empty?
+          puts 'No events for service'
+        else
+          puts "Events for #{service.service_arn}:"
+          service.events.each do |event|
+            event_count += 1
+            break if event_count > max_event_count
+            if event.message =~ /\(task (\w+)\)/
+              task_ids << $1
+            end
+            puts "#{event.created_at.strftime('%Y-%m-%d %H:%M:%S %z')} #{event.message}"
+          end
+        end
+      end
+
+      if get_logs && running_task
+        puts
+        log_stream_name = "task/ssh/#{running_task.task_uuid}"
+        log_stream = logs_client.describe_log_streams(
+          log_group_name: AWS_AUTH_ECS_LOG_GROUP,
+          log_stream_name_prefix: log_stream_name,
+        ).log_streams.first
+        if log_stream
+          log_events = logs_client.get_log_events(
+            log_group_name: AWS_AUTH_ECS_LOG_GROUP,
+            log_stream_name: log_stream_name,
+            end_time: Time.now.to_i * 1000,
+            limit: 100,
+          ).events
+          if log_events.any?
+            puts "Task logs for task #{running_task.task_uuid}:"
+            log_events.each do |event|
+              puts "[#{Time.at(event.timestamp/1000r).strftime('%Y-%m-%d %H:%M:%S %z')}] #{event.message}"
+            end
+          else
+            puts "No CloudWatch events in the log stream for task #{running_task.task_uuid}"
+          end
+        else
+          puts "No CloudWatch log stream for task #{running_task.task_uuid}"
+        end
+      end
+
+      if running_public_ip
+        puts
+        puts "ip=#{running_public_ip}"
+        puts "ssh -o StrictHostKeyChecking=false root@#{running_public_ip}"
+      end
+
+      {
+        private_ip: running_private_ip,
+      }
+    end
+
+    private
+
+    def ucfirst(str)
+      str[0].upcase + str[1...str.length]
+    end
+  end
+end