mongo 2.12.4 → 2.13.0.beta1

data/spec/stress/fork_reconnect_stress_spec.rb

@@ -0,0 +1,109 @@
+require 'spec_helper'
+
+describe 'fork reconnect' do
+  require_fork
+  only_mri
+
+  # On multi-shard sharded clusters a succeeding write request does not
+  # guarantee that the next operation will succeed (since it could be sent to
+  # another shard with a dead connection).
+  require_no_multi_shard
+
+  # On Ruby 2.3 $?.exitstatus is sometimes nil after Process.wait returns which
+  # is not supposed to happen.
+  ruby_version_gte '2.4'
+
+  require_stress
+
+  let(:client) { authorized_client }
+
+  describe 'client' do
+    it 'works after fork' do
+      # Perform a write so that we discover the current primary.
+      # Previous test may have stepped down the server that authorized client
+      # considers the primary.
+      # In standalone deployments there are no retries, hence execute the
+      # operation twice manually.
+      client['foo'].insert_one(test: 1) rescue nil
+      client['foo'].insert_one(test: 1)
+
+      pids = []
+      deadline = Time.now + 5
+      1.upto(10) do
+        if pid = fork
+          pids << pid
+        else
+          Utils.wrap_forked_child do
+            while Time.now < deadline
+              client.database.command(ismaster: 1).should be_a(Mongo::Operation::Result)
+            end
+          end
+        end
+      end
+
+      while Time.now < deadline
+        # Use a read which is retried in case of an error
+        client['foo'].find(test: 1).to_a
+      end
+
+      pids.each do |pid|
+        pid, status = Process.wait2(pid)
+        status.exitstatus.should == 0
+      end
+    end
+
+    context 'when parent is operating on client during the fork', retry: 3 do
+      # This test intermittently fails in evergreen with pool size of 5,
+      # with a number of pending connections in the pool.
+      # The reason could be that handshaking is slow or operations are slow
+      # post handshakes.
+      # Sometimes it seems the monitoring connection experiences network
+      # errors (despite being a loopback connection) which causes the test
+      # to fail as then server selection fails.
+      # The retry: 3 is to deal with network errors on monitoring connection.
+
+      let(:client) { authorized_client.with(max_pool_size: 10,
+        wait_queue_timeout: 10, socket_timeout: 2, connect_timeout: 2) }
+
+      it 'works' do
+        client.database.command(ismaster: 1).should be_a(Mongo::Operation::Result)
+
+        threads = []
+        5.times do
+          threads << Thread.new do
+            loop do
+              client['foo'].find(test: 1).to_a
+            end
+          end
+        end
+
+        pids = []
+        deadline = Time.now + 5
+        10.times do
+          if pid = fork
+            pids << pid
+          else
+            Utils.wrap_forked_child do
+              while Time.now < deadline
+                client.database.command(ismaster: 1).should be_a(Mongo::Operation::Result)
+              end
+            end
+          end
+        end
+
+        while Time.now < deadline
+          sleep 0.1
+        end
+
+        threads.map(&:kill)
+        threads.map(&:join)
+
+        pids.each do |pid|
+          pid, status = Process.wait2(pid)
+          status.exitstatus.should == 0
+        end
+      end
+
+    end
+  end
+end

data/spec/support/authorization.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2009-2019 MongoDB, Inc.
+# Copyright (C) 2009-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -65,12 +65,10 @@ module Authorization
     #
     # @since 2.5.1
     context.let(:authorized_client_with_retry_writes) do
-      EventSubscriber.clear_events!
       ClientRegistry.instance.global_client('authorized_with_retry_writes')
     end
 
     context.let(:authorized_client_without_retry_writes) do
-      EventSubscriber.clear_events!
       ClientRegistry.instance.global_client('authorized_without_retry_writes')
     end
 
@@ -86,14 +84,6 @@ module Authorization
       ClientRegistry.instance.global_client('authorized_without_any_retries')
     end
 
-    # Provides an authorized mongo client that has a Command subscriber.
-    #
-    # @since 2.5.1
-    context.let(:subscribed_client) do
-      EventSubscriber.clear_events!
-      ClientRegistry.instance.global_client('subscribed')
-    end
-
     # Provides an unauthorized mongo client on the default test database.
     #
     # @since 2.0.0

data/spec/support/aws_utils.rb

@@ -0,0 +1,62 @@
+autoload :Byebug, 'byebug'
+autoload :Paint, 'paint'
+
+module Aws
+  autoload :CloudWatchLogs, 'aws-sdk-cloudwatchlogs'
+  autoload :Credentials, 'aws-sdk-core'
+  autoload :EC2, 'aws-sdk-ec2'
+  autoload :ECS, 'aws-sdk-ecs'
+  autoload :IAM, 'aws-sdk-iam'
+  autoload :STS, 'aws-sdk-core'
+end
+
+module AwsUtils
+  NAMESPACE = 'mdb-ruby'.freeze
+
+  AWS_AUTH_REGULAR_USER_NAME = "#{NAMESPACE}.aws-auth-regular".freeze
+
+  AWS_AUTH_ASSUME_ROLE_NAME = "#{NAMESPACE}.assume-role".freeze
+
+  AWS_AUTH_SECURITY_GROUP_NAME = "#{NAMESPACE}.ssh".freeze
+
+  AWS_AUTH_VPC_GATEWAY_NAME = NAMESPACE
+
+  AWS_AUTH_VPC_SECURITY_GROUP_NAME = "#{NAMESPACE}.vpc-ssh".freeze
+
+  AWS_AUTH_VPC_CIDR = "10.42.142.64/28".freeze
+
+  AWS_AUTH_EC2_AMI_NAMES = {
+    # https://wiki.debian.org/Cloud/AmazonEC2Image/Buster
+    'debian10' => 'debian-10-amd64-20200210-166',
+    'ubuntu1604' => 'ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-20200317',
+  }.freeze
+
+  AWS_AUTH_EC2_INSTANCE_NAME = "#{NAMESPACE}.aws-auth)".freeze
+
+  AWS_AUTH_INSTANCE_PROFILE_NAME = "#{NAMESPACE}.ip".freeze
+
+  AWS_AUTH_ASSUME_ROLE_USER_POLICY_NAME = "#{NAMESPACE}.assume-role-user-policy".freeze
+
+  AWS_AUTH_EC2_ROLE_NAME = "#{NAMESPACE}.ec2-role".freeze
+
+  AWS_AUTH_ECS_CLUSTER_NAME = "#{NAMESPACE}_aws-auth".freeze
+
+  AWS_AUTH_ECS_TASK_FAMILY = "#{NAMESPACE}_aws-auth".freeze
+
+  AWS_AUTH_ECS_SERVICE_NAME = "#{NAMESPACE}_aws-auth".freeze
+
+  AWS_AUTH_ECS_LOG_GROUP = "/ecs/#{NAMESPACE}/aws-auth-ecs".freeze
+
+  AWS_AUTH_ECS_LOG_STREAM_PREFIX = "task".freeze
+
+  # This role allows ECS tasks access to output logs to CloudWatch.
+  AWS_AUTH_ECS_EXECUTION_ROLE_NAME = "#{NAMESPACE}.ecs-execution-role".freeze
+
+  # This role is assumed by ECS tasks.
+  AWS_AUTH_ECS_TASK_ROLE_NAME = "#{NAMESPACE}.ecs-task-role".freeze
+
+  autoload :Base, 'support/aws_utils/base'
+  autoload :Inspector, 'support/aws_utils/inspector'
+  autoload :Orchestrator, 'support/aws_utils/orchestrator'
+  autoload :Provisioner, 'support/aws_utils/provisioner'
+end

data/spec/support/aws_utils/base.rb

@@ -0,0 +1,134 @@
+module AwsUtils
+  class Base
+
+    def initialize(access_key_id: nil, secret_access_key: nil, region: nil, **options)
+      @access_key_id = access_key_id || ENV['AWS_ACCESS_KEY_ID']
+      @secret_access_key = secret_access_key || ENV['AWS_SECRET_ACCESS_KEY']
+      @region = region || ENV['AWS_REGION']
+      @options = options
+    end
+
+    attr_reader :access_key_id, :secret_access_key, :region, :options
+
+    private
+
+    def detect_object(resp, resp_attr, object_attr, value)
+      resp.each do |batch|
+        batch.send(resp_attr).each do |object|
+          if object.send(object_attr) == value
+            return object
+          end
+        end
+      end
+      nil
+    end
+
+    def ssh_security_group_id
+      begin
+        sg = ec2_client.describe_security_groups(
+          group_names: [AWS_AUTH_SECURITY_GROUP_NAME],
+        ).security_groups.first
+        sg&.group_id
+      rescue Aws::EC2::Errors::InvalidGroupNotFound
+        # Unlike almost all other describe calls, this one raises an exception
+        # if there isn't a security group matching the criteria.
+        nil
+      end
+    end
+
+    def ssh_security_group_id!
+      ssh_security_group_id.tap do |security_group_id|
+        if security_group_id.nil?
+          raise 'Security group does not exist, please provision'
+        end
+      end
+    end
+
+    def ssh_vpc_security_group_id
+      begin
+        # If the top-level group_name parameter is used, only non-VPC
+        # security groups are returned which does not find the VPC group
+        # we are looking for here.
+        sg = ec2_client.describe_security_groups(
+          filters: [{
+            name: 'group-name',
+            values: [AWS_AUTH_VPC_SECURITY_GROUP_NAME],
+          }],
+        ).security_groups.first
+        sg&.group_id
+      rescue Aws::EC2::Errors::InvalidGroupNotFound
+        # Unlike almost all other describe calls, this one raises an exception
+        # if there isn't a security group matching the criteria.
+        nil
+      end
+    end
+
+    def ssh_vpc_security_group_id!
+      ssh_vpc_security_group_id.tap do |security_group_id|
+        if security_group_id.nil?
+          raise 'Security group does not exist, please provision'
+        end
+      end
+    end
+
+    def subnet_id
+      # This directly queries the subnets for the one with the expected
+      # CIDR block, to save on the number of requests made to AWS.
+      ec2_client.describe_subnets(
+        filters: [{
+          name: 'cidr-block',
+          values: [AWS_AUTH_VPC_CIDR],
+        }],
+      ).subnets.first&.subnet_id
+    end
+
+    def subnet_id!
+      subnet_id.tap do |subnet_id|
+        if subnet_id.nil?
+          raise 'Subnet does not exist, please provision'
+        end
+      end
+    end
+
+    def credentials
+      Aws::Credentials.new(access_key_id, secret_access_key)
+    end
+
+    public
+
+    def ec2_client
+      @ec2_client ||= Aws::EC2::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def iam_client
+      iam_client = Aws::IAM::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def ecs_client
+      @ecs_client ||= Aws::ECS::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def logs_client
+      @logs_client ||= Aws::CloudWatchLogs::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+
+    def sts_client
+      @sts_client ||= Aws::STS::Client.new(
+        region: region,
+        credentials: credentials,
+      )
+    end
+  end
+end

data/spec/support/aws_utils/inspector.rb

@@ -0,0 +1,224 @@
+module AwsUtils
+  class Inspector < Base
+
+    def list_key_pairs
+      ec2_client.describe_key_pairs.key_pairs.each do |key_pair|
+        puts key_pair.key_name
+      end
+    end
+
+    def assume_role_arn
+      assume_role = detect_object(iam_client.list_roles, :roles, :role_name, AWS_AUTH_ASSUME_ROLE_NAME)
+      if assume_role.nil?
+        raise 'No user found, please run `aws setup-resources`'
+      end
+      assume_role.arn
+    end
+
+    def ecs_status(cluster_name: AWS_AUTH_ECS_CLUSTER_NAME,
+      service_name: AWS_AUTH_ECS_SERVICE_NAME,
+      get_public_ip: true, get_logs: true
+    )
+      service = ecs_client.describe_services(
+        cluster: cluster_name,
+        services: [service_name],
+      ).services.first
+      if service.nil?
+        raise "No service #{service_name} in cluster #{cluster_name} - provision first"
+      end
+
+      # When Ruby driver tooling is used, task definition generation is
+      # going up on each service launch, and service name is the fixed.
+      # When testing in Evergreen, generation is fixed because we do not
+      # change the task definition, but service name is different for
+      # each test run.
+      if service.task_definition =~ /:(\d+)$/
+        generation = $1
+        puts "Current task definition generation: #{generation} for service: #{service_name}"
+      else
+        raise 'Could not determine task definition generation'
+      end
+
+      colors = {
+        'running' => :green,
+        'pending' => :yellow,
+        'stopped' => :red,
+      }
+      # Pending status in the API includes tasks in provisioning status as
+      # show in the AWS console.
+      #
+      # The API returns the tasks unordered, in particular the latest task
+      # may be in the middle of the list following relatively ancient tasks.
+      # Collect all tasks in a single list and order them by generation.
+      # We expect to have a single task per generation.
+      tasks = []
+      %w(running pending stopped).each do |status|
+        resp = ecs_client.list_tasks(
+          cluster: cluster_name,
+          service_name: service_name,
+          desired_status: status,
+        )
+        task_arns = resp.map(&:task_arns).flatten
+        if task_arns.empty?
+          next
+        end
+        ecs_client.describe_tasks(
+          cluster: cluster_name,
+          tasks: task_arns,
+        ).each do |tbatch|
+          unless tbatch.failures.empty?
+            # The task list endpoint does not raise an exception if it can't
+            # find the tasks, but reports "failures".
+            puts "Failures for #{task_arns.join(', ')}:"
+            tbatch.failures.each do |failure|
+              puts "#{failure.arn}: #{failure.reason}"
+              next
+            end
+          end
+          tbatch.tasks.each do |task|
+            tasks << task
+          end
+        end
+      end
+
+      tasks.each do |task|
+        class << task
+          def generation
+            @generation ||= if task_definition_arn =~ /:(\d+)$/
+              $1.to_i
+            else
+              raise 'Could not determine generation'
+            end
+          end
+
+          def task_uuid
+            @uuid ||= task_arn.split('/').last
+          end
+        end
+      end
+
+      tasks = tasks.sort_by do |task|
+        -task.generation
+      end.first(3)
+
+      running_task = nil
+      running_private_ip = nil
+      running_public_ip = nil
+
+      if tasks.empty?
+        puts 'No tasks in the cluster'
+      end
+      tasks.each do |task|
+        status = task.last_status.downcase
+
+        status_ext = case status
+        when 'stopped'
+          ": #{task.stopped_reason}"
+        else
+          ''
+        end
+        decorated_status = Paint[status.upcase, colors[status]]
+        puts "Task for generation #{task.generation}: #{decorated_status}#{status_ext} (uuid: #{task.task_uuid})"
+        if status == 'running'
+          puts "Task ARN: #{task.task_arn}"
+          running_task ||= task
+        end
+        task.containers.each do |container|
+          if container.reason
+            puts container.reason
+          end
+        end
+
+        if status == 'running'
+          attachment = detect_object([task], :attachments, :type, 'ElasticNetworkInterface')
+          ip = detect_object([attachment], :details, :name, 'privateIPv4Address')
+          if ip
+            private_ip = ip.value
+            running_private_ip ||= private_ip
+          end
+          msg = "Private IP: #{private_ip}"
+          if get_public_ip
+            niid = detect_object([attachment], :details, :name, 'networkInterfaceId')
+            network_interface = ec2_client.describe_network_interfaces(
+              network_interface_ids: [niid.value],
+            ).network_interfaces.first
+            public_ip =  network_interface&.association&.public_ip
+            running_public_ip ||= public_ip
+            msg += ", public IP: #{public_ip}"
+          end
+          puts msg
+        end
+        puts
+      end
+
+      puts
+      task_ids = []
+      max_event_count = 5
+      event_count = 0
+      service = ecs_client.describe_services(
+        cluster: cluster_name,
+        services: [service_name],
+      ).services.first
+      if service.nil?
+        puts 'Service is missing'
+      else
+        if service.events.empty?
+          puts 'No events for service'
+        else
+          puts "Events for #{service.service_arn}:"
+          service.events.each do |event|
+            event_count += 1
+            break if event_count > max_event_count
+            if event.message =~ /\(task (\w+)\)/
+              task_ids << $1
+            end
+            puts "#{event.created_at.strftime('%Y-%m-%d %H:%M:%S %z')} #{event.message}"
+          end
+        end
+      end
+
+      if get_logs && running_task
+        puts
+        log_stream_name = "task/ssh/#{running_task.task_uuid}"
+        log_stream = logs_client.describe_log_streams(
+          log_group_name: AWS_AUTH_ECS_LOG_GROUP,
+          log_stream_name_prefix: log_stream_name,
+        ).log_streams.first
+        if log_stream
+          log_events = logs_client.get_log_events(
+            log_group_name: AWS_AUTH_ECS_LOG_GROUP,
+            log_stream_name: log_stream_name,
+            end_time: Time.now.to_i * 1000,
+            limit: 100,
+          ).events
+          if log_events.any?
+            puts "Task logs for task #{running_task.task_uuid}:"
+            log_events.each do |event|
+              puts "[#{Time.at(event.timestamp/1000r).strftime('%Y-%m-%d %H:%M:%S %z')}] #{event.message}"
+            end
+          else
+            puts "No CloudWatch events in the log stream for task #{running_task.task_uuid}"
+          end
+        else
+          puts "No CloudWatch log stream for task #{running_task.task_uuid}"
+        end
+      end
+
+      if running_public_ip
+        puts
+        puts "ip=#{running_public_ip}"
+        puts "ssh -o StrictHostKeyChecking=false root@#{running_public_ip}"
+      end
+
+      {
+        private_ip: running_private_ip,
+      }
+    end
+
+    private
+
+    def ucfirst(str)
+      str[0].upcase + str[1...str.length]
+    end
+  end
+end