mongo 2.12.1 → 2.13.0

data/spec/support/aws_utils/inspector.rb

@@ -0,0 +1,224 @@
+module AwsUtils
+  class Inspector < Base
+
+    def list_key_pairs
+      ec2_client.describe_key_pairs.key_pairs.each do |key_pair|
+        puts key_pair.key_name
+      end
+    end
+
+    def assume_role_arn
+      assume_role = detect_object(iam_client.list_roles, :roles, :role_name, AWS_AUTH_ASSUME_ROLE_NAME)
+      if assume_role.nil?
+        raise 'No user found, please run `aws setup-resources`'
+      end
+      assume_role.arn
+    end
+
+    def ecs_status(cluster_name: AWS_AUTH_ECS_CLUSTER_NAME,
+      service_name: AWS_AUTH_ECS_SERVICE_NAME,
+      get_public_ip: true, get_logs: true
+    )
+      service = ecs_client.describe_services(
+        cluster: cluster_name,
+        services: [service_name],
+      ).services.first
+      if service.nil?
+        raise "No service #{service_name} in cluster #{cluster_name} - provision first"
+      end
+
+      # When Ruby driver tooling is used, task definition generation is
+      # going up on each service launch, and service name is the fixed.
+      # When testing in Evergreen, generation is fixed because we do not
+      # change the task definition, but service name is different for
+      # each test run.
+      if service.task_definition =~ /:(\d+)$/
+        generation = $1
+        puts "Current task definition generation: #{generation} for service: #{service_name}"
+      else
+        raise 'Could not determine task definition generation'
+      end
+
+      colors = {
+        'running' => :green,
+        'pending' => :yellow,
+        'stopped' => :red,
+      }
+      # Pending status in the API includes tasks in provisioning status as
+      # show in the AWS console.
+      #
+      # The API returns the tasks unordered, in particular the latest task
+      # may be in the middle of the list following relatively ancient tasks.
+      # Collect all tasks in a single list and order them by generation.
+      # We expect to have a single task per generation.
+      tasks = []
+      %w(running pending stopped).each do |status|
+        resp = ecs_client.list_tasks(
+          cluster: cluster_name,
+          service_name: service_name,
+          desired_status: status,
+        )
+        task_arns = resp.map(&:task_arns).flatten
+        if task_arns.empty?
+          next
+        end
+        ecs_client.describe_tasks(
+          cluster: cluster_name,
+          tasks: task_arns,
+        ).each do |tbatch|
+          unless tbatch.failures.empty?
+            # The task list endpoint does not raise an exception if it can't
+            # find the tasks, but reports "failures".
+            puts "Failures for #{task_arns.join(', ')}:"
+            tbatch.failures.each do |failure|
+              puts "#{failure.arn}: #{failure.reason}"
+              next
+            end
+          end
+          tbatch.tasks.each do |task|
+            tasks << task
+          end
+        end
+      end
+
+      tasks.each do |task|
+        class << task
+          def generation
+            @generation ||= if task_definition_arn =~ /:(\d+)$/
+              $1.to_i
+            else
+              raise 'Could not determine generation'
+            end
+          end
+
+          def task_uuid
+            @uuid ||= task_arn.split('/').last
+          end
+        end
+      end
+
+      tasks = tasks.sort_by do |task|
+        -task.generation
+      end.first(3)
+
+      running_task = nil
+      running_private_ip = nil
+      running_public_ip = nil
+
+      if tasks.empty?
+        puts 'No tasks in the cluster'
+      end
+      tasks.each do |task|
+        status = task.last_status.downcase
+
+        status_ext = case status
+        when 'stopped'
+          ": #{task.stopped_reason}"
+        else
+          ''
+        end
+        decorated_status = Paint[status.upcase, colors[status]]
+        puts "Task for generation #{task.generation}: #{decorated_status}#{status_ext} (uuid: #{task.task_uuid})"
+        if status == 'running'
+          puts "Task ARN: #{task.task_arn}"
+          running_task ||= task
+        end
+        task.containers.each do |container|
+          if container.reason
+            puts container.reason
+          end
+        end
+
+        if status == 'running'
+          attachment = detect_object([task], :attachments, :type, 'ElasticNetworkInterface')
+          ip = detect_object([attachment], :details, :name, 'privateIPv4Address')
+          if ip
+            private_ip = ip.value
+            running_private_ip ||= private_ip
+          end
+          msg = "Private IP: #{private_ip}"
+          if get_public_ip
+            niid = detect_object([attachment], :details, :name, 'networkInterfaceId')
+            network_interface = ec2_client.describe_network_interfaces(
+              network_interface_ids: [niid.value],
+            ).network_interfaces.first
+            public_ip =  network_interface&.association&.public_ip
+            running_public_ip ||= public_ip
+            msg += ", public IP: #{public_ip}"
+          end
+          puts msg
+        end
+        puts
+      end
+
+      puts
+      task_ids = []
+      max_event_count = 5
+      event_count = 0
+      service = ecs_client.describe_services(
+        cluster: cluster_name,
+        services: [service_name],
+      ).services.first
+      if service.nil?
+        puts 'Service is missing'
+      else
+        if service.events.empty?
+          puts 'No events for service'
+        else
+          puts "Events for #{service.service_arn}:"
+          service.events.each do |event|
+            event_count += 1
+            break if event_count > max_event_count
+            if event.message =~ /\(task (\w+)\)/
+              task_ids << $1
+            end
+            puts "#{event.created_at.strftime('%Y-%m-%d %H:%M:%S %z')} #{event.message}"
+          end
+        end
+      end
+
+      if get_logs && running_task
+        puts
+        log_stream_name = "task/ssh/#{running_task.task_uuid}"
+        log_stream = logs_client.describe_log_streams(
+          log_group_name: AWS_AUTH_ECS_LOG_GROUP,
+          log_stream_name_prefix: log_stream_name,
+        ).log_streams.first
+        if log_stream
+          log_events = logs_client.get_log_events(
+            log_group_name: AWS_AUTH_ECS_LOG_GROUP,
+            log_stream_name: log_stream_name,
+            end_time: Time.now.to_i * 1000,
+            limit: 100,
+          ).events
+          if log_events.any?
+            puts "Task logs for task #{running_task.task_uuid}:"
+            log_events.each do |event|
+              puts "[#{Time.at(event.timestamp/1000r).strftime('%Y-%m-%d %H:%M:%S %z')}] #{event.message}"
+            end
+          else
+            puts "No CloudWatch events in the log stream for task #{running_task.task_uuid}"
+          end
+        else
+          puts "No CloudWatch log stream for task #{running_task.task_uuid}"
+        end
+      end
+
+      if running_public_ip
+        puts
+        puts "ip=#{running_public_ip}"
+        puts "ssh -o StrictHostKeyChecking=false root@#{running_public_ip}"
+      end
+
+      {
+        private_ip: running_private_ip,
+      }
+    end
+
+    private
+
+    def ucfirst(str)
+      str[0].upcase + str[1...str.length]
+    end
+  end
+end

data/spec/support/aws_utils/orchestrator.rb

@@ -0,0 +1,370 @@
+module AwsUtils
+  class Orchestrator < Base
+
+    def assume_role(role_arn)
+      # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/STS/Client.html#assume_role-instance_method
+      resp = sts_client.assume_role(
+        role_arn: role_arn,
+        role_session_name: "#{NAMESPACE}.test",
+      )
+      resp.credentials
+    end
+
+    def set_instance_profile(instance_id,
+      instance_profile_name: AWS_AUTH_INSTANCE_PROFILE_NAME,
+      instance_profile_arn: nil
+    )
+      clear_instance_profile(instance_id)
+
+      deadline = Time.now + 30
+      begin
+        ec2_client.associate_iam_instance_profile(
+          iam_instance_profile: {
+            name: instance_profile_name,
+            arn: instance_profile_arn,
+          },
+          instance_id: instance_id,
+        )
+      rescue Aws::EC2::Errors::RequestLimitExceeded => e
+        if Time.now >= deadline
+          raise
+        end
+        STDERR.puts("AWS request limit exceeded: #{e.class}: #{e}, will retry")
+        sleep 5
+        retry
+      end
+    end
+
+    def clear_instance_profile(instance_id)
+      assoc = detect_object(ec2_client.describe_iam_instance_profile_associations,
+        :iam_instance_profile_associations, :instance_id, instance_id)
+
+      if assoc
+        deadline = Time.now + 30
+        begin
+          ec2_client.disassociate_iam_instance_profile(
+            association_id: assoc.association_id,
+          )
+        rescue Aws::EC2::Errors::RequestLimitExceeded => e
+          if Time.now >= deadline
+            raise
+          end
+          STDERR.puts("AWS request limit exceeded: #{e.class}: #{e}, will retry")
+          sleep 5
+          retry
+        end
+      end
+    end
+
+    def provision_auth_ec2_instance(key_pair_name: nil, public_key_path: nil,
+      distro: 'ubuntu1604'
+    )
+      security_group_id = ssh_security_group_id!
+      reservations = ec2_client.describe_instances(filters: [
+        {name: 'tag:name', values: [AWS_AUTH_EC2_INSTANCE_NAME]},
+      ]).reservations
+      instance = find_running_instance(reservations)
+      if instance.nil?
+        ami_name = AWS_AUTH_EC2_AMI_NAMES.fetch(distro)
+        image = ec2_client.describe_images(
+          filters: [{name: 'name', values: [ami_name]}],
+        ).images.first
+        if public_key_path
+          public_key = File.read(public_key_path)
+          user_data = Base64.encode64(<<-CMD)
+#!/bin/sh
+for user in `ls /home`; do
+cd /home/$user &&
+mkdir -p .ssh &&
+chmod 0700 .ssh &&
+chown $user:$user .ssh &&
+cat <<-EOT |tee -a .ssh/authorized_keys
+#{public_key}
+EOT
+done
+CMD
+        end
+        resp = ec2_client.run_instances(
+          instance_type: 't3a.small',
+          image_id: image.image_id,
+          min_count: 1,
+          max_count: 1,
+          key_name: key_pair_name,
+          user_data: user_data,
+          tag_specifications: [{
+            resource_type: 'instance',
+            tags: [{key: 'name', value: AWS_AUTH_EC2_INSTANCE_NAME}],
+          }],
+          monitoring: {enabled: false},
+          credit_specification: {cpu_credits: 'standard'},
+          security_group_ids: [security_group_id],
+          metadata_options: {
+            # This is required for Docker containers on the instance to be able
+            # to use the instance metadata endpoints.
+            http_put_response_hop_limit: 2,
+          },
+        ).to_h
+        instance_id = resp[:instances].first[:instance_id]
+        reservations = ec2_client.describe_instances(instance_ids: [instance_id]).reservations
+        instance = find_running_instance(reservations)
+      end
+      if instance.nil?
+        raise "Instance should have been found here"
+      end
+      if instance.state.name == 'stopped'
+        ec2_client.start_instances(instance_ids: [instance.instance_id])
+      end
+      10.times do
+        if %w(stopped pending).include?(instance.state.name)
+          puts "Waiting for instance #{instance.instance_id} to start (current state: #{instance.state.name})"
+          sleep 5
+        end
+        reservations = ec2_client.describe_instances(instance_ids: [instance.instance_id]).reservations
+        instance = find_running_instance(reservations)
+      end
+      puts "Found usable instance #{instance.instance_id} at #{instance.public_ip_address}"
+    end
+
+    def terminate_auth_ec2_instance
+      ec2_client.describe_instances(filters: [
+        {name: 'tag:name', values: [AWS_AUTH_EC2_INSTANCE_NAME]},
+      ]).each do |resp|
+        resp.reservations.each do |res|
+          res.instances.each do |instance|
+            puts "Terminating #{instance.instance_id}"
+            ec2_client.terminate_instances(instance_ids: [instance.instance_id])
+          end
+        end
+      end
+    end
+
+    def provision_auth_ecs_task(public_key_path: nil,
+      cluster_name: AWS_AUTH_ECS_CLUSTER_NAME,
+      service_name: AWS_AUTH_ECS_SERVICE_NAME,
+      security_group_id: nil,
+      subnet_ids: nil,
+      task_definition_ref: AWS_AUTH_ECS_TASK_FAMILY
+    )
+      security_group_id ||= ssh_vpc_security_group_id!
+      subnet_ids ||= [subnet_id!]
+
+      # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ECS_AWSCLI_Fargate.html
+      resp = ecs_client.describe_clusters(
+        clusters: [cluster_name],
+      )
+      cluster = detect_object(resp, :clusters, :cluster_name, cluster_name)
+      if cluster.nil?
+        raise 'No cluster found, please run `aws setup-resources`'
+      end
+
+      if public_key_path
+        public_key = File.read(public_key_path)
+        unless public_key =~ /\Assh-/
+          raise "The file at #{public_key_path} does not look like a public key"
+        end
+        entry_point = ['bash', '-c', <<-CMD]
+          apt-get update &&
+          apt-get install -y openssh-server &&
+          cd /root &&
+          mkdir -p .ssh &&
+          chmod 0700 .ssh &&
+          cat >.ssh/authorized_keys  <<-EOT &&
+#{public_key}
+EOT
+          service ssh start &&
+          sleep 10000000
+          #mkdir /run/sshd && /usr/sbin/sshd -d
+CMD
+      else
+        entry_point = nil
+      end
+      launch_type = if options[:ec2]
+        'EC2'
+      else
+        'FARGATE'
+      end
+      # When testing in Evergreen, we are given the task definition ARN
+      # and we always launch the tasks with that ARN.
+      # When testing locally, we repace task definition every time we launch
+      # the service.
+      if task_definition_ref !~ /^arn:/
+        execution_role = detect_object(iam_client.list_roles, :roles, :role_name, AWS_AUTH_ECS_EXECUTION_ROLE_NAME)
+        if execution_role.nil?
+          raise 'Execution role not configured'
+        end
+        task_role = detect_object(iam_client.list_roles, :roles, :role_name, AWS_AUTH_ECS_TASK_ROLE_NAME)
+        if task_role.nil?
+          raise 'Task role not configured'
+        end
+
+        task_definition = ecs_client.register_task_definition(
+          family: AWS_AUTH_ECS_TASK_FAMILY,
+          container_definitions: [{
+            name: 'ssh',
+            essential: true,
+            entry_point: entry_point,
+            image: 'debian:9',
+            port_mappings: [{
+              container_port: 22,
+              protocol: 'tcp',
+            }],
+            log_configuration: {
+              log_driver: 'awslogs',
+              options: {
+                'awslogs-group' => AWS_AUTH_ECS_LOG_GROUP,
+                'awslogs-region' => region,
+                'awslogs-stream-prefix' => AWS_AUTH_ECS_LOG_STREAM_PREFIX,
+              },
+            },
+          }],
+          requires_compatibilities: [launch_type],
+          network_mode: 'awsvpc',
+          cpu: '512',
+          memory: '2048',
+          # This is the ECS task role used for AWS auth testing
+          task_role_arn: task_role.arn,
+          # The execution role is required to support awslogs (logging to
+          # CloudWatch).
+          execution_role_arn: execution_role.arn,
+        ).task_definition
+        task_definition_ref = AWS_AUTH_ECS_TASK_FAMILY
+      end
+
+      service = ecs_client.describe_services(
+        cluster: cluster_name,
+        services: [service_name],
+      ).services.first
+
+      if service && service.status.downcase == 'draining'
+        puts "Waiting for #{service_name} to drain"
+        ecs_client.wait_until(
+          :services_inactive, {
+            cluster: cluster.cluster_name,
+            services: [service_name],
+          },
+          delay: 5,
+          max_attempts: 36,
+        )
+        puts "... done."
+        service = nil
+      end
+      if service && service.status.downcase == 'inactive'
+        service = nil
+      end
+      if service
+        puts "Updating service with status #{service.status}"
+        service = ecs_client.update_service(
+          cluster: cluster_name,
+          service: service_name,
+          task_definition: task_definition_ref,
+        ).service
+      else
+        puts "Creating a new service"
+        vpc_config = {}
+        unless options[:ec2]
+          vpc_config[:assign_public_ip] = 'ENABLED'
+        end
+        service = ecs_client.create_service(
+          desired_count: 1,
+          service_name: service_name,
+          task_definition: task_definition_ref,
+          cluster: cluster_name,
+          launch_type: launch_type,
+          network_configuration: {
+            awsvpc_configuration: vpc_config.merge(
+              subnets: subnet_ids,
+              security_groups: [security_group_id],
+            ),
+          },
+        ).service
+      end
+    end
+
+    def wait_for_ecs_ready(
+      cluster_name: AWS_AUTH_ECS_CLUSTER_NAME,
+      service_name: AWS_AUTH_ECS_SERVICE_NAME,
+      timeout: 20
+    )
+      deadline = Time.now + timeout
+
+      # The AWS SDK waiter seems to immediately fail sometimes right after
+      # the service is created, so wait for the service to become active
+      # manually and then use the waiter to wait for the service to become
+      # stable.
+      #
+      # The failure may be due to the fact that apparently, it is possible for
+      # describe_services to not return an existing service for some time.
+      # Therefore, allow the lack of service to be a transient error.
+      loop do
+        service = ecs_client.describe_services(
+          cluster: cluster_name,
+          services: [service_name],
+        ).services.first
+
+        if service.nil?
+          puts "Service #{service_name} in cluster #{cluster_name} does not exist (yet?)"
+          status = 'MISSING'
+        elsif service.status.downcase == 'active'
+          break
+        else
+          status = service.status
+        end
+
+        now = Time.now
+        if now >= deadline
+          raise "Service #{service_name} in cluster #{cluster_name} did not become ready in #{timeout} seconds (current status: #{status})"
+        end
+
+        puts "Wating for service #{service_name} in cluster #{cluster_name} to become ready (#{'%2.1f' % (deadline - now)} seconds remaining, current status: #{status})"
+        sleep 5
+      end
+
+      puts "Wating for service #{service_name} in cluster #{cluster_name} to become stable"
+      ecs_client.wait_until(
+        :services_stable, {
+          cluster: cluster_name,
+          services: [service_name],
+        },
+        delay: 5,
+        max_attempts: 36,
+      )
+    end
+
+    def terminate_auth_ecs_task
+      ecs_client.describe_services(
+        cluster: AWS_AUTH_ECS_CLUSTER_NAME,
+        services: [AWS_AUTH_ECS_SERVICE_NAME],
+      ).each do |resp|
+        resp.services.each do |service|
+          puts "Terminating #{service.service_name}"
+          begin
+            ecs_client.update_service(
+              cluster: AWS_AUTH_ECS_CLUSTER_NAME,
+              service: service.service_name,
+              desired_count: 0,
+            )
+          rescue Aws::ECS::Errors::ServiceNotActiveException
+            # No action needed
+          end
+          ecs_client.delete_service(
+            cluster: AWS_AUTH_ECS_CLUSTER_NAME,
+            service: service.service_name,
+          )
+        end
+      end
+    end
+
+    private
+
+    def find_running_instance(reservations)
+      instance = nil
+      reservations.each do |reservation|
+        instance = reservation.instances.detect do |instance|
+          %w(pending running stopped).include?(instance.state.name)
+        end
+        break if instance
+      end
+      instance
+    end
+  end
+end