simplygenius-atmos 0.7.0

data/lib/atmos/provider_factory.rb

@@ -0,0 +1,19 @@
+require_relative '../atmos'
+
+module Atmos
+  class ProviderFactory
+    include GemLogger::LoggerSupport
+
+    def self.get(name)
+      @provider ||= begin
+        logger.debug("Loading provider: #{name}")
+        require "atmos/providers/#{name}/provider"
+        provider = "Atmos::Providers::#{name.camelize}::Provider".constantize
+        logger.debug("Loaded provider #{provider}")
+        provider.new(name)
+      end
+      return @provider
+    end
+
+  end
+end

data/lib/atmos/providers/aws/account_manager.rb

@@ -0,0 +1,82 @@
+require_relative '../../../atmos'
+require 'aws-sdk-organizations'
+
+module Atmos
+  module Providers
+    module Aws
+
+      class AccountManager
+        include GemLogger::LoggerSupport
+        include FileUtils
+
+        def initialize(provider)
+          @provider = provider
+        end
+
+        def create_account(env, name: nil, email: nil)
+          result = {}
+          org = ::Aws::Organizations::Client.new
+          resp = nil
+          name ||= "Atmos #{env} account"
+
+          begin
+            logger.info "Looking up organization"
+            resp = org.describe_organization()
+            logger.debug "Described organization: #{resp.to_h}"
+          rescue ::Aws::Organizations::Errors::AWSOrganizationsNotInUseException
+            logger.info "Organization doesn't exist, creating"
+            resp = org.create_organization()
+            logger.debug "Created organization: #{resp.to_h}"
+          end
+
+          if email.blank?
+            master_email = resp.organization.master_account_email
+            email = master_email.sub('@', "+#{env}@")
+          end
+          result[:email] = email
+          result[:name] = name
+
+
+          begin
+            logger.info "Creating account named #{name}"
+            resp = org.create_account(account_name: name, email: email)
+          rescue ::Aws::Organizations::Errors::FinalizingOrganizationException
+            logger.info "Waiting to retry account creation as the organization needs to finalize"
+            logger.info "This will eventually succeed after receiving a"
+            logger.info "'Consolidated Billing verification' email from AWS"
+            logger.info "You can leave this running or cancel and restart later."
+            sleep 60
+            retry
+          end
+
+          logger.debug "Created account: #{resp.to_h}"
+
+          status_id = resp.create_account_status.id
+          status = resp.create_account_status.state
+          account_id = resp.create_account_status.account_id
+
+          while status =~ /in_progress/i
+            logger.info "Waiting for account creation to complete, status: #{status}"
+            resp = org.describe_create_account_status(create_account_request_id: status_id)
+            logger.debug("Account creation status check: #{resp.to_h}")
+            status = resp.create_account_status.state
+            account_id = resp.create_account_status.account_id
+            sleep 5
+          end
+
+          if status =~ /failed/i
+            logger.error "Failed to create account: #{resp.create_account_status.failure_reason}"
+            exit(1)
+          end
+
+          result[:account_id] = account_id
+
+          return result
+        end
+
+      end
+
+    end
+  end
+end
+

data/lib/atmos/providers/aws/auth_manager.rb

@@ -0,0 +1,208 @@
+require_relative '../../../atmos'
+require_relative '../../../atmos/utils'
+require_relative '../../../atmos/ui'
+require_relative '../../../atmos/otp'
+require 'aws-sdk-core'
+require 'aws-sdk-iam'
+require 'json'
+
+module Atmos
+  module Providers
+    module Aws
+
+      class AuthManager
+        include GemLogger::LoggerSupport
+        include FileUtils
+        include Atmos::UI
+
+        def initialize(provider)
+          @provider = provider
+        end
+
+        def authenticate(system_env, **opts, &block)
+
+          profile = system_env['AWS_PROFILE']
+          key = system_env['AWS_ACCESS_KEY_ID']
+          secret = system_env['AWS_SECRET_ACCESS_KEY']
+          if profile.blank? && (key.blank? || secret.blank?)
+            logger.warn("An aws profile or key/secret should be supplied via the environment")
+          end
+
+          # Handle bootstrapping a new env account.  Newly created organization
+          # accounts only have the default role that can only be assumed by an
+          # iam user, so use that as the target for assume_role, and the root
+          # check below will ensure iam user
+          assume_role_name = nil
+          if opts[:bootstrap] && Atmos.config.atmos_env != 'ops'
+            # TODO: do this hack better
+            assume_role_name = Atmos.config["auth.bootstrap_assume_role_name"]
+          else
+            assume_role_name = opts[:role] || Atmos.config["auth.assume_role_name"]
+          end
+          account_id = Atmos.config.account_hash[Atmos.config.atmos_env].to_s
+          role_arn = "arn:aws:iam::#{account_id}:role/#{assume_role_name}"
+
+          user_name = nil
+          begin
+            sts = ::Aws::STS::Client.new
+            resp = sts.get_caller_identity
+            arn_pieces = resp.arn.split(":")
+            user_name = arn_pieces.last.split("/").last
+
+            # root credentials can't assume role, but they should have full
+            # access for the current account, so proceed (e.g. for bootstrap).
+            if arn_pieces.last == "root"
+
+              # We check the account of the caller to prevent root user of ops
+              # account from bootstrapping an env account, but still allow a
+              # root user of the env account itself to be able to bootstrap
+              # (i.e. to allow not organizational accounts to bootstrap using
+              # their root user)
+              if arn_pieces[-2] != account_id
+                logger.error <<~EOF
+                  Account doesn't match credentials.  Bootstrapping a new
+                  account should be done as an iam user from the ops account or
+                  using credentials for a root user of the env account.
+                EOF
+                exit(1)
+              end
+
+              # Should only use root credentials for  bootstrap, and thus we
+              # won't have role requirement for mfa, etc, even if root account
+              # uses mfa for login.  Thus skip all the other stuff, to
+              # encourage/force use of non-root accounts for normal use
+              logger.warn("Using aws root credentials - should only be neccessary for bootstrap")
+              return block.call(Hash[system_env])
+            end
+
+          rescue ::Aws::STS::Errors::ServiceError => e
+            logger.error "Could not discover aws credentials"
+            exit(1)
+          end
+
+          auth_needed = true
+          cache_key = "#{user_name}-#{assume_role_name}"
+          credentials = read_auth_cache[cache_key]
+
+          if credentials.present?
+            logger.debug("Session cache present, checking expiration...")
+            expiration = Time.parse(credentials['expiration'])
+            session_renew_interval = (session_duration / 4).to_i
+
+            if Time.now > expiration
+              logger.debug "Session cache is expired, performing normal auth"
+              auth_needed = true
+            elsif Time.now > (expiration - session_renew_interval)
+              begin
+                logger.info "Session approaching expiration, renewing..."
+                credentials = assume_role(role_arn, credentials: credentials)
+                auth_needed = false
+              rescue => e
+                logger.info "Failed to renew credentials using session cache, reason: #{e.message}"
+                auth_needed = true
+              end
+            else
+              logger.debug "Session cache is current, skipping auth"
+              auth_needed = false
+            end
+          end
+
+          if auth_needed
+            begin
+              logger.info "No active session cache, authenticating..."
+
+              credentials = assume_role(role_arn)
+              write_auth_cache(cache_key => credentials)
+
+            rescue ::Aws::STS::Errors::AccessDenied => e
+              if e.message !~ /explicit deny/
+                logger.debug "Access Denied, reason: #{e.message}"
+              end
+
+              logger.info "Normal auth failed, checking for mfa"
+
+              iam = ::Aws::IAM::Client.new
+              response = iam.list_mfa_devices(user_name: user_name)
+              mfa_serial = response.mfa_devices.first.try(:serial_number)
+              token = nil
+              if mfa_serial.present?
+
+                token = Atmos::Otp.instance.generate(user_name)
+                if token.nil?
+                  token = ask("Enter token to retry with mfa: ")
+                else
+                  logger.info "Used integrated atmos mfa to generate token"
+                end
+
+                if token.blank?
+                  logger.error "A MFA token must be supplied"
+                  exit(1)
+                end
+
+              else
+                logger.error "MFA is not setup for your account, retry after doing so"
+                exit(1)
+              end
+
+              credentials = assume_role(role_arn, serial_number: mfa_serial, token_code: token)
+              write_auth_cache(cache_key => credentials)
+
+            end
+          end
+
+          process_env = {}
+          process_env['AWS_ACCESS_KEY_ID'] = credentials['access_key_id']
+          process_env['AWS_SECRET_ACCESS_KEY'] = credentials['secret_access_key']
+          process_env['AWS_SESSION_TOKEN'] = credentials['session_token']
+          logger.debug("Calling authentication target with env: #{process_env.inspect}")
+          block.call(Hash[system_env].merge(process_env))
+        end
+
+        private
+
+        def session_duration
+          @session_duration ||= (Atmos.config["auth.session_duration"] || 3600).to_i
+        end
+
+        def assume_role(role_arn, **opts)
+          # use Aws::AssumeRoleCredentials ?
+          if opts[:credentials]
+            c = opts.delete(:credentials)
+            creds = ::Aws::Credentials.new(
+                c[:access_key_id], c[:secret_access_key], c[:session_token]
+            )
+            client_opts = {credentials: creds}
+          else
+            client_opts = {}
+          end
+          sts = ::Aws::STS::Client.new(client_opts)
+          params = {
+              duration_seconds: session_duration,
+              role_session_name: "Atmos",
+              role_arn: role_arn
+          }.merge(opts)
+          logger.debug("Assuming role: #{params}")
+          resp = sts.assume_role(params)
+          return Atmos::Utils::SymbolizedMash.new(resp.credentials.to_h)
+        end
+
+        def auth_cache_file
+          File.join(Atmos.config.auth_cache_dir, 'aws-assume-role.json')
+        end
+
+        def write_auth_cache(h)
+          File.open(auth_cache_file, 'w') do |f|
+            f.puts(JSON.pretty_generate(h))
+          end
+        end
+
+        def read_auth_cache
+          data = JSON.parse(File.read(auth_cache_file)) rescue {}
+          Atmos::Utils::SymbolizedMash.new(data)
+        end
+
+      end
+
+    end
+  end
+end

data/lib/atmos/providers/aws/container_manager.rb

@@ -0,0 +1,116 @@
+require_relative '../../../atmos'
+require 'aws-sdk-ecs'
+require 'aws-sdk-ecr'
+require 'open3'
+
+module Atmos
+  module Providers
+    module Aws
+
+      class ContainerManager
+        include GemLogger::LoggerSupport
+
+        def initialize(provider)
+          @provider = provider
+        end
+
+        def push(ecs_name, local_image,
+                 ecr_repo: ecs_name, revision: nil)
+
+          revision = Time.now.strftime('%Y%m%d%H%M%S') unless revision.present?
+          result = {}
+
+          ecr = ::Aws::ECR::Client.new
+          resp = nil
+
+          resp = ecr.get_authorization_token
+          auth_data = resp.authorization_data.first
+          token = auth_data.authorization_token
+          endpoint = auth_data.proxy_endpoint
+          user, password = Base64.decode64(token).split(':')
+
+          # docker login into the ECR repo for the current account so that we can pull/push to it
+          run("docker", "login", "-u", user, "-p", password, endpoint)#, stdin_data: token)
+
+          image="#{ecs_name}:latest"
+          ecs_image="#{endpoint.sub(/https?:\/\//, '')}/#{ecr_repo}"
+
+          tags = ['latest', revision]
+          logger.info "Tagging local image '#{local_image}' with #{tags}"
+          tags.each {|t| run("docker", "tag", local_image, "#{ecs_image}:#{t}") }
+
+          logger.info "Pushing tagged image to ECR repo"
+          tags.each {|t| run("docker", "push", "#{ecs_image}:#{t}") }
+
+          result[:remote_image] = "#{ecs_image}:#{revision}"
+          return result
+        end
+
+        def deploy_task(task, remote_image)
+          result = {}
+
+          ecs = ::Aws::ECS::Client.new
+          resp = nil
+
+          resp = ecs.list_task_definitions(family_prefix: task, sort: 'DESC')
+          latest_defn_arn = resp.task_definition_arns.first
+
+          logger.info "Latest task definition: #{latest_defn_arn}"
+
+          resp = ecs.describe_task_definition(task_definition: latest_defn_arn)
+          latest_defn = resp.task_definition
+
+          new_defn = latest_defn.to_h
+          [:revision, :status, :task_definition_arn,
+           :requires_attributes, :compatibilities].each do |attr|
+            new_defn.delete(attr)
+          end
+          new_defn[:container_definitions].each {|c| c[:image] = remote_image}
+
+          resp = ecs.register_task_definition(**new_defn)
+          result[:task_definition] = resp.task_definition.task_definition_arn
+
+          logger.info "Updated task=#{task} to #{result[:task_definition]} with image #{remote_image}"
+
+          return result
+        end
+
+        def deploy_service(cluster, service, remote_image)
+           result = {}
+
+           ecs = ::Aws::ECS::Client.new
+           resp = nil
+
+           # Get current task definition name from service
+           resp = ecs.describe_services(cluster: cluster, services: [service])
+           current_defn_arn = resp.services.first.task_definition
+           defn_name = current_defn_arn.split("/").last.split(":").first
+
+           logger.info "Current task definition (name=#{defn_name}): #{current_defn_arn}"
+           result = deploy_task(defn_name, remote_image)
+           new_taskdef = result[:task_definition]
+
+           logger.info "Updating service with new task definition: #{new_taskdef}"
+
+           resp = ecs.update_service(cluster: cluster, service: service, task_definition: new_taskdef)
+
+           logger.info "Updated service=#{service} on cluster=#{cluster} to #{new_taskdef} with image #{remote_image}"
+
+           return result
+        end
+
+        private
+
+        def run(*args, **opts)
+          logger.debug("Running: #{args}")
+          stdout, status = Open3.capture2e(ENV, *args, **opts)
+          logger.debug(stdout)
+          raise "Failed to run #{args}: #{stdout}" unless status.success?
+          return stdout
+        end
+
+      end
+
+    end
+  end
+end

data/lib/atmos/providers/aws/provider.rb

@@ -0,0 +1,51 @@
+require_relative '../../../atmos'
+
+Dir.glob(File.join(__dir__, '*.rb')) do |f|
+  require_relative "#{File.basename(f).sub(/\.rb$/, "")}"
+end
+
+module Atmos
+  module Providers
+    module Aws
+
+      class Provider
+        include GemLogger::LoggerSupport
+
+        def initialize(name)
+          @name = name
+        end
+
+        def auth_manager
+          @auth_manager ||= begin
+            Atmos::Providers::Aws::AuthManager.new(self)
+          end
+        end
+
+        def user_manager
+          @user_manager ||= begin
+            Atmos::Providers::Aws::UserManager.new(self)
+          end
+        end
+
+        def account_manager
+          @account_manager ||= begin
+            Atmos::Providers::Aws::AccountManager.new(self)
+          end
+        end
+
+        def secret_manager
+          @secret_manager ||= begin
+            Atmos::Providers::Aws::S3SecretManager.new(self)
+          end
+        end
+
+        def container_manager
+          @container_manager ||= begin
+            Atmos::Providers::Aws::ContainerManager.new(self)
+          end
+        end
+      end
+
+    end
+  end
+end