cloud-mu 3.3.2 → 3.4.0

data/modules/mu/providers/aws.rb

@@ -550,7 +550,9 @@ end
       def self.credToAcct(name = nil)
         creds = credConfig(name)
 
-        return creds['account_number'] if creds['account_number']
+        if creds['account_number'] and !creds['account_number'].empty?
+          return creds['account_number']
+        end
 
         acct_num = MU::Cloud::AWS.iam(credentials: name).list_users.users.first.arn.split(/:/)[4]
         acct_num.to_s
@@ -672,8 +674,8 @@ end
                 next
               end
               acct_num = MU::Cloud::AWS.iam(credentials: acctname).list_users.users.first.arn.split(/:/)[4]
-              if acct_num.to_s ==  name.to_s
-                cfg['account_number'] = acct_num.to_s
+              cfg['account_number'] ||= acct_num.to_s
+              if acct_num.to_s == name.to_s
                 @@acct_to_profile_map[name.to_s] = cfg
                 return name_only ? name.to_s : cfg
               end
@@ -1516,6 +1518,7 @@ end
         def initialize(region: nil, api: "EC2", credentials: nil)
           @cred_obj = MU::Cloud::AWS.loadCredentials(credentials)
           @credentials = MU::Cloud::AWS.credConfig(credentials, name_only: true)
+          @api_name = api
 
           if !@cred_obj
             raise MuError, "Unable to locate valid AWS credentials for #{api} API. #{credentials ? "Credentials requested were '#{credentials}'": ""}"
@@ -1533,6 +1536,8 @@ end
           params[:credentials] = @cred_obj
 
           MU.log "Initializing #{api} object with credentials #{credentials}", MU::DEBUG, details: params
+          require "aws-sdk-#{api.downcase}"
+
           @api = Object.const_get("Aws::#{api}::Client").new(params)
         end
 
@@ -1541,27 +1546,31 @@ end
         # rescues for known silly endpoint behavior.
         def method_missing(method_sym, *arguments)
           # make sure error symbols are loaded for our exception handling later
-          require "aws-sdk-core"
-          require "aws-sdk-core/rds"
-          require "aws-sdk-core/ec2"
-          require "aws-sdk-core/route53"
-          require "aws-sdk-core/iam"
-          require "aws-sdk-core/efs"
-          require "aws-sdk-core/pricing"
-          require "aws-sdk-core/apigateway"
-          require "aws-sdk-core/ecs"
-          require "aws-sdk-core/eks"
-          require "aws-sdk-core/cloudwatchlogs"
-          require "aws-sdk-core/cloudwatchevents"
-          require "aws-sdk-core/elasticloadbalancing"
-          require "aws-sdk-core/elasticloadbalancingv2"
-          require "aws-sdk-core/autoscaling"
-          require "aws-sdk-core/client_waiters"
-          require "aws-sdk-core/waiters/errors"
+          require "aws-sdk-lambda"
+          require "aws-sdk-rds"
+          require "aws-sdk-ec2"
+          require "aws-sdk-route53"
+          require "aws-sdk-iam"
+          require "aws-sdk-efs"
+          require "aws-sdk-pricing"
+          require "aws-sdk-apigateway"
+          require "aws-sdk-ecs"
+          require "aws-sdk-eks"
+          require "aws-sdk-cloudwatchlogs"
+          require "aws-sdk-cloudwatchevents"
+          require "aws-sdk-elasticloadbalancing"
+          require "aws-sdk-elasticloadbalancingv2"
+          require "aws-sdk-autoscaling"
+
+          known_concats = {
+            "Pricing" => {
+              :get_products => :price_list
+            }
+          }
 
           retries = 0
           begin
-            MU.log "Calling #{method_sym} in #{@region}", MU::DEBUG, details: arguments
+            MU.log "Calling #{@api_name}.#{method_sym} in #{@region}", MU::DEBUG, details: arguments
 
               retval = if !arguments.nil? and arguments.size == 1
                 @api.method(method_sym).call(arguments[0])
@@ -1590,11 +1599,22 @@ end
 
               if paginator and new_page and !new_page.empty?
                 resp = retval.respond_to?(:__getobj__) ? retval.__getobj__ : retval
-                concat_to = resp.class.instance_methods(false).reject { |m|
+                concat_to = MU.structToHash(resp).keys.reject { |m|
                   m.to_s.match(/=$/) or m == paginator or resp.send(m).nil? or !resp.send(m).is_a?(Array)
                 }
+
+                if concat_to.empty? and known_concats[@api_name] and
+                   known_concats[@api_name][method_sym]
+                  concat_to << known_concats[@api_name][method_sym]
+                end
+
+                if concat_to.empty? and method_sym.to_s.match(/^(?:describe|list)_(.*)/)
+                  my_attr = Regexp.last_match[1].to_sym
+                  concat_to << my_attr if resp.respond_to?(my_attr)
+                end
+
                 if concat_to.size != 1
-                  MU.log "Tried to figure out where I might append paginated results for a #{resp.class.name}, but failed", MU::DEBUG, details: concat_to
+                  raise MuError.new "Tried to figure out where I might append paginated results for a #{@api_name}.#{method_sym}, but failed", details: MU.structToHash(resp).keys
                 else
                   concat_to = concat_to.first
                   new_args = arguments ? arguments.dup : [{}]

data/modules/mu/providers/aws/database.rb

@@ -797,14 +797,17 @@ dependencies
         # @return [void]
         def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, region: MU.curRegion, flags: {})
 
+          threads = []
+
           ["instance", "cluster"].each { |type|
-            threaded_resource_purge("describe_db_#{type}s".to_sym, "db_#{type}s".to_sym, "db_#{type}_identifier".to_sym, (type == "instance" ? "db" : "cluster"), region, credentials, ignoremaster, known: flags['known'], deploy_id: deploy_id) { |id|
+            threads.concat threaded_resource_purge("describe_db_#{type}s".to_sym, "db_#{type}s".to_sym, "db_#{type}_identifier".to_sym, (type == "instance" ? "db" : "cluster"), region, credentials, ignoremaster, known: flags['known'], deploy_id: deploy_id) { |id|
               terminate_rds_instance(nil, noop: noop, skipsnapshots: flags["skipsnapshots"], region: region, deploy_id: deploy_id, cloud_id: id, mu_name: id.upcase, credentials: credentials, cluster: (type == "cluster"), known: flags['known'])
   
-            }.each { |t|
-              t.join
             }
           }
+          threads.each { |t|
+            t.join
+          }
 
           threads = threaded_resource_purge(:describe_db_subnet_groups, :db_subnet_groups, :db_subnet_group_name, "subgrp", region, credentials, ignoremaster, known: flags['known'], deploy_id: deploy_id) { |id|
             MU.log "Deleting RDS subnet group #{id}"
@@ -1653,7 +1656,7 @@ dependencies
 
           raise MuError, "terminate_rds_instance requires a non-nil database descriptor (#{cloud_id})" if db.nil? or cloud_id.nil?
 
-          MU.retrier([], wait: 60, loop_if: Proc.new { %w{creating modifying backing-up}.include?(cluster ? db.status : db.db_instance_status) }) {
+          MU.retrier([], wait: 60, loop_if: Proc.new { %w{creating modifying backing-up}.include?(cluster ? db.status : db.db_instance_status) }, loop_msg: "Waiting for RDS #{cluster ? "cluster" : "instance"} #{cloud_id} to be in a valid state for deletion") {
             db = MU::Cloud::AWS::Database.find(cloud_id: cloud_id, region: region, credentials: credentials, cluster: cluster).values.first
             return if db.nil?
           }
@@ -1673,7 +1676,7 @@ dependencies
               params[:skip_final_snapshot] = false
               params[:final_db_snapshot_identifier] = "#{cloud_id}-mufinal"
             end
-
+sleep 30
             if !noop
               on_retry = Proc.new { |e|
                 if [Aws::RDS::Errors::DBSnapshotAlreadyExists, Aws::RDS::Errors::DBClusterSnapshotAlreadyExistsFault, Aws::RDS::Errors::DBClusterQuotaExceeded].include?(e.class)
@@ -1688,7 +1691,7 @@ dependencies
                 end
               }
               del_db = nil
-              MU.retrier([], wait: 10, ignoreme: [Aws::RDS::Errors::DBInstanceNotFound], loop_if: Proc.new { del_db and ((!cluster and del_db.db_instance_status != "deleted") or (cluster and del_db.status != "deleted")) }) {
+              MU.retrier([], wait: 10, ignoreme: [Aws::RDS::Errors::DBInstanceNotFound], loop_if: Proc.new { del_db and ((!cluster and del_db.db_instance_status != "deleted") or (cluster and del_db.status != "deleted")) }, loop_msg: "Waiting for RDS #{cluster ? "cluster" : "instance"} #{cloud_id} to delete") {
                 del_db = MU::Cloud::AWS::Database.find(cloud_id: cloud_id, region: region, cluster: cluster).values.first
               }
             end

data/modules/mu/providers/aws/function.rb

@@ -58,16 +58,19 @@ module MU
           lambda_properties = get_properties
 
           MU.retrier([Aws::Lambda::Errors::InvalidParameterValueException], max: 5, wait: 10) {
-            resp = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).create_function(lambda_properties)
+            resp = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @credentials).create_function(lambda_properties)
             @cloud_id = resp.function_name
           }
 
           # the console does this and docs expect it to be there, so mimic the
           # behavior
-          MU::Cloud::AWS.cloudwatchlogs(region: @config["region"], credentials: @credentials).create_log_group(
-            log_group_name: "/aws/lambda/#{@cloud_id}",
-            tags: @tags
-          )
+          begin
+            MU::Cloud::AWS.cloudwatchlogs(region: @config["region"], credentials: @credentials).create_log_group(
+              log_group_name: "/aws/lambda/#{@cloud_id}",
+              tags: @tags
+            )
+          rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
+          end
         end
 
         # Called automatically by {MU::Deploy#createResources}

data/modules/mu/providers/aws/job.rb

@@ -57,33 +57,36 @@ module MU
             target_params = []
             @config['targets'].each { |t|
               MU.retrier([MuNonFatal], max:5, wait: 9) {
-              target_ref = MU::Config::Ref.get(t)
-              target_obj = target_ref.kitten(cloud: "AWS")
-              this_target = if target_ref.is_mu_type? and target_obj and
-                               !target_obj.arn.nil?
-                {
-                  id: target_obj.cloud_id,
-                  arn: target_obj.arn
-                }
-              elsif target_ref.id and target_ref.id.match(/^arn:/)
-                {
-                  id: target_ref.id || target_ref.name,
-                  arn: target_ref.id
-                }
-              else
-                raise MuNonFatal.new "Failed to retrieve ARN from CLoudWatch Event target descriptor", details: target_ref.to_h
-              end
-              if t['role']
-                role_obj = MU::Config::Ref.get(t['role']).kitten(@deploy, cloud: "AWS")
-                  raise MuError.new "Failed to fetch object from role reference", details: t['role'].to_h if !role_obj
-                  params[:role_arn] = role_obj.arn
-              end
-              [:input, :input_path, :input_transformer, :kinesis_parameters, :run_command_parameters, :batch_parameters, :sqs_parameters, :ecs_parameters].each { |attr|
-                if t[attr.to_s]
-                  this_target[attr] = MU.structToHash(t[attr.to_s])
+                target_ref = MU::Config::Ref.get(t)
+                target_obj = target_ref.kitten(@deploy, cloud: "AWS")
+                this_target = if target_ref.is_mu_type? and target_obj and
+                                 !target_obj.arn.nil?
+                  if target_ref.type == "functions"
+                    target_obj.addTrigger(arn, "events", @mu_name)
+                  end
+                  {
+                    id: target_obj.cloud_id,
+                    arn: target_obj.arn
+                  }
+                elsif target_ref.id and target_ref.id.match(/^arn:/)
+                  {
+                    id: target_ref.id || target_ref.name,
+                    arn: target_ref.id
+                  }
+                else
+                  raise MuNonFatal.new "Failed to retrieve ARN from CLoudWatch Event target descriptor", details: target_ref.to_h
                 end
-              }
-              target_params << this_target
+                if t['role']
+                  role_obj = MU::Config::Ref.get(t['role']).kitten(@deploy, cloud: "AWS")
+                    raise MuError.new "Failed to fetch object from role reference", details: t['role'].to_h if !role_obj
+                    params[:role_arn] = role_obj.arn
+                end
+                [:input, :input_path, :input_transformer, :kinesis_parameters, :run_command_parameters, :batch_parameters, :sqs_parameters, :ecs_parameters].each { |attr|
+                  if t[attr.to_s]
+                    this_target[attr] = MU.structToHash(t[attr.to_s])
+                  end
+                }
+                target_params << this_target
               }
             }
             MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_targets(

data/modules/mu/providers/aws/role.rb

@@ -43,7 +43,7 @@ module MU
 
               policy_name = @mu_name+"-"+policy.keys.first.upcase
               MU.log "Creating IAM policy #{policy_name}"
-              MU::Cloud::AWS.iam(credentials: @config['credentials']).create_policy(
+              MU::Cloud::AWS.iam(credentials: @credentials).create_policy(
                 policy_name: policy_name,
                 path: "/"+@deploy.deploy_id+"/",
                 policy_document: JSON.generate(policy.values.first),
@@ -53,16 +53,18 @@ module MU
           end
 
           if !@config['bare_policies']
-            MU.log "Creating IAM role #{@mu_name}"
             @cloud_id = @mu_name
             path = @config['strip_path'] ? nil : "/"+@deploy.deploy_id+"/"
-            MU::Cloud::AWS.iam(credentials: @config['credentials']).create_role(
-              path: path,
-              role_name: @mu_name,
-              description: "Generated by Mu",
-              assume_role_policy_document: gen_assume_role_policy_doc,
-              tags: get_tag_params
-            )
+            params = {
+              :path => path,
+              :role_name => @mu_name,
+              :description => "Generated by Mu",
+              :assume_role_policy_document => gen_assume_role_policy_doc,
+              :tags => get_tag_params
+            }
+
+            MU.log "Creating IAM role #{@mu_name} (#{@credentials})", details: params
+            MU::Cloud::AWS.iam(credentials: @credentials).create_role(params)
           end
         end
 
@@ -75,7 +77,7 @@ module MU
           end
 
           if !@config['bare_policies']
-            resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_role(
+            resp = MU::Cloud::AWS.iam(credentials: @credentials).get_role(
               role_name: @mu_name
             ).role
             ext_tags = resp.tags.map { |t| t.to_h }
@@ -84,7 +86,7 @@ module MU
 
             if tag_param.size > 0
               MU.log "Updating tags on IAM role #{@mu_name}", MU::NOTICE, details: tag_param
-              MU::Cloud::AWS.iam(credentials: @config['credentials']).tag_role(role_name: @mu_name, tags: tag_param)
+              MU::Cloud::AWS.iam(credentials: @credentials).tag_role(role_name: @mu_name, tags: tag_param)
             end
           end
 
@@ -114,13 +116,13 @@ module MU
 
             # Purge anything that doesn't belong
             if !@config['bare_policies']
-              attached_policies = MU::Cloud::AWS.iam(credentials: @config['credentials']).list_attached_role_policies(
+              attached_policies = MU::Cloud::AWS.iam(credentials: @credentials).list_attached_role_policies(
                 role_name: @mu_name
               ).attached_policies
               attached_policies.each { |a|
                 if !configured_policies.include?(a.policy_name)
                   MU.log "Removing IAM policy #{a.policy_name} from role #{@mu_name}", MU::NOTICE, details: configured_policies
-                  MU::Cloud::AWS::Role.purgePolicy(a.policy_arn, @config['credentials'])
+                  MU::Cloud::AWS::Role.purgePolicy(a.policy_arn, @credentials)
                 end
               }
             end
@@ -155,7 +157,6 @@ module MU
             }
 
             policy_name = basename+"-"+policy.keys.first.upcase
-
             arn = "arn:"+(MU::Cloud::AWS.isGovCloud? ? "aws-us-gov" : "aws")+":iam::"+MU::Cloud::AWS.credToAcct(credentials)+":policy#{path}/#{policy_name}"
             resp = begin
               desc = MU::Cloud::AWS.iam(credentials: credentials).get_policy(policy_arn: arn)
@@ -185,12 +186,17 @@ module MU
 
             rescue Aws::IAM::Errors::NoSuchEntity
               MU.log "Creating IAM policy #{policy_name}", details: policy.values.first
-              MU::Cloud::AWS.iam(credentials: credentials).create_policy(
+              desc = MU::Cloud::AWS.iam(credentials: credentials).create_policy(
                 policy_name: policy_name,
                 path: path+"/",
                 policy_document: JSON.generate(policy.values.first),
                 description: "Raw policy from #{basename}"
               )
+              MU.retrier([Aws::IAM::Errors::NoSuchEntity], loop_if: Proc.new { desc.nil? }) {
+                desc = MU::Cloud::AWS.iam(credentials: credentials).get_policy(policy_arn: arn)
+                pp desc
+              }
+              desc
             end
             arns << resp.policy.arn
           }
@@ -317,7 +323,7 @@ end
           my_policies.each { |p|
             if p.policy_name == policy
               seen_policy = true
-              old = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_policy_version(
+              old = MU::Cloud::AWS.iam(credentials: @credentials).get_policy_version(
                 policy_arn: p.arn,
                 version_id: p.default_version_id
               ).policy_version
@@ -576,7 +582,7 @@ end
         def toKitten(**_args)
           bok = {
             "cloud" => "AWS",
-            "credentials" => @config['credentials'],
+            "credentials" => @credentials,
             "cloud_id" => @cloud_id
           }
 
@@ -769,12 +775,12 @@ end
         def bindTo(entitytype, entityname)
           if entitytype == "instance_profile"
             begin
-              resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_instance_profile(
+              resp = MU::Cloud::AWS.iam(credentials: @credentials).get_instance_profile(
                 instance_profile_name: entityname
               ).instance_profile
 
               if !resp.roles.map { |r| r.role_name}.include?(@mu_name)
-                MU::Cloud::AWS.iam(credentials: @config['credentials']).add_role_to_instance_profile(
+                MU::Cloud::AWS.iam(credentials: @credentials).add_role_to_instance_profile(
                   instance_profile_name: entityname,
                   role_name: @mu_name
                 )
@@ -784,7 +790,7 @@ end
               raise e
             end
           elsif ["user", "group", "role"].include?(entitytype)
-            mypolicies = MU::Cloud::AWS.iam(credentials: @config['credentials']).list_policies(
+            mypolicies = MU::Cloud::AWS.iam(credentials: @credentials).list_policies(
               path_prefix: "/"+@deploy.deploy_id+"/"
             ).policies
             mypolicies.reject! { |p|
@@ -802,7 +808,7 @@ end
 
                 subpaths = ["service-role", "aws-service-role", "job-function"]
                 begin
-                  mypolicies << MU::Cloud::AWS.iam(credentials: @config['credentials']).get_policy(
+                  mypolicies << MU::Cloud::AWS.iam(credentials: @credentials).get_policy(
                     policy_arn: p_arn
                   ).policy
                 rescue Aws::IAM::Errors::NoSuchEntity => e
@@ -822,7 +828,7 @@ end
                 credentials: @credentials
               )
               raw_arns.each { |p_arn|
-                mypolicies << MU::Cloud::AWS.iam(credentials: @config['credentials']).get_policy(
+                mypolicies << MU::Cloud::AWS.iam(credentials: @credentials).get_policy(
                   policy_arn: p_arn
                 ).policy
               }
@@ -830,37 +836,37 @@ end
 
             mypolicies.each { |p|
               if entitytype == "user"
-                resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).list_attached_user_policies(
+                resp = MU::Cloud::AWS.iam(credentials: @credentials).list_attached_user_policies(
                   path_prefix: "/"+@deploy.deploy_id+"/",
                   user_name: entityname
                 )
                 if !resp or !resp.attached_policies.map { |a_p| a_p.policy_name }.include?(p.policy_name)
                   MU.log "Attaching IAM policy #{p.policy_name} to user #{entityname}", MU::NOTICE
-                  MU::Cloud::AWS.iam(credentials: @config['credentials']).attach_user_policy(
+                  MU::Cloud::AWS.iam(credentials: @credentials).attach_user_policy(
                     policy_arn: p.arn,
                     user_name: entityname
                   )
                 end
               elsif entitytype == "group"
-                resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).list_attached_group_policies(
+                resp = MU::Cloud::AWS.iam(credentials: @credentials).list_attached_group_policies(
                   path_prefix: "/"+@deploy.deploy_id+"/",
                   group_name: entityname
                 )
                 if !resp or !resp.attached_policies.map { |a_p| a_p.policy_name }.include?(p.policy_name)
                   MU.log "Attaching policy #{p.policy_name} to group #{entityname}", MU::NOTICE
-                  MU::Cloud::AWS.iam(credentials: @config['credentials']).attach_group_policy(
+                  MU::Cloud::AWS.iam(credentials: @credentials).attach_group_policy(
                     policy_arn: p.arn,
                     group_name: entityname
                   )
                 end
               elsif entitytype == "role"
-                resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).list_attached_role_policies(
+                resp = MU::Cloud::AWS.iam(credentials: @credentials).list_attached_role_policies(
                   role_name: entityname
                 )
 
                 if !resp or !resp.attached_policies.map { |a_p| a_p.policy_name }.include?(p.policy_name)
                   MU.log "Attaching policy #{p.policy_name} to role #{entityname}", MU::NOTICE
-                  MU::Cloud::AWS.iam(credentials: @config['credentials']).attach_role_policy(
+                  MU::Cloud::AWS.iam(credentials: @credentials).attach_role_policy(
                     policy_arn: p.arn,
                     role_name: entityname
                   )
@@ -881,19 +887,19 @@ end
           end
 
           resp = begin
-            MU.log "Creating instance profile #{@mu_name} #{@config['credentials']}"
-            MU::Cloud::AWS.iam(credentials: @config['credentials']).create_instance_profile(
+            MU.log "Creating instance profile #{@mu_name} #{@credentials}"
+            MU::Cloud::AWS.iam(credentials: @credentials).create_instance_profile(
               instance_profile_name: @mu_name
             )
           rescue Aws::IAM::Errors::EntityAlreadyExists
-            MU::Cloud::AWS.iam(credentials: @config['credentials']).get_instance_profile(
+            MU::Cloud::AWS.iam(credentials: @credentials).get_instance_profile(
               instance_profile_name: @mu_name
             )
           end
 
           # make sure it's really there before moving on
           begin
-            MU::Cloud::AWS.iam(credentials: @config['credentials']).get_instance_profile(instance_profile_name: @mu_name)
+            MU::Cloud::AWS.iam(credentials: @credentials).get_instance_profile(instance_profile_name: @mu_name)
           rescue Aws::IAM::Errors::NoSuchEntity => e
             MU.log e.inspect, MU::WARN
             sleep 10