cloud-mu 3.1.3 → 3.1.4

data/modules/mu/clouds/aws/habitat.rb

@@ -45,7 +45,6 @@ module MU
             resp = MU::Cloud::AWS.orgs(credentials: @config['credentials']).describe_create_account_status(
               create_account_request_id: createid
             )
-            createstatus = resp.create_account_status.state
             if !["SUCCEEDED", "IN_PROGRESS"].include?(resp.create_account_status.state)
               raise MuError, "Failed to create account #{@mu_name}: #{resp.create_account_status.failure_reason}"
             end
@@ -59,9 +58,12 @@ module MU
           MU.log "Creation of account #{@mu_name} (#{resp.create_account_status.account_id}) complete"
         end
 
+        @cloud_desc_cache = nil
         # Return the cloud descriptor for the Habitat
-        def cloud_desc
-          MU::Cloud::AWS::Habitat.find(cloud_id: @cloud_id).values.first
+        def cloud_desc(use_cache: true)
+          return @cloud_desc_cache if @cloud_desc_cache and use_cache
+          @cloud_desc_cache = MU::Cloud::AWS::Habitat.find(cloud_id: @cloud_id).values.first
+          @cloud_desc_cache
         end
 
         # Canonical Amazon Resource Number for this resource
@@ -87,10 +89,11 @@ module MU
         # Remove all AWS accounts associated with the currently loaded deployment. Try to, anyway.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
-        # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
           return if !orgMasterCreds?(credentials)
+          MU.log "AWS::Habitat.cleanup: need to support flags['known']", MU::DEBUG, details: flags
+          MU.log "Placeholder: AWS Habitat artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
 
           resp = MU::Cloud::AWS.orgs(credentials: credentials).list_accounts
 
@@ -108,14 +111,14 @@ module MU
 
         # Locate an existing account
         # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching account
-        def self.find(**args)
+        def self.find(**_args)
           {}
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
             "email" => {
@@ -126,9 +129,10 @@ module MU
           [toplevel_required, schema]
         end
 
-        # @param account_number [String]
+        # @param _account_number [String]
+        # @param _credentials [String]
         # @return [Boolean]
-        def self.isLive?(account_number, credentials = nil)
+        def self.isLive?(_account_number, _credentials = nil)
           true
         end
 
@@ -138,7 +142,6 @@ module MU
         # @param credentials [String]
         # @return [Boolean]
         def self.orgMasterCreds?(credentials = nil)
-          user_list = MU::Cloud::AWS.iam(credentials:  credentials).list_users.users
           acct_num = MU::Cloud::AWS.iam(credentials:  credentials).list_users.users.first.arn.split(/:/)[4]
 
           parentorg = MU::Cloud::AWS::Folder.find(credentials: credentials).values.first
@@ -147,9 +150,9 @@ module MU
 
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::habitats}, bare and unvalidated.
         # @param habitat [Hash]: The resource to process and validate
-        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+        # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
         # @return [Boolean]: True if validation succeeded, False otherwise
-        def self.validateConfig(habitat, configurator)
+        def self.validateConfig(habitat, _configurator)
           ok = true
 
           if !habitat["email"]

data/modules/mu/clouds/aws/loadbalancer.rb

@@ -288,7 +288,7 @@ module MU
           else
             @config["listeners"].each { |l|
               if l['ssl_certificate_id']
-                resp = MU::Cloud::AWS.elb(region: @config['region'], credentials: @config['credentials']).set_load_balancer_policies_of_listener(
+                MU::Cloud::AWS.elb(region: @config['region'], credentials: @config['credentials']).set_load_balancer_policies_of_listener(
                   load_balancer_name: @cloud_id, 
                   load_balancer_port: l['lb_port'], 
                   policy_names: [
@@ -330,7 +330,7 @@ module MU
                 }
               )
             else
-              @targetgroups.each_pair { |tg_name, tg|
+              @targetgroups.values.each { |tg|
                 MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).modify_target_group_attributes(
                   target_group_arn: tg.target_group_arn,
                   attributes: [
@@ -400,7 +400,7 @@ module MU
                 timeout = 0
                 MU.log "Disabling connection draining on #{lb.dns_name}"
               end
-              @targetgroups.each_pair { |tg_name, tg|
+              @targetgroups.values.each { |tg|
                 MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).modify_target_group_attributes(
                   target_group_arn: tg.target_group_arn,
                   attributes: [
@@ -473,7 +473,7 @@ module MU
                 end
               end
             else
-              @targetgroups.each_pair { |tg_name, tg|
+              @targetgroups.values.each { |tg|
                 MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).modify_target_group_attributes(
                   target_group_arn: tg.target_group_arn,
                   attributes: [
@@ -553,15 +553,17 @@ module MU
           end
         end
 
+        @cloud_desc_cache = nil
         # Wrapper for cloud_desc method that deals with elb vs. elb2 resources.
-        def cloud_desc
+        def cloud_desc(use_cache: true)
+          return @cloud_desc_cache if @cloud_desc_cache and use_cache
           if @config['classic']
-            resp = MU::Cloud::AWS.elb(region: @config['region'], credentials: @config['credentials']).describe_load_balancers(
+            @cloud_desc_cache = MU::Cloud::AWS.elb(region: @config['region'], credentials: @config['credentials']).describe_load_balancers(
               load_balancer_names: [@cloud_id]
             ).load_balancer_descriptions.first
-            return resp
+            return @cloud_desc_cache
           else
-            resp = MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_load_balancers(
+            @cloud_desc_cache = MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_load_balancers(
               names: [@cloud_id]
             ).load_balancers.first
             if @targetgroups.nil? and !@deploy.nil? and
@@ -573,7 +575,7 @@ module MU
               }
             end
 
-            return resp
+            return @cloud_desc_cache
           end
         end
 
@@ -692,7 +694,6 @@ module MU
               classic = false
             end
             begin
-              tags = []
               matched = false
               if flags and flags['vpc_id']
                 matched = true if lb.vpc_id == flags['vpc_id']
@@ -773,9 +774,9 @@ module MU
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
             "targetgroups" => {
@@ -818,9 +819,9 @@ module MU
 
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::loadbalancers}, bare and unvalidated.
         # @param lb [Hash]: The resource to process and validate
-        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+        # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
         # @return [Boolean]: True if validation succeeded, False otherwise
-        def self.validateConfig(lb, configurator)
+        def self.validateConfig(lb, _configurator)
           ok = true
 
           # XXX what about raw targetgroup ssl declarations?
@@ -830,7 +831,7 @@ module MU
               if lb['cloud'] != "CloudFormation" # XXX or maybe do this anyway?
                 begin
                   listener["ssl_certificate_id"] = MU::Cloud::AWS.findSSLCertificate(name: listener["ssl_certificate_name"].to_s, id: listener["ssl_certificate_id"].to_s, region: lb['region'])
-                rescue MuError => e
+                rescue MuError
                   ok = false
                   next
                 end

data/modules/mu/clouds/aws/log.rb

@@ -203,6 +203,9 @@ module MU
         # @param region [String]: The cloud provider region
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+          MU.log "AWS::Log.cleanup: need to support flags['known']", MU::DEBUG, details: flags
+          MU.log "Placeholder: AWS Log artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
+
           log_groups = self.find(credentials: credentials, region: region).values
           if !log_groups.empty?
             log_groups.each{ |lg|
@@ -227,13 +230,13 @@ module MU
             }
           end
 
-          unless noop
-            MU::Cloud::AWS.iam(credentials: credentials).list_roles.roles.each{ |role|
-              match_string = "#{MU.deploy_id}.*CloudTrail"
+#          unless noop
+#            MU::Cloud::AWS.iam(credentials: credentials).list_roles.roles.each{ |role|
+#              match_string = "#{MU.deploy_id}.*CloudTrail"
               # Maybe we should have a more generic way to delete IAM profiles and policies. The call itself should be moved from MU::Cloud::AWS::Server.
 #              MU::Cloud::AWS::Server.removeIAMProfile(role.role_name) if role.role_name.match(match_string)
-            }
-          end
+#            }
+#          end
         end
 
         # Locate an existing log group.
@@ -264,7 +267,7 @@ module MU
         # Reverse-map our cloud description into a runnable config hash.
         # We assume that any values we have in +@config+ are placeholders, and
         # calculate our own accordingly based on what's live in the cloud.
-        def toKitten(rootparent: nil, billing: nil, habitats: nil)
+        def toKitten(**_args)
           bok = {
             "cloud" => "AWS",
             "credentials" => @config['credentials'],
@@ -304,9 +307,9 @@ module MU
 
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
             "retention_period" => {
@@ -357,9 +360,9 @@ module MU
 
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::logs}, bare and unvalidated.
         # @param log [Hash]: The resource to process and validate
-        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+        # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
         # @return [Boolean]: True if validation succeeded, False otherwise
-        def self.validateConfig(log, configurator)
+        def self.validateConfig(log, _configurator)
           ok = true
 
           if log["filters"] && !log["filters"].empty?

data/modules/mu/clouds/aws/msg_queue.rb

@@ -53,14 +53,14 @@ module MU
           new_attrs = genQueueAttrs
 
           changed = false
-          new_attrs.each_pair { |k, v|
+          new_attrs.each_pair { |k, _v|
             if !cur_attrs.has_key?(k) or cur_attrs[k] != new_attrs[k]
               changed = true
             end
           }
           if changed
             MU.log "Updating SQS queue #{@mu_name}", MU::NOTICE, details: new_attrs
-            resp = MU::Cloud::AWS.sqs(region: @config['region'], credentials: @config['credentials']).set_queue_attributes(
+            MU::Cloud::AWS.sqs(region: @config['region'], credentials: @config['credentials']).set_queue_attributes(
               queue_url: @cloud_id,
               attributes: new_attrs
             )
@@ -74,10 +74,13 @@ module MU
           "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":sqs:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":"+@cloud_id
         end
 
+        @cloud_desc_cache = nil
         # Retrieve the AWS descriptor for this SQS queue. AWS doesn't exactly
         # provide one; if you want real information for SQS ask notify()
         # @return [Hash]: AWS doesn't return anything but the SQS URL, so supplement with attributes
-        def cloud_desc
+        def cloud_desc(use_cache: true)
+          return @cloud_desc_cache if @cloud_desc_cache and use_cache
+
           if !@cloud_id
             resp = MU::Cloud::AWS.sqs(region: @config['region'], credentials: @config['credentials']).list_queues(
               queue_name_prefix: @mu_name
@@ -92,11 +95,12 @@ module MU
           end
 
           return nil if !@cloud_id
-          MU::Cloud::AWS::MsgQueue.find(
+          @cloud_desc_cache = MU::Cloud::AWS::MsgQueue.find(
             cloud_id: @cloud_id.dup,
             region: @config['region'],
             credentials: @config['credentials']
           )
+          @cloud_desc_cache
         end
 
         # Return the metadata for this MsgQueue rule
@@ -130,6 +134,9 @@ module MU
         # @param region [String]: The cloud provider region
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+          MU.log "AWS::MsgQueue.cleanup: need to support flags['known']", MU::DEBUG, details: flags
+          MU.log "Placeholder: AWS MsgQueue artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
+
           resp = MU::Cloud::AWS.sqs(credentials: credentials, region: region).list_queues(
             queue_name_prefix: MU.deploy_id
           )
@@ -182,7 +189,7 @@ module MU
                 args[:cloud_id] = resp.queue_url if resp and resp.queue_url
               end
             end
-          rescue ::Aws::SQS::Errors::NonExistentQueue => e
+          rescue ::Aws::SQS::Errors::NonExistentQueue
           end
 
           # Go fetch its attributes
@@ -211,9 +218,9 @@ module MU
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
             "max_msg_size" => {
@@ -382,7 +389,7 @@ module MU
             end
             begin
               MU::Cloud::AWS.kms(region: queue['region']).describe_key(key_id: queue['kms']['key_id'])
-            rescue Aws::KMS::Errors::NotFoundException => e
+            rescue Aws::KMS::Errors::NotFoundException
               MU.log "KMS key '#{queue['kms']['key_id']}' specified in Queue '#{queue['name']}' was not found.", MU::ERR, details: "Key IDs are of the form bf64a093-2c3d-46fa-0d4f-8232fa7ed53. Keys can be created at https://console.aws.amazon.com/iam/home#/encryptionKeys/#{queue['region']}"
               ok = false
             end

data/modules/mu/clouds/aws/nosqldb.rb

@@ -164,6 +164,8 @@ module MU
         # @param region [String]: The cloud provider region
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+          MU.log "AWS::NoSQLDb.cleanup: need to support flags['known']", MU::DEBUG, details: flags
+
           resp = MU::Cloud::AWS.dynamo(credentials: credentials, region: region).list_tables
           if resp and resp.table_names
             resp.table_names.each { |table|
@@ -178,16 +180,23 @@ module MU
               begin
                 tags = MU::Cloud::AWS.dynamo(credentials: credentials, region: region).list_tags_of_resource(resource_arn: desc.table_arn)
                 if tags and tags.tags
+                  deploy_match = false
+                  master_match = false
                   tags.tags.each { |tag|
                     if tag.key == "MU-ID" and tag.value == MU.deploy_id
-                      MU.log "Deleting DynamoDB table #{desc.table_name}"
-                      if !noop
-                        MU::Cloud::AWS.dynamo(credentials: credentials, region: region).delete_table(table_name: desc.table_name)
-                      end
+                      deploy_match = true
+                    elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip
+                      master_match = true
                     end
                   }
+                  if deploy_match and (master_match or ignoremaster)
+                    MU.log "Deleting DynamoDB table #{desc.table_name}"
+                    if !noop
+                      MU::Cloud::AWS.dynamo(credentials: credentials, region: region).delete_table(table_name: desc.table_name)
+                    end
+                  end
                 end
-              rescue Aws::DynamoDB::Errors::ResourceNotFoundException => e
+              rescue Aws::DynamoDB::Errors::ResourceNotFoundException
               end
 
             }
@@ -236,9 +245,9 @@ module MU
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = ["attributes"]
 
 
@@ -360,9 +369,9 @@ module MU
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::nosqldbs}, bare and unvalidated.
 
         # @param db [Hash]: The resource to process and validate
-        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+        # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
         # @return [Boolean]: True if validation succeeded, False otherwise
-        def self.validateConfig(db, configurator)
+        def self.validateConfig(db, _configurator)
           ok = true
 
           partition = nil
@@ -399,8 +408,6 @@ module MU
           ok
         end
 
-        private
-
       end
     end
   end

data/modules/mu/clouds/aws/notifier.rb

@@ -66,12 +66,17 @@ module MU
         # @param region [String]: The cloud provider region
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+          MU.log "AWS::Notifier.cleanup: need to support flags['known']", MU::DEBUG, details: flags
+          MU.log "Placeholder: AWS Notifier artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
+
           MU::Cloud::AWS.sns(region: region, credentials: credentials).list_topics.topics.each { |topic|
             if topic.topic_arn.match(MU.deploy_id)
               # We don't have a way to tag our SNS topics, so we will delete any topic that has the MU-ID in its ARN. 
               # This may fail to find notifier groups in some cases (eg. cache_cluster) so we might want to delete from each API as well.
-              MU::Cloud::AWS.sns(region: region, credentials: credentials).delete_topic(topic_arn: topic.topic_arn)
-              MU.log "Deleted SNS topic: #{topic.topic_arn}"
+              MU.log "Deleting SNS topic: #{topic.topic_arn}"
+              if !noop
+                MU::Cloud::AWS.sns(region: region, credentials: credentials).delete_topic(topic_arn: topic.topic_arn)
+              end
             end
           }
         end
@@ -116,9 +121,9 @@ module MU
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
             "subscriptions" => {
@@ -143,9 +148,9 @@ module MU
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::notifier}, bare and unvalidated.
 
         # @param notifier [Hash]: The resource to process and validate
-        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+        # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
         # @return [Boolean]: True if validation succeeded, False otherwise
-        def self.validateConfig(notifier, configurator)
+        def self.validateConfig(notifier, _configurator)
           ok = true
 
           if notifier['subscriptions']

data/modules/mu/clouds/aws/role.rb

@@ -43,7 +43,7 @@ module MU
 
               policy_name = @mu_name+"-"+policy.keys.first.upcase
               MU.log "Creating IAM policy #{policy_name}"
-              resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).create_policy(
+              MU::Cloud::AWS.iam(credentials: @config['credentials']).create_policy(
                 policy_name: policy_name,
                 path: "/"+@deploy.deploy_id+"/",
                 policy_document: JSON.generate(policy.values.first),
@@ -56,8 +56,8 @@ module MU
             MU.log "Creating IAM role #{@mu_name}"
             @cloud_id = @mu_name
             path = @config['strip_path'] ? nil : "/"+@deploy.deploy_id+"/"
-            resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).create_role(
-              path: nil,
+            MU::Cloud::AWS.iam(credentials: @config['credentials']).create_role(
+              path: path,
               role_name: @mu_name,
               description: "Generated by Mu",
               assume_role_policy_document: gen_assume_role_policy_doc,
@@ -89,7 +89,6 @@ module MU
           end
 
           if @config['raw_policies'] or @config['attachable_policies']
-            attached_policies = []
             configured_policies = []
 
             if @config['raw_policies']
@@ -128,7 +127,7 @@ module MU
 
             # XXX not sure we're binding these sanely, validate that
             if @config['raw_policies']
-              pol_arns = MU::Cloud::AWS::Role.manageRawPolicies(
+              MU::Cloud::AWS::Role.manageRawPolicies(
                 @config['raw_policies'],
                 basename: @deploy.getResourceName(@config['name']),
                 credentials: @credentials
@@ -183,7 +182,7 @@ module MU
                 desc
               end
 
-            rescue Aws::IAM::Errors::NoSuchEntity => e
+            rescue Aws::IAM::Errors::NoSuchEntity
               MU.log "Creating IAM policy #{policy_name}", details: policy.values.first
               MU::Cloud::AWS.iam(credentials: credentials).create_policy(
                 policy_name: policy_name,
@@ -208,59 +207,62 @@ module MU
           end
         end
 
+        @cloud_desc_cache = nil
         # Return a hash containing a +role+ element and a +policies+ element,
         # populated with one or both depending on what this resource has
         # defined.
-        def cloud_desc
-          desc = {}
+        def cloud_desc(use_cache: true)
+          return @cloud_desc_cache if @cloud_desc_cache and use_cache
+
+          @cloud_desc_cache = {}
           if @config['bare_policies']
             if @cloud_id
               pol_desc = MU::Cloud::AWS::Role.find(credentials: @credentials, cloud_id: @cloud_id).values.first
               if pol_desc
-                desc['policies'] = [pol_desc]
-                return desc
+                @cloud_desc_cache['policies'] = [pol_desc]
+                return @cloud_desc_cache
               end
             end
 
             if @deploy and @deploy.deploy_id
-              desc["policies"] = MU::Cloud::AWS.iam(credentials: @credentials).list_policies(
+              @cloud_desc_cache["policies"] = MU::Cloud::AWS.iam(credentials: @credentials).list_policies(
                 path_prefix: "/"+@deploy.deploy_id+"/"
               ).policies
-              desc["policies"].reject! { |p|
+              @cloud_desc_cache["policies"].reject! { |p|
                 !p.policy_name.match(/^#{Regexp.quote(@mu_name)}-/)
               }
               # this is quasi-wrong because we can be mulitple cloud is, but
               # we can't really set this type to has_multiples because that's
               # just how managed policies work not anything else, goddammit
               # AWS why can't you just bundle everything in roles
-              if desc["policies"] and desc["policies"].size > 0
-                @cloud_id ||= desc["policies"].first.arn
+              if @cloud_desc_cache["policies"] and @cloud_desc_cache["policies"].size > 0
+                @cloud_id ||= @cloud_desc_cache["policies"].first.arn
               end
             end
           else
             if @cloud_id.match(/^arn:aws(:?-us-gov)?:[^:]*:[^:]*:\d*:policy\//)
               pol_desc = MU::Cloud::AWS::Role.find(credentials: @credentials, cloud_id: @cloud_id).values.first
               if pol_desc
-                desc['policies'] = [pol_desc]
-                return desc
+                @cloud_desc_cache['policies'] = [pol_desc]
+                return @cloud_desc_cache
               end
             end
 begin
-            desc['role'] = MU::Cloud::AWS::Role.find(credentials: @credentials, cloud_id: @cloud_id).values.first
-            desc['role'] ||= MU::Cloud::AWS::Role.find(credentials: @credentials, cloud_id: @mu_name).values.first
+            @cloud_desc_cache['role'] = MU::Cloud::AWS::Role.find(credentials: @credentials, cloud_id: @cloud_id).values.first
+            @cloud_desc_cache['role'] ||= MU::Cloud::AWS::Role.find(credentials: @credentials, cloud_id: @mu_name).values.first
             MU::Cloud::AWS.iam(credentials: @credentials).list_attached_role_policies(
               role_name: @mu_name
             ).attached_policies.each { |p|
-              desc["policies"] ||= []
-              desc["policies"] << MU::Cloud::AWS.iam(credentials: @credentials).get_policy(
+              @cloud_desc_cache["policies"] ||= []
+              @cloud_desc_cache["policies"] << MU::Cloud::AWS.iam(credentials: @credentials).get_policy(
                 policy_arn: p.policy_arn
               ).policy
             }
 
             inline = MU::Cloud::AWS.iam(credentials: @credentials).list_role_policies(role_name: @mu_name).policy_names
             inline.each { |pol_name|
-              desc["policies"] ||= []
-              desc["policies"] << MU::Cloud::AWS.iam(credentials: @credentials).get_role_policy(
+              @cloud_desc_cache["policies"] ||= []
+              @cloud_desc_cache["policies"] << MU::Cloud::AWS.iam(credentials: @credentials).get_role_policy(
                 role_name: @mu_name,
                 policy_name: pol_name
               )
@@ -270,9 +272,9 @@ rescue ::Aws::IAM::Errors::ValidationError => e
 MU.log @cloud_id+" "+@mu_name, MU::WARN, details: e.inspect
 end
           end
-          desc['cloud_id'] ||= @cloud_id
+          @cloud_desc_cache['cloud_id'] ||= @cloud_id
 
-          desc
+          @cloud_desc_cache
         end
 
         # Return the metadata for this user cofiguration
@@ -284,8 +286,7 @@ end
         # Insert a new target entity into an existing policy. 
         # @param policy [String]: The name of the policy to which we're appending, which must already exist as part of this role resource
         # @param targets [Array<String>]: The target resource. If +target_type+ isn't specified, this should be a fully-resolved ARN.
-        # @param mu_type [String]: A valid Mu resource type
-        def injectPolicyTargets(policy, targets, mu_type = nil)
+        def injectPolicyTargets(policy, targets)
           if !policy.match(/^#{@deploy.deploy_id}/)
             policy = @mu_name+"-"+policy.upcase
           end
@@ -409,9 +410,8 @@ end
         # Remove all roles associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
-        # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
 
           resp = MU::Cloud::AWS.iam(credentials: credentials).list_policies(
             path_prefix: "/"+MU.deploy_id+"/"
@@ -438,12 +438,18 @@ end
             # check tags. Hardly seems efficient.
             desc = MU::Cloud::AWS.iam(credentials: credentials).get_role(role_name: r.role_name)
             if desc.role and desc.role.tags and desc.role.tags
+              master_match = false
+              deploy_match = false
               desc.role.tags.each { |t|
                 if t.key == "MU-ID" and t.value == MU.deploy_id
-                  deleteme << r
-                  break
+                  deploy_match = true
+                elsif t.key == "MU-MASTER-IP" and t.value == MU.mu_public_ip
+                  master_match = true
                 end
               }
+              if deploy_match and (master_match or ignoremaster)
+                deleteme << r
+              end
             end
           }
 
@@ -481,7 +487,7 @@ end
                   MU::Cloud::AWS.iam(credentials: credentials).delete_instance_profile(instance_profile_name: r.role_name)
                 rescue Aws::IAM::Errors::ValidationError => e
                   MU.log "Cleaning up IAM role #{r.role_name}: #{e.inspect}", MU::WARN
-                rescue Aws::IAM::Errors::NoSuchEntity => e
+                rescue Aws::IAM::Errors::NoSuchEntity
                 end
 
                 MU::Cloud::AWS.iam(credentials: credentials).delete_role(
@@ -552,7 +558,7 @@ end
         # Reverse-map our cloud description into a runnable config hash.
         # We assume that any values we have in +@config+ are placeholders, and
         # calculate our own accordingly based on what's live in the cloud.
-        def toKitten(rootparent: nil, billing: nil, habitats: nil)
+        def toKitten(**_args)
           bok = {
             "cloud" => "AWS",
             "credentials" => @config['credentials'],
@@ -721,7 +727,7 @@ end
               "targets" => s["Resource"].map { |r|
                 if r.match(/^arn:aws(-us-gov)?:([^:]+):.*?:([^:]*)$/)
 # XXX which cases even count for blind references to sibling resources?
-                  type = if Regexp.last_match[1] == "s3"
+                  if Regexp.last_match[1] == "s3"
                     "bucket"
                   elsif Regexp.last_match[1]
                     MU.log "Service #{Regexp.last_match[1]} to type...", MU::WARN, details: r
@@ -741,7 +747,7 @@ end
 
         # Attach this role or group of loose policies to the specified entity.
         # @param entitytype [String]: The type of entity (user, group or role for policies; instance_profile for roles)
-        def bindTo(entitytype, entityname, policies = [])
+        def bindTo(entitytype, entityname)
           if entitytype == "instance_profile"
             begin
               resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_instance_profile(
@@ -754,7 +760,7 @@ end
                   role_name: @mu_name
                 )
               end
-            rescue Exception => e
+            rescue StandardError => e
               MU.log "Error binding role #{@mu_name} to instance profile #{entityname}: #{e.message}", MU::ERR
               raise e
             end
@@ -783,7 +789,6 @@ end
                 rescue Aws::IAM::Errors::NoSuchEntity => e
                   if subpaths.size > 0
                     p_arn = "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":iam::aws:policy/#{subpaths.shift}/"+policy
-                    retried = true
                     retry
                   end
                   raise e
@@ -847,7 +852,7 @@ end
             MU::Cloud::AWS.iam(credentials: @config['credentials']).create_instance_profile(
               instance_profile_name: @mu_name
             )
-          rescue Aws::IAM::Errors::EntityAlreadyExists => e
+          rescue Aws::IAM::Errors::EntityAlreadyExists
             MU::Cloud::AWS.iam(credentials: @config['credentials']).get_instance_profile(
               instance_profile_name: @mu_name
             )
@@ -905,9 +910,9 @@ end
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           aws_resource_types = MU::Cloud.resource_types.keys.reject { |t|
             begin
@@ -1012,10 +1017,9 @@ end
             subpaths = ["service-role", "aws-service-role", "job-function"]
             begin
               MU::Cloud::AWS.iam(credentials: credentials).get_policy(policy_arn: arn)
-            rescue Aws::IAM::Errors::NoSuchEntity => e
+            rescue Aws::IAM::Errors::NoSuchEntity
               if subpaths.size > 0
                 arn = "arn:"+(MU::Cloud::AWS.isGovCloud?(region) ? "aws-us-gov" : "aws")+":iam::aws:policy/#{subpaths.shift}/"+ref["id"]
-                retried = true
                 retry
               end
               MU.log "No such canned AWS IAM policy '#{arn}'", MU::ERR
@@ -1029,9 +1033,9 @@ end
 
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::roles}, bare and unvalidated.
         # @param role [Hash]: The resource to process and validate
-        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+        # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
         # @return [Boolean]: True if validation succeeded, False otherwise
-        def self.validateConfig(role, configurator)
+        def self.validateConfig(role, _configurator)
           ok = true
 
           # munge things declared with the deprecated import keyword into
@@ -1094,9 +1098,7 @@ end
         # @param policies [Array<Hash>]: One or more policy chunks
         # @param deploy_obj [MU::MommaCat]: Deployment object to use when looking up sibling Mu resources
         # @return [Array<Hash>]
-        def self.genPolicyDocument(policies, deploy_obj: nil)
-          iam_policies = []
-
+        def self.genPolicyDocument(policies, deploy_obj: nil, bucket_style: false)
           if policies
             name = nil
             doc = {
@@ -1134,12 +1136,21 @@ end
                     )
                     if sibling
                       id = sibling.cloudobj.arn
-                      statement["Principal"] << id
+                      if bucket_style
+                        statement["Principal"] << { "AWS" => id }
+                      else
+                        statement["Principal"] << id
+                      end
                     else
                       raise MuError, "Couldn't find a #{grantee["type"]} named #{grantee["identifier"]} when generating IAM policy"
                     end
                   else
-                    statement["Principal"] << grantee["identifier"]
+                    bucket_prefix = grantee["identifier"].match(/^[^\.]+\.amazonaws\.com$/) ? "Service" : "AWS"
+                    if bucket_style
+                      statement["Principal"] << { bucket_prefix => grantee["identifier"] }
+                    else
+                      statement["Principal"] << grantee["identifier"]
+                    end
                   end
                 }
                 if policy["grant_to"].size == 1
@@ -1162,6 +1173,8 @@ end
                         stream_id = id.sub(/:([^:]+)$/, ":log-stream:*")
 #                        "arn:aws:logs:us-east-2:accountID:log-group:log_group_name:log-stream:CloudTrail_log_stream_name_prefix*"
                         statement["Resource"] << stream_id
+                      elsif id.match(/:s3:/)
+                        statement["Resource"] << id+"/*"
                       end
                     else
                       raise MuError, "Couldn't find a #{target["entity_type"]} named #{target["identifier"]} when generating IAM policy"
@@ -1180,6 +1193,32 @@ end
           []
         end
 
+        # Update a policy, handling deletion of old versions as needed
+        # @param arn [String]:
+        # @param doc [Hash]:
+        # @param credentials [String]:
+        def self.update_policy(arn, doc, credentials: nil)
+# XXX this is just blindly replacing identical versions, when it should check
+# and guard
+          begin
+            MU::Cloud::AWS.iam(credentials: credentials).create_policy_version(
+              policy_arn: arn,
+              set_as_default: true,
+              policy_document: JSON.generate(doc)
+            )
+          rescue Aws::IAM::Errors::LimitExceeded
+            delete_version = MU::Cloud::AWS.iam(credentials: credentials).list_policy_versions(
+              policy_arn: arn,
+            ).versions.last.version_id
+            MU.log "Purging oldest version (#{delete_version}) of IAM policy #{arn}", MU::NOTICE
+            MU::Cloud::AWS.iam(credentials: credentials).delete_policy_version(
+              policy_arn: arn,
+              version_id: delete_version
+            )
+            retry
+          end
+        end
+
         private
 
         # Convert entries from the cloud-neutral @config['policies'] list into
@@ -1261,28 +1300,6 @@ end
           MU::Cloud::AWS::Role.update_policy(arn, doc, credentials: @credentials)
         end
 
-        # Update a policy, handling deletion of old versions as needed
-        def self.update_policy(arn, doc, credentials: nil)
-# XXX this is just blindly replacing identical versions, when it should check
-# and guard
-          begin
-            MU::Cloud::AWS.iam(credentials: credentials).create_policy_version(
-              policy_arn: arn,
-              set_as_default: true,
-              policy_document: JSON.generate(doc)
-            )
-          rescue Aws::IAM::Errors::LimitExceeded => e
-            delete_version = MU::Cloud::AWS.iam(credentials: credentials).list_policy_versions(
-              policy_arn: arn,
-            ).versions.last.version_id
-            MU.log "Purging oldest version (#{delete_version}) of IAM policy #{arn}", MU::NOTICE
-            MU::Cloud::AWS.iam(credentials: credentials).delete_policy_version(
-              policy_arn: arn,
-              version_id: delete_version
-            )
-            retry
-          end
-        end
 
       end
     end