cloud-mu 3.1.6 → 3.4.0

data/modules/mu/{clouds → providers}/aws/notifier.rb

@@ -27,8 +27,8 @@ module MU
 
         # Called automatically by {MU::Deploy#createResources}
         def create
-          MU::Cloud::AWS.sns(region: @config['region'], credentials: @config['credentials']).create_topic(name: @mu_name)
           @cloud_id = @mu_name
+          MU::Cloud::AWS.sns(region: @config['region'], credentials: @config['credentials']).create_topic(name: @cloud_id)
           MU.log "Created SNS topic #{@mu_name}"
         end
 
@@ -36,17 +36,48 @@ module MU
         def groom
           if @config['subscriptions']
             @config['subscriptions'].each { |sub|
-              MU::Cloud::AWS::Notifier.subscribe(
-                arn: arn,
-                endpoint: sub['endpoint'],
-                region: @config['region'],
-                credentials: @config['credentials'],
-                protocol: sub['type']
-              )
+              if sub['resource'] and !sub['endpoint']
+                endpoint_obj = nil
+                MU.retrier([], max: 5, wait: 9, loop_if: Proc.new { endpoint_obj.nil? }) {
+                  endpoint_obj = MU::Config::Ref.get(sub['resource']).kitten(@deploy)
+                }
+                sub['endpoint'] = endpoint_obj.arn
+              end
+              subscribe(sub['endpoint'], sub['type'])
             }
           end
         end
 
+        # Subscribe something to this SNS topic
+        # @param endpoint [String]: The address, identifier, or ARN of the resource being subscribed
+        # @param protocol [String]: The protocol being subscribed
+        def subscribe(endpoint, protocol)
+          self.class.subscribe(arn, endpoint, protocol, region: @config['region'], credentials: @credentials)
+        end
+
+        # Subscribe something to an SNS topic
+        # @param cloud_id [String]: The short name or ARN of an existing SNS topic
+        # @param endpoint [String]: The address, identifier, or ARN of the resource being subscribed
+        # @param protocol [String]: The protocol being subscribed
+        # @param region [String]: The region of the target SNS topic
+        # @param credentials [String]: 
+        def self.subscribe(cloud_id, endpoint, protocol, region: nil, credentials: nil)
+          topic = find(cloud_id: cloud_id, region: region, credentials: credentials).values.first
+          if !topic
+            raise MuError, "Failed to find SNS Topic #{cloud_id} in #{region}"
+          end
+          arn = topic["TopicArn"]
+
+          resp = MU::Cloud::AWS.sns(region: region, credentials: credentials).list_subscriptions_by_topic(topic_arn: arn).subscriptions
+
+          resp.each { |subscription|
+            return subscription if subscription.protocol == protocol and subscription.endpoint == endpoint
+          }
+
+          MU.log "Subscribing #{endpoint} (#{protocol}) to SNS topic #{arn}", MU::NOTICE
+          MU::Cloud::AWS.sns(region: region, credentials: credentials).subscribe(topic_arn: arn, protocol: protocol, endpoint: endpoint)
+        end
+
         # Does this resource type exist as a global (cloud-wide) artifact, or
         # is it localized to a region/zone?
         # @return [Boolean]
@@ -65,12 +96,12 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           MU.log "AWS::Notifier.cleanup: need to support flags['known']", MU::DEBUG, details: flags
           MU.log "Placeholder: AWS Notifier artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
 
           MU::Cloud::AWS.sns(region: region, credentials: credentials).list_topics.topics.each { |topic|
-            if topic.topic_arn.match(MU.deploy_id)
+            if topic.topic_arn.match(deploy_id)
               # We don't have a way to tag our SNS topics, so we will delete any topic that has the MU-ID in its ARN. 
               # This may fail to find notifier groups in some cases (eg. cache_cluster) so we might want to delete from each API as well.
               MU.log "Deleting SNS topic: #{topic.topic_arn}"
@@ -91,6 +122,7 @@ module MU
         # Return the metadata for this user cofiguration
         # @return [Hash]
         def notify
+          return nil if !@cloud_id or !cloud_desc(use_cache: false)
           desc = MU::Cloud::AWS.sns(region: @config["region"], credentials: @config["credentials"]).get_topic_attributes(topic_arn: arn).attributes
           MU.structToHash(desc)
         end
@@ -101,9 +133,16 @@ module MU
           found = {}
 
           if args[:cloud_id]
-            arn = "arn:"+(MU::Cloud::AWS.isGovCloud?(args[:region]) ? "aws-us-gov" : "aws")+":sns:"+args[:region]+":"+MU::Cloud::AWS.credToAcct(args[:credentials])+":"+args[:cloud_id]
-            desc = MU::Cloud::AWS.sns(region: args[:region], credentials: args[:credentials]).get_topic_attributes(topic_arn: arn).attributes
-            found[args[:cloud_id]] = desc if desc
+            arn = if args[:cloud_id].match(/^arn:/)
+              args[:cloud_id] 
+            else
+              "arn:"+(MU::Cloud::AWS.isGovCloud?(args[:region]) ? "aws-us-gov" : "aws")+":sns:"+args[:region]+":"+MU::Cloud::AWS.credToAcct(args[:credentials])+":"+args[:cloud_id]
+            end
+            begin
+              desc = MU::Cloud::AWS.sns(region: args[:region], credentials: args[:credentials]).get_topic_attributes(topic_arn: arn).attributes
+              found[args[:cloud_id]] = desc if desc
+            rescue ::Aws::SNS::Errors::NotFound
+            end
           else
             next_token = nil
             begin
@@ -120,6 +159,58 @@ module MU
           found
         end
 
+        # Reverse-map our cloud description into a runnable config hash.
+        # We assume that any values we have in +@config+ are placeholders, and
+        # calculate our own accordingly based on what's live in the cloud.
+        def toKitten(**_args)
+          bok = {
+            "cloud" => "AWS",
+            "credentials" => @config['credentials'],
+            "cloud_id" => @cloud_id,
+            "region" => @config['region']
+          }
+
+          if !cloud_desc
+            MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
+            return nil
+          end
+
+          bok['name'] = cloud_desc["DisplayName"].empty? ? @cloud_id : cloud_desc["DisplayName"]
+          svcmap = {
+            "lambda" => "functions",
+            "sqs" => "msg_queues"
+          }
+          MU::Cloud::AWS.sns(region: @config['region'], credentials: @credentials).list_subscriptions_by_topic(topic_arn: cloud_desc["TopicArn"]).subscriptions.each { |sub|
+            bok['subscriptions'] ||= []
+
+            bok['subscriptions'] << if sub.endpoint.match(/^arn:[^:]+:(sqs|lambda):([^:]+):(\d+):.*?([^:\/]+)$/)
+              _wholestring, service, region, account, id = Regexp.last_match.to_a
+              {
+                "type" => sub.protocol,
+                "resource" => MU::Config::Ref.get(
+                  type: svcmap[service],
+                  region: region,
+                  credentials: @credentials,
+                  id: id,
+                  cloud: "AWS",
+                  habitat: MU::Config::Ref.get(
+                    id: account,
+                    cloud: "AWS",
+                    credentials: @credentials
+                  )
+                )
+              }
+            else
+              {
+                "type" => sub.protocol,
+                "endpoint" => sub.endpoint
+              }
+            end
+          }
+
+          bok
+        end
+
         # Cloud-specific configuration properties.
         # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
@@ -130,11 +221,10 @@ module MU
               "type" => "array",
               "items" => {
                 "type" => "object",
-                "required" => ["endpoint"],
                 "properties" => {
                   "type" => {
                     "type" => "string",
-                    "description" => "",
+                    "description" => "Type of endpoint or resource which should receive notifications. If not specified, will attempt to auto-detect.",
                     "enum" => ["http", "https", "email", "email-json", "sms", "sqs", "application", "lambda"]
                   }
                 }
@@ -148,26 +238,42 @@ module MU
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::notifier}, bare and unvalidated.
 
         # @param notifier [Hash]: The resource to process and validate
-        # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
         # @return [Boolean]: True if validation succeeded, False otherwise
-        def self.validateConfig(notifier, _configurator)
+        def self.validateConfig(notifier, configurator)
           ok = true
 
           if notifier['subscriptions']
             notifier['subscriptions'].each { |sub|
+              if sub['resource'] and configurator.haveLitterMate?(sub['resource']['name'], sub['resource']['type'])
+                sub['resource']['cloud'] = "AWS"
+                MU::Config.addDependency(notifier, sub['resource']['name'], sub['resource']['type'])
+              end
               if !sub["type"]
-                if sub["endpoint"].match(/^http:/i)
-                  sub["type"] = "http"
-                elsif sub["endpoint"].match(/^https:/i)
-                  sub["type"] = "https"
-                elsif sub["endpoint"].match(/^sqs:/i)
-                  sub["type"] = "sqs"
-                elsif sub["endpoint"].match(/^\+?[\d\-]+$/)
-                  sub["type"] = "sms"
-                elsif sub["endpoint"].match(/\A[\w+\-.]+@[a-z\d\-]+(\.[a-z]+)*\.[a-z]+\z/i)
-                  sub["type"] = "email"
-                else
-                  MU.log "Notifier #{notifier['name']} subscription #{sub['endpoint']} did not specify a type, and I'm unable to guess one", MU::ERR
+                sub['type'] = if sub['resource']
+                  if sub['resource']['type'] == "functions"
+                    "lambda"
+                  elsif sub['resource']['type'] == "msg_queues"
+                    "sqs"
+                  end
+                elsif sub['endpoint']
+                  if sub["endpoint"].match(/^http:/i)
+                    "http"
+                  elsif sub["endpoint"].match(/^https:/i)
+                    "https"
+                  elsif sub["endpoint"].match(/:sqs:/i)
+                    "sqs"
+                  elsif sub["endpoint"].match(/:lambda:/i)
+                    "lambda"
+                  elsif sub["endpoint"].match(/^\+?[\d\-]+$/)
+                    "sms"
+                  elsif sub["endpoint"].match(/\A[\w+\-.]+@[a-z\d\-]+(\.[a-z]+)*\.[a-z]+\z/i)
+                    "email"
+                  end
+                end
+
+                if !sub['type']
+                  MU.log "Notifier #{notifier['name']} subscription did not specify a type, and I'm unable to guess one", MU::ERR, details: sub
                   ok = false
                 end
               end
@@ -178,40 +284,6 @@ module MU
         end
 
 
-        # Subscribe to a notifier group. This can either be an email address, SQS queue, application endpoint, etc...
-        # Will create the subscription only if it doesn't already exist.
-        # @param arn [String]: The cloud provider's identifier of the notifier group.
-        # @param protocol [String]: The type of the subscription (eg. email,https, etc..).
-        # @param endpoint [String]: The endpoint of the subscription. This will depend on the 'protocol' (as an example if protocol is email, endpoint will be the email address) ..
-        # @param region [String]: The cloud provider region.
-        def self.subscribe(arn: nil, protocol: nil, endpoint: nil, region: MU.curRegion, credentials: nil)
-          retries = 0
-          begin 
-            resp = MU::Cloud::AWS.sns(region: region, credentials: credentials).list_subscriptions_by_topic(topic_arn: arn).subscriptions
-          rescue Aws::SNS::Errors::NotFound
-            if retries < 5
-              MU.log "Couldn't find topic #{arn}, retrying several times in case of a lagging resource"
-              retries += 1
-              sleep 30
-              retry
-            else
-              raise MuError, "Couldn't find topic #{arn}, giving up"
-            end
-          end
-
-          already_subscribed = false
-          if resp && !resp.empty?
-            resp.each { |subscription|
-             already_subscribed = true if subscription.protocol == protocol && subscription.endpoint == endpoint
-            }
-          end
-
-          unless already_subscribed
-            MU::Cloud::AWS.sns(region: region, credentials: credentials).subscribe(topic_arn: arn, protocol: protocol, endpoint: endpoint)
-            MU.log "Subscribed #{endpoint} to SNS topic #{arn}"
-          end
-        end
-
         # Test if a notifier group exists
         # Create a new notifier group. Will check if the group exists before creating it.
         # @param topic_name [String]: The cloud provider's name for the notifier group.

data/modules/mu/{clouds → providers}/aws/role.rb

@@ -30,7 +30,7 @@ module MU
             end
           end
 
-          @mu_name ||= @deploy.getResourceName(@config["name"])
+          @mu_name ||= @deploy.getResourceName(@config["name"], max_length: 64)
         end
 
         # Called automatically by {MU::Deploy#createResources}
@@ -43,7 +43,7 @@ module MU
 
               policy_name = @mu_name+"-"+policy.keys.first.upcase
               MU.log "Creating IAM policy #{policy_name}"
-              MU::Cloud::AWS.iam(credentials: @config['credentials']).create_policy(
+              MU::Cloud::AWS.iam(credentials: @credentials).create_policy(
                 policy_name: policy_name,
                 path: "/"+@deploy.deploy_id+"/",
                 policy_document: JSON.generate(policy.values.first),
@@ -53,16 +53,18 @@ module MU
           end
 
           if !@config['bare_policies']
-            MU.log "Creating IAM role #{@mu_name}"
             @cloud_id = @mu_name
             path = @config['strip_path'] ? nil : "/"+@deploy.deploy_id+"/"
-            MU::Cloud::AWS.iam(credentials: @config['credentials']).create_role(
-              path: path,
-              role_name: @mu_name,
-              description: "Generated by Mu",
-              assume_role_policy_document: gen_assume_role_policy_doc,
-              tags: get_tag_params
-            )
+            params = {
+              :path => path,
+              :role_name => @mu_name,
+              :description => "Generated by Mu",
+              :assume_role_policy_document => gen_assume_role_policy_doc,
+              :tags => get_tag_params
+            }
+
+            MU.log "Creating IAM role #{@mu_name} (#{@credentials})", details: params
+            MU::Cloud::AWS.iam(credentials: @credentials).create_role(params)
           end
         end
 
@@ -75,7 +77,7 @@ module MU
           end
 
           if !@config['bare_policies']
-            resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_role(
+            resp = MU::Cloud::AWS.iam(credentials: @credentials).get_role(
               role_name: @mu_name
             ).role
             ext_tags = resp.tags.map { |t| t.to_h }
@@ -84,7 +86,7 @@ module MU
 
             if tag_param.size > 0
               MU.log "Updating tags on IAM role #{@mu_name}", MU::NOTICE, details: tag_param
-              MU::Cloud::AWS.iam(credentials: @config['credentials']).tag_role(role_name: @mu_name, tags: tag_param)
+              MU::Cloud::AWS.iam(credentials: @credentials).tag_role(role_name: @mu_name, tags: tag_param)
             end
           end
 
@@ -92,13 +94,14 @@ module MU
             configured_policies = []
 
             if @config['raw_policies']
+              MU.log "Attaching #{@config['raw_policies'].size.to_s} raw #{@config['raw_policies'].size > 1 ? "policies" : "policy"} to role #{@mu_name}", MU::NOTICE
               configured_policies = @config['raw_policies'].map { |p|
                 @mu_name+"-"+p.keys.first.upcase
               }
             end
 
             if @config['attachable_policies']
-              MU.log "Attaching #{@config['attachable_policies'].size.to_s} #{@config['attachable_policies'].size > 1 ? "policies" : "policy"} to role #{@mu_name}", MU::NOTICE
+              MU.log "Attaching #{@config['attachable_policies'].size.to_s} external #{@config['attachable_policies'].size > 1 ? "policies" : "policy"} to role #{@mu_name}", MU::NOTICE
               configured_policies.concat(@config['attachable_policies'].map { |p|
                 id = if p.is_a?(MU::Config::Ref)
                   p.cloud_id
@@ -109,18 +112,17 @@ module MU
                 end
                 id.gsub(/.*?\/([^:\/]+)$/, '\1')
               })
-              configured_policies.each { |pol|
-              }
             end
 
+            # Purge anything that doesn't belong
             if !@config['bare_policies']
-              attached_policies = MU::Cloud::AWS.iam(credentials: @config['credentials']).list_attached_role_policies(
+              attached_policies = MU::Cloud::AWS.iam(credentials: @credentials).list_attached_role_policies(
                 role_name: @mu_name
               ).attached_policies
               attached_policies.each { |a|
                 if !configured_policies.include?(a.policy_name)
-                  MU.log "Removing IAM policy #{a.policy_name} from role #{@mu_name}", MU::NOTICE
-                  MU::Cloud::AWS::Role.purgePolicy(a.policy_arn, @config['credentials'])
+                  MU.log "Removing IAM policy #{a.policy_name} from role #{@mu_name}", MU::NOTICE, details: configured_policies
+                  MU::Cloud::AWS::Role.purgePolicy(a.policy_arn, @credentials)
                 end
               }
             end
@@ -153,8 +155,8 @@ module MU
             policy.values.each { |p|
               p["Version"] ||= "2012-10-17"
             }
-            policy_name = basename+"-"+policy.keys.first.upcase
 
+            policy_name = basename+"-"+policy.keys.first.upcase
             arn = "arn:"+(MU::Cloud::AWS.isGovCloud? ? "aws-us-gov" : "aws")+":iam::"+MU::Cloud::AWS.credToAcct(credentials)+":policy#{path}/#{policy_name}"
             resp = begin
               desc = MU::Cloud::AWS.iam(credentials: credentials).get_policy(policy_arn: arn)
@@ -184,12 +186,17 @@ module MU
 
             rescue Aws::IAM::Errors::NoSuchEntity
               MU.log "Creating IAM policy #{policy_name}", details: policy.values.first
-              MU::Cloud::AWS.iam(credentials: credentials).create_policy(
+              desc = MU::Cloud::AWS.iam(credentials: credentials).create_policy(
                 policy_name: policy_name,
                 path: path+"/",
                 policy_document: JSON.generate(policy.values.first),
                 description: "Raw policy from #{basename}"
               )
+              MU.retrier([Aws::IAM::Errors::NoSuchEntity], loop_if: Proc.new { desc.nil? }) {
+                desc = MU::Cloud::AWS.iam(credentials: credentials).get_policy(policy_arn: arn)
+                pp desc
+              }
+              desc
             end
             arns << resp.policy.arn
           }
@@ -216,7 +223,22 @@ module MU
         # populated with one or both depending on what this resource has
         # defined.
         def cloud_desc(use_cache: true)
-          return @cloud_desc_cache if @cloud_desc_cache and use_cache
+
+          # we might inherit a naive cached description from the base cloud
+          # layer; rearrange it to our tastes
+          if @cloud_desc_cache.is_a?(::Aws::IAM::Types::Role)
+            new_desc = {
+              "role" => @cloud_desc_cache
+            }
+            @cloud_desc_cache = new_desc
+          elsif @cloud_desc_cache.is_a?(::Aws::IAM::Types::Policy)
+            new_desc = {
+              "policies" => [@cloud_desc_cache]
+            }
+            @cloud_desc_cache = new_desc
+          end
+
+          return @cloud_desc_cache if @cloud_desc_cache and !@cloud_desc_cache.empty? and use_cache
 
           @cloud_desc_cache = {}
           if @config['bare_policies']
@@ -301,7 +323,7 @@ end
           my_policies.each { |p|
             if p.policy_name == policy
               seen_policy = true
-              old = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_policy_version(
+              old = MU::Cloud::AWS.iam(credentials: @credentials).get_policy_version(
                 policy_arn: p.arn,
                 version_id: p.default_version_id
               ).policy_version
@@ -419,14 +441,14 @@ end
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
 
           resp = MU::Cloud::AWS.iam(credentials: credentials).list_policies(
-            path_prefix: "/"+MU.deploy_id+"/"
+            path_prefix: "/"+deploy_id+"/"
           )
           if resp and resp.policies
             resp.policies.each { |policy|
-              MU.log "Deleting IAM policy /#{MU.deploy_id}/#{policy.policy_name}"
+              MU.log "Deleting IAM policy /#{deploy_id}/#{policy.policy_name}"
               if !noop
                 purgePolicy(policy.arn, credentials)
               end
@@ -437,19 +459,23 @@ end
           roles = MU::Cloud::AWS::Role.find(credentials: credentials).values
           roles.each { |r|
             next if !r.respond_to?(:role_name)
-            if r.path.match(/^\/#{Regexp.quote(MU.deploy_id)}/)
+            if r.path.match(/^\/#{Regexp.quote(deploy_id)}/)
               deleteme << r
               next
             end
             # For some dumb reason, the list output that .find gets doesn't
             # include the tags, so we need to fetch each role individually to
             # check tags. Hardly seems efficient.
-            desc = MU::Cloud::AWS.iam(credentials: credentials).get_role(role_name: r.role_name)
+            desc = begin
+              MU::Cloud::AWS.iam(credentials: credentials).get_role(role_name: r.role_name)
+            rescue Aws::IAM::Errors::NoSuchEntity
+              next
+            end
             if desc.role and desc.role.tags and desc.role.tags
               master_match = false
               deploy_match = false
               desc.role.tags.each { |t|
-                if t.key == "MU-ID" and t.value == MU.deploy_id
+                if t.key == "MU-ID" and t.value == deploy_id
                   deploy_match = true
                 elsif t.key == "MU-MASTER-IP" and t.value == MU.mu_public_ip
                   master_match = true
@@ -516,7 +542,7 @@ end
 
             begin
               # managed policies get fetched by ARN, roles by plain name. Ok!
-              if args[:cloud_id].match(/^arn:/)
+              if args[:cloud_id].match(/^arn:.*?:policy\//)
                 resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).get_policy(
                   policy_arn: args[:cloud_id]
                 )
@@ -525,39 +551,26 @@ end
                 end
               else
                 resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).get_role(
-                  role_name: args[:cloud_id]
+                  role_name: args[:cloud_id].sub(/^arn:.*?\/([^:\/]+)$/, '\1') # XXX if it's an ARN, actually parse it and look in the correct account when applicable
                 )
+
                 if resp and resp.role
-                  found[args[:cloud_id]] = resp.role
+                  found[resp.role.role_name] = resp.role
                 end
               end
             rescue ::Aws::IAM::Errors::NoSuchEntity
             end
 
           else
-            marker = nil
-            begin
-              resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_roles(
-                marker: marker
-              )
-              break if !resp or !resp.roles
-              resp.roles.each { |role|
-                found[role.role_name] = role
-              }
-              marker = resp.marker
-            end while marker
+            resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_roles
+            resp.roles.each { |role|
+              found[role.role_name] = role
+            }
 
-            begin
-              resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_policies(
-                scope: "Local",
-                marker: marker
-              )
-              break if !resp or !resp.policies
-              resp.policies.each { |pol|
-                found[pol.arn] = pol
-              }
-              marker = resp.marker
-            end while marker
+            resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_policies(scope: "Local")
+            resp.policies.each { |pol|
+              found[pol.arn] = pol
+            }
           end
 
           found
@@ -569,7 +582,7 @@ end
         def toKitten(**_args)
           bok = {
             "cloud" => "AWS",
-            "credentials" => @config['credentials'],
+            "credentials" => @credentials,
             "cloud_id" => @cloud_id
           }
 
@@ -615,7 +628,6 @@ end
                   )
                   JSON.parse(URI.decode(version.policy_version.document))
                 end
-
                 bok["policies"] = MU::Cloud::AWS::Role.doc2MuPolicies(pol.policy_name, doc, bok["policies"])
               end
             }
@@ -695,6 +707,7 @@ end
           end
 
           bok["attachable_policies"].uniq! if bok["attachable_policies"]
+          bok["name"].gsub!(/[^a-zA-Z0-9_\-]/, "_")
 
           bok
         end
@@ -707,6 +720,10 @@ end
         def self.doc2MuPolicies(basename, doc, policies = [])
           policies ||= []
 
+          if !doc["Statement"].is_a?(Array)
+            doc["Statement"] = [doc["Statement"]]
+          end
+
           doc["Statement"].each { |s|
             if !s["Action"]
               MU.log "Statement in policy document for #{basename} didn't have an Action field", MU::WARN, details: doc
@@ -758,12 +775,12 @@ end
         def bindTo(entitytype, entityname)
           if entitytype == "instance_profile"
             begin
-              resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_instance_profile(
+              resp = MU::Cloud::AWS.iam(credentials: @credentials).get_instance_profile(
                 instance_profile_name: entityname
               ).instance_profile
 
               if !resp.roles.map { |r| r.role_name}.include?(@mu_name)
-                MU::Cloud::AWS.iam(credentials: @config['credentials']).add_role_to_instance_profile(
+                MU::Cloud::AWS.iam(credentials: @credentials).add_role_to_instance_profile(
                   instance_profile_name: entityname,
                   role_name: @mu_name
                 )
@@ -773,7 +790,7 @@ end
               raise e
             end
           elsif ["user", "group", "role"].include?(entitytype)
-            mypolicies = MU::Cloud::AWS.iam(credentials: @config['credentials']).list_policies(
+            mypolicies = MU::Cloud::AWS.iam(credentials: @credentials).list_policies(
               path_prefix: "/"+@deploy.deploy_id+"/"
             ).policies
             mypolicies.reject! { |p|
@@ -791,7 +808,7 @@ end
 
                 subpaths = ["service-role", "aws-service-role", "job-function"]
                 begin
-                  mypolicies << MU::Cloud::AWS.iam(credentials: @config['credentials']).get_policy(
+                  mypolicies << MU::Cloud::AWS.iam(credentials: @credentials).get_policy(
                     policy_arn: p_arn
                   ).policy
                 rescue Aws::IAM::Errors::NoSuchEntity => e
@@ -804,39 +821,52 @@ end
               }
             end
 
+            if @config['raw_policies']
+              raw_arns = MU::Cloud::AWS::Role.manageRawPolicies(
+                @config['raw_policies'],
+                basename: @deploy.getResourceName(@config['name']),
+                credentials: @credentials
+              )
+              raw_arns.each { |p_arn|
+                mypolicies << MU::Cloud::AWS.iam(credentials: @credentials).get_policy(
+                  policy_arn: p_arn
+                ).policy
+              }
+            end
+
             mypolicies.each { |p|
               if entitytype == "user"
-                resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).list_attached_user_policies(
+                resp = MU::Cloud::AWS.iam(credentials: @credentials).list_attached_user_policies(
                   path_prefix: "/"+@deploy.deploy_id+"/",
                   user_name: entityname
                 )
                 if !resp or !resp.attached_policies.map { |a_p| a_p.policy_name }.include?(p.policy_name)
                   MU.log "Attaching IAM policy #{p.policy_name} to user #{entityname}", MU::NOTICE
-                  MU::Cloud::AWS.iam(credentials: @config['credentials']).attach_user_policy(
+                  MU::Cloud::AWS.iam(credentials: @credentials).attach_user_policy(
                     policy_arn: p.arn,
                     user_name: entityname
                   )
                 end
               elsif entitytype == "group"
-                resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).list_attached_group_policies(
+                resp = MU::Cloud::AWS.iam(credentials: @credentials).list_attached_group_policies(
                   path_prefix: "/"+@deploy.deploy_id+"/",
                   group_name: entityname
                 )
                 if !resp or !resp.attached_policies.map { |a_p| a_p.policy_name }.include?(p.policy_name)
                   MU.log "Attaching policy #{p.policy_name} to group #{entityname}", MU::NOTICE
-                  MU::Cloud::AWS.iam(credentials: @config['credentials']).attach_group_policy(
+                  MU::Cloud::AWS.iam(credentials: @credentials).attach_group_policy(
                     policy_arn: p.arn,
                     group_name: entityname
                   )
                 end
               elsif entitytype == "role"
-                resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).list_attached_role_policies(
+                resp = MU::Cloud::AWS.iam(credentials: @credentials).list_attached_role_policies(
                   role_name: entityname
                 )
 
                 if !resp or !resp.attached_policies.map { |a_p| a_p.policy_name }.include?(p.policy_name)
                   MU.log "Attaching policy #{p.policy_name} to role #{entityname}", MU::NOTICE
-                  MU::Cloud::AWS.iam(credentials: @config['credentials']).attach_role_policy(
+                  MU::Cloud::AWS.iam(credentials: @credentials).attach_role_policy(
                     policy_arn: p.arn,
                     role_name: entityname
                   )
@@ -857,19 +887,19 @@ end
           end
 
           resp = begin
-            MU.log "Creating instance profile #{@mu_name} #{@config['credentials']}"
-            MU::Cloud::AWS.iam(credentials: @config['credentials']).create_instance_profile(
+            MU.log "Creating instance profile #{@mu_name} #{@credentials}"
+            MU::Cloud::AWS.iam(credentials: @credentials).create_instance_profile(
               instance_profile_name: @mu_name
             )
           rescue Aws::IAM::Errors::EntityAlreadyExists
-            MU::Cloud::AWS.iam(credentials: @config['credentials']).get_instance_profile(
+            MU::Cloud::AWS.iam(credentials: @credentials).get_instance_profile(
               instance_profile_name: @mu_name
             )
           end
 
           # make sure it's really there before moving on
           begin
-            MU::Cloud::AWS.iam(credentials: @config['credentials']).get_instance_profile(instance_profile_name: @mu_name)
+            MU::Cloud::AWS.iam(credentials: @credentials).get_instance_profile(instance_profile_name: @mu_name)
           rescue Aws::IAM::Errors::NoSuchEntity => e
             MU.log e.inspect, MU::WARN
             sleep 10
@@ -925,7 +955,7 @@ end
           toplevel_required = []
           aws_resource_types = MU::Cloud.resource_types.keys.reject { |t|
             begin
-              MU::Cloud.loadCloudType("AWS", t)
+              MU::Cloud.resourceClass("AWS", t)
               false
             rescue MuCloudResourceNotImplemented
               true
@@ -1087,11 +1117,7 @@ end
             role['policies'].each { |policy|
               policy['targets'].each { |target|
                 if target['type']
-                  role['dependencies'] ||= []
-                  role['dependencies'] << {
-                    "name" => target['identifier'],
-                    "type" => target['type']
-                  }
+                  MU::Config.addDependency(role, target['identifier'], target['type'], no_create_wait: true)
                 end
               }
             }
@@ -1107,13 +1133,14 @@ end
         # @param policies [Array<Hash>]: One or more policy chunks
         # @param deploy_obj [MU::MommaCat]: Deployment object to use when looking up sibling Mu resources
         # @return [Array<Hash>]
-        def self.genPolicyDocument(policies, deploy_obj: nil, bucket_style: false)
+        def self.genPolicyDocument(policies, deploy_obj: nil, bucket_style: false, version: "2012-10-17", doc_id: nil)
           if policies
             name = nil
             doc = {
-              "Version" => "2012-10-17",
+              "Version" => version,
               "Statement" => []
             }
+            doc["Id"] = doc_id if doc_id
             policies.each { |policy|
               policy["flag"] ||= "Allow"
               statement = {
@@ -1154,7 +1181,14 @@ end
                       raise MuError, "Couldn't find a #{grantee["type"]} named #{grantee["identifier"]} when generating IAM policy"
                     end
                   else
-                    bucket_prefix = grantee["identifier"].match(/^[^\.]+\.amazonaws\.com$/) ? "Service" : "AWS"
+                    bucket_prefix = if grantee["identifier"].match(/^[^\.]+\.amazonaws\.com$/)
+                      "Service"
+                    elsif grantee["identifier"] =~ /^[a-f0-9]+$/
+                      "CanonicalUser"
+                    else
+                      "AWS"
+                    end
+
                     if bucket_style
                       statement["Principal"] << { bucket_prefix => grantee["identifier"] }
                     else