cloud-mu 3.2.0 → 3.3.0

data/modules/mu/providers/aws/role.rb

@@ -30,7 +30,7 @@ module MU
             end
           end
 
-          @mu_name ||= @deploy.getResourceName(@config["name"])
+          @mu_name ||= @deploy.getResourceName(@config["name"], max_length: 64)
         end
 
         # Called automatically by {MU::Deploy#createResources}
@@ -216,7 +216,22 @@ module MU
         # populated with one or both depending on what this resource has
         # defined.
         def cloud_desc(use_cache: true)
-          return @cloud_desc_cache if @cloud_desc_cache and use_cache
+
+          # we might inherit a naive cached description from the base cloud
+          # layer; rearrange it to our tastes
+          if @cloud_desc_cache.is_a?(::Aws::IAM::Types::Role)
+            new_desc = {
+              "role" => @cloud_desc_cache
+            }
+            @cloud_desc_cache = new_desc
+          elsif @cloud_desc_cache.is_a?(::Aws::IAM::Types::Policy)
+            new_desc = {
+              "policies" => [@cloud_desc_cache]
+            }
+            @cloud_desc_cache = new_desc
+          end
+
+          return @cloud_desc_cache if @cloud_desc_cache and !@cloud_desc_cache.empty? and use_cache
 
           @cloud_desc_cache = {}
           if @config['bare_policies']
@@ -419,14 +434,14 @@ end
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
 
           resp = MU::Cloud::AWS.iam(credentials: credentials).list_policies(
-            path_prefix: "/"+MU.deploy_id+"/"
+            path_prefix: "/"+deploy_id+"/"
           )
           if resp and resp.policies
             resp.policies.each { |policy|
-              MU.log "Deleting IAM policy /#{MU.deploy_id}/#{policy.policy_name}"
+              MU.log "Deleting IAM policy /#{deploy_id}/#{policy.policy_name}"
               if !noop
                 purgePolicy(policy.arn, credentials)
               end
@@ -437,19 +452,23 @@ end
           roles = MU::Cloud::AWS::Role.find(credentials: credentials).values
           roles.each { |r|
             next if !r.respond_to?(:role_name)
-            if r.path.match(/^\/#{Regexp.quote(MU.deploy_id)}/)
+            if r.path.match(/^\/#{Regexp.quote(deploy_id)}/)
               deleteme << r
               next
             end
             # For some dumb reason, the list output that .find gets doesn't
             # include the tags, so we need to fetch each role individually to
             # check tags. Hardly seems efficient.
-            desc = MU::Cloud::AWS.iam(credentials: credentials).get_role(role_name: r.role_name)
+            desc = begin
+              MU::Cloud::AWS.iam(credentials: credentials).get_role(role_name: r.role_name)
+            rescue Aws::IAM::Errors::NoSuchEntity
+              next
+            end
             if desc.role and desc.role.tags and desc.role.tags
               master_match = false
               deploy_match = false
               desc.role.tags.each { |t|
-                if t.key == "MU-ID" and t.value == MU.deploy_id
+                if t.key == "MU-ID" and t.value == deploy_id
                   deploy_match = true
                 elsif t.key == "MU-MASTER-IP" and t.value == MU.mu_public_ip
                   master_match = true
@@ -516,7 +535,7 @@ end
 
             begin
               # managed policies get fetched by ARN, roles by plain name. Ok!
-              if args[:cloud_id].match(/^arn:/)
+              if args[:cloud_id].match(/^arn:.*?:policy\//)
                 resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).get_policy(
                   policy_arn: args[:cloud_id]
                 )
@@ -525,39 +544,26 @@ end
                 end
               else
                 resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).get_role(
-                  role_name: args[:cloud_id]
+                  role_name: args[:cloud_id].sub(/^arn:.*?\/([^:\/]+)$/, '\1') # XXX if it's an ARN, actually parse it and look in the correct account when applicable
                 )
+
                 if resp and resp.role
-                  found[args[:cloud_id]] = resp.role
+                  found[resp.role.role_name] = resp.role
                 end
               end
             rescue ::Aws::IAM::Errors::NoSuchEntity
             end
 
           else
-            marker = nil
-            begin
-              resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_roles(
-                marker: marker
-              )
-              break if !resp or !resp.roles
-              resp.roles.each { |role|
-                found[role.role_name] = role
-              }
-              marker = resp.marker
-            end while marker
+            resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_roles
+            resp.roles.each { |role|
+              found[role.role_name] = role
+            }
 
-            begin
-              resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_policies(
-                scope: "Local",
-                marker: marker
-              )
-              break if !resp or !resp.policies
-              resp.policies.each { |pol|
-                found[pol.arn] = pol
-              }
-              marker = resp.marker
-            end while marker
+            resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_policies(scope: "Local")
+            resp.policies.each { |pol|
+              found[pol.arn] = pol
+            }
           end
 
           found
@@ -1091,7 +1097,7 @@ end
             role['policies'].each { |policy|
               policy['targets'].each { |target|
                 if target['type']
-                  MU::Config.addDependency(role, target['identifier'], target['type'])
+                  MU::Config.addDependency(role, target['identifier'], target['type'], no_create_wait: true)
                 end
               }
             }
@@ -1107,13 +1113,14 @@ end
         # @param policies [Array<Hash>]: One or more policy chunks
         # @param deploy_obj [MU::MommaCat]: Deployment object to use when looking up sibling Mu resources
         # @return [Array<Hash>]
-        def self.genPolicyDocument(policies, deploy_obj: nil, bucket_style: false)
+        def self.genPolicyDocument(policies, deploy_obj: nil, bucket_style: false, version: "2012-10-17", doc_id: nil)
           if policies
             name = nil
             doc = {
-              "Version" => "2012-10-17",
+              "Version" => version,
               "Statement" => []
             }
+            doc["Id"] = doc_id if doc_id
             policies.each { |policy|
               policy["flag"] ||= "Allow"
               statement = {
@@ -1154,7 +1161,14 @@ end
                       raise MuError, "Couldn't find a #{grantee["type"]} named #{grantee["identifier"]} when generating IAM policy"
                     end
                   else
-                    bucket_prefix = grantee["identifier"].match(/^[^\.]+\.amazonaws\.com$/) ? "Service" : "AWS"
+                    bucket_prefix = if grantee["identifier"].match(/^[^\.]+\.amazonaws\.com$/)
+                      "Service"
+                    elsif grantee["identifier"] =~ /^[a-f0-9]+$/
+                      "CanonicalUser"
+                    else
+                      "AWS"
+                    end
+
                     if bucket_style
                       statement["Principal"] << { bucket_prefix => grantee["identifier"] }
                     else

data/modules/mu/providers/aws/search_domain.rb

@@ -22,9 +22,9 @@ module MU
         # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
         def initialize(**args)
           super
-          if @cloud_id and !@config['domain_name']
-            @config['domain_name'] = @cloud_id
-          end
+          describe if @mu_name and !@deploydata
+          @cloud_id ||= @deploydata['domain_name'] if @deploydata
+
           @mu_name ||= @deploy.getResourceName(@config["name"])
         end
 
@@ -35,7 +35,8 @@ module MU
           params = genParams
 
           MU.log "Creating ElasticSearch domain #{@config['domain_name']}", details: params
-          MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).create_elasticsearch_domain(params).domain_status
+          @cloud_id = @config['domain_name']
+          MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).create_elasticsearch_domain(params).domain_status
 
           tagDomain
 
@@ -44,17 +45,18 @@ module MU
         # Called automatically by {MU::Deploy#createResources}
         def groom
           tagDomain
-          @config['domain_name'] ||= @deploydata['domain_name']
+          @config['domain_name'] ||= @cloud_id
           params = genParams(cloud_desc) # get parameters that would change only
 
           if params.size > 1
             waitWhileProcessing # wait until the create finishes, if still going
 
             MU.log "Updating ElasticSearch domain #{@config['domain_name']}", MU::NOTICE, details: params
-            MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).update_elasticsearch_domain_config(params)
+            MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).update_elasticsearch_domain_config(params)
           end
 
           waitWhileProcessing # don't return until creation/updating is complete
+          MU.log "Search Domain #{@config['name']}: #{cloud_desc.endpoint}", MU::SUMMARY
         end
 
         @cloud_desc_cache = nil
@@ -63,31 +65,30 @@ module MU
         # our druthers.
         def cloud_desc(use_cache: true)
           return @cloud_desc_cache if @cloud_desc_cache and use_cache
-          @cloud_desc_cache = if @config['domain_name']
-            MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).describe_elasticsearch_domain(
-              domain_name: @config['domain_name']
+          @cloud_id ||= @config['domain_name']
+          return nil if !@cloud_id
+          MU.retrier([::Aws::ElasticsearchService::Errors::ResourceNotFoundException], wait: 10, max: 12) {
+            @cloud_desc_cache = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).describe_elasticsearch_domain(
+              domain_name: @cloud_id
             ).domain_status
-          elsif @deploydata and @deploydata['domain_name']
-            MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).describe_elasticsearch_domain(
-              domain_name: @deploydata['domain_name']
-            ).domain_status
-          else
-            raise MuError, "#{@mu_name} can't find its official Elasticsearch domain name!"
-          end
+          }
+
           @cloud_desc_cache
         end
 
         # Canonical Amazon Resource Number for this resource
         # @return [String]
         def arn
-          cloud_desc.arn
+          return nil if !cloud_desc
+          cloud_desc.arn.dup
         end
 
         # Return the metadata for this SearchDomain rule
         # @return [Hash]
         def notify
-          deploy_struct = MU.structToHash(cloud_desc)
-          tags = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).list_tags(arn: deploy_struct[:arn]).tag_list
+          return nil if !cloud_desc(use_cache: false)
+          deploy_struct = MU.structToHash(cloud_desc, stringify_keys: true)
+          tags = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).list_tags(arn: arn).tag_list
           deploy_struct['tags'] = tags.map { |t| { t.key => t.value } }
           if deploy_struct['endpoint']
             deploy_struct['kibana'] = deploy_struct['endpoint']+"/_plugin/kibana/"
@@ -119,7 +120,7 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           MU.log "AWS::SearchDomain.cleanup: need to support flags['known']", MU::DEBUG, details: flags
 
           list = MU::Cloud::AWS.elasticsearch(region: region, credentials: credentials).list_domain_names
@@ -135,7 +136,7 @@ module MU
                 deploy_match = false
                 master_match = false
                 tags.tag_list.each { |tag|
-                  if tag.key == "MU-ID" and tag.value == MU.deploy_id
+                  if tag.key == "MU-ID" and tag.value == deploy_id
                     deploy_match = true
                   elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip
                     master_match = true
@@ -157,7 +158,7 @@ module MU
               resp = MU::Cloud::AWS.iam(credentials: credentials).list_roles(marker: marker)
               resp.roles.each{ |role|
                 # XXX Maybe we should have a more generic way to delete IAM profiles and policies. The call itself should be moved from MU::Cloud.resourceClass("AWS", "Server").
-#                MU::Cloud.resourceClass("AWS", "Server").removeIAMProfile(role.role_name) if role.role_name.match(/^#{Regexp.quote(MU.deploy_id)}/)
+#                MU::Cloud.resourceClass("AWS", "Server").removeIAMProfile(role.role_name) if role.role_name.match(/^#{Regexp.quote(deploy_id)}/)
               }
               marker = resp.marker
             end while resp.is_truncated
@@ -191,6 +192,96 @@ module MU
           found
         end
 
+        # Reverse-map our cloud description into a runnable config hash.
+        # We assume that any values we have in +@config+ are placeholders, and
+        # calculate our own accordingly based on what's live in the cloud.
+        def toKitten(**_args)
+          bok = {
+            "cloud" => "AWS",
+            "credentials" => @credentials,
+            "cloud_id" => @cloud_id,
+            "region" => @config['region']
+          }
+
+          if !cloud_desc
+            MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
+            return nil
+          end
+
+          bok['name'] = cloud_desc.domain_name
+          bok['elasticsearch_version'] = cloud_desc.elasticsearch_version
+          bok['instance_count'] = cloud_desc.elasticsearch_cluster_config.instance_count
+          bok['instance_type'] = cloud_desc.elasticsearch_cluster_config.instance_type
+          bok['zone_aware'] = cloud_desc.elasticsearch_cluster_config.zone_awareness_enabled
+
+          if cloud_desc.elasticsearch_cluster_config.dedicated_master_enabled
+            bok['dedicated_masters'] = cloud_desc.elasticsearch_cluster_config.dedicated_master_count
+            bok['master_instance_type'] = cloud_desc.elasticsearch_cluster_config.dedicated_master_type
+          end
+
+          if cloud_desc.access_policies and !cloud_desc.access_policies.empty?
+            bok['access_policies'] = JSON.parse(cloud_desc.access_policies)
+          end
+
+          if cloud_desc.advanced_options and !cloud_desc.advanced_options.empty?
+            bok['advanced_options'] = cloud_desc.advanced_options
+          end
+
+          bok['ebs_size'] = cloud_desc.ebs_options.volume_size
+          bok['ebs_type'] = cloud_desc.ebs_options.volume_type
+          bok['ebs_iops'] = cloud_desc.ebs_options.iops if cloud_desc.ebs_options.iops
+
+          if cloud_desc.snapshot_options and cloud_desc.snapshot_options.automated_snapshot_start_hour
+            bok['snapshot_hour'] = cloud_desc.snapshot_options.automated_snapshot_start_hour
+          end
+
+          if cloud_desc.cognito_options.user_pool_id and
+             cloud_desc.cognito_options.identity_pool_id
+            bok['user_pool_id'] = cloud_desc.cognito_options.user_pool_id
+            bok['identity_pool_id'] = cloud_desc.cognito_options.identity_pool_id
+          end
+
+          tags = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).list_tags(arn: cloud_desc.arn).tag_list
+          if tags and !tags.empty?
+            bok['tags'] = MU.structToHash(tags)
+          end
+
+          if cloud_desc.vpc_options
+            bok['vpc'] = MU::Config::Ref.get(
+              id: cloud_desc.vpc_options.vpc_id,
+              cloud: "AWS",
+              credentials: @credentials,
+              type: "vpcs",
+              region: @config['region'],
+              subnets: cloud_desc.vpc_options.subnet_ids.map { |s| { "subnet_id" => s } }
+            )
+            if cloud_desc.vpc_options.security_group_ids and
+               !cloud_desc.vpc_options.security_group_ids.empty?
+              bok['add_firewall_rules'] = cloud_desc.vpc_options.security_group_ids.map { |sg|
+                MU::Config::Ref.get(
+                  id: sg,
+                  cloud: "AWS",
+                  credentials: @credentials,
+                  region: @config['region'],
+                  type: "firewall_rules",
+                )
+              }
+            end
+          end
+
+          if cloud_desc.log_publishing_options
+            # XXX this is primitive... there are multiple other log types now,
+            # and this should be a Ref blob, not a flat string
+            cloud_desc.log_publishing_options.each_pair { |type, whither|
+              if type == "SEARCH_SLOW_LOGS"
+                bok['slow_logs'] = whither.cloud_watch_logs_log_group_arn
+              end
+            }
+          end
+
+          bok
+        end
+
         # Cloud-specific configuration properties.
         # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
@@ -200,7 +291,7 @@ module MU
           versions = begin
             MU::Cloud::AWS.elasticsearch.list_elasticsearch_versions.elasticsearch_versions
           rescue MuError
-            ["7.1", "6.8", "6.7", "6.5", "6.4", "6.3", "6.2", "6.0", "5.6"]
+            ["7.4", "7.1", "6.8", "6.7", "6.5", "6.4", "6.3", "6.2", "6.0", "5.6"]
           end
           instance_types = begin
             MU::Cloud::AWS.elasticsearch.list_elasticsearch_instance_types(
@@ -215,6 +306,8 @@ module MU
             ).elasticsearch_instance_types
           end
 
+          polschema = MU::Config::Role.schema["properties"]["policies"]
+          polschema.deep_merge!(MU::Cloud.resourceClass("AWS", "Role").condition_schema)
 
           schema = {
             "name" => {
@@ -236,9 +329,10 @@ module MU
               "default" => 0,
               "description" => "Separate, dedicated master node(s), over and above the search instances specified in instance_count."
             },
+            "policies" => polschema,
             "access_policies" => {
               "type" => "object",
-              "description" => "An IAM policy document for access to ElasticSearch. Our parser expects this to be defined inline like the rest of your YAML/JSON Basket of Kittens, not as raw JSON. For guidance on ElasticSearch IAM capabilities, see: https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-ac.html"
+              "description" => "An IAM policy document for access to ElasticSearch (see {policies} for setting complex access policies with runtime dependencies). Our parser expects this to be defined inline like the rest of your YAML/JSON Basket of Kittens, not as raw JSON. For guidance on ElasticSearch IAM capabilities, see: https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-ac.html"
             },
             "master_instance_type" => {
               "type" => "string",
@@ -246,7 +340,7 @@ module MU
             },
             "ebs_type" => {
               "type" => "string",
-              "default" => "standard",
+              "default" => "gp2",
               "description" => "Type of EBS storage to use for cluster nodes. If 'none' is specified, EBS storage will not be used, but this is only valid for certain instance types.",
               "enum" => ["standard", "gp2", "io1", "none"]
             },
@@ -509,9 +603,51 @@ module MU
             params[:snapshot_options][:automated_snapshot_start_hour] = @config['snapshot_hour']
           end
 
-          if @config['access_policies']
-            # TODO check against ext.access_policies.options
-            params[:access_policies] = JSON.generate(@config['access_policies'])
+          if ext
+            # Despite being called access_policies, this parameter actually
+            # only accepts one policy. So, we'll munge everything we have
+            # together into one policy with multiple Statements.
+            policy = nil
+            # TODO check against ext.access_policy.options
+
+            if @config['access_policies']
+              policy = @config['access_policies']
+              # ensure the "Statement" key is cased in a predictable way
+              statement_key = nil
+              policy.each_pair { |k, v|
+                if k.downcase == "statement" and k != "Statement"
+                  statement_key = k
+                  break
+                end
+              }
+              if statement_key
+                policy["Statement"] = policy.delete(statement_key)
+              end
+              if !policy["Statement"].is_a?(Array)
+                policy["Statement"] = [policy["Statement"]]
+              end
+            end
+
+            if @config['policies']
+              @config['policies'].each { |p|
+                p['targets'].each { |t|
+                  if t['path']
+                    t['path'].gsub!(/#SELF/, @mu_name.downcase)
+                  end
+                }
+                parsed = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument([p], deploy_obj: @deploy, bucket_style: true).first.values.first
+
+                if policy and policy["Statement"]
+                  policy["Statement"].concat(parsed["Statement"])
+                else
+                  policy = parsed
+                end
+              }
+            end
+
+            if policy
+              params[:access_policies] = JSON.generate(policy)
+            end
           end
 
           if @config['slow_logs']
@@ -677,7 +813,7 @@ module MU
             raise MU::MuError, "Can't tag ElasticSearch domain, cloud descriptor came back without an ARN"
           end
 
-          MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).add_tags(
+          MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @credentials).add_tags(
             arn: domain.arn,
             tag_list: tags
           )