cloud-mu 3.5.0 → 3.6.3

data/modules/mu/providers/aws/database.rb

@@ -148,8 +148,11 @@ module MU
             end
 
           @mu_name.gsub(/(--|-$)/i, "").gsub(/(_)/, "-").gsub!(/^[^a-z]/i, "")
+          @cloud_id ||= @mu_name # XXX ??? MMMFFF
+
           if @config.has_key?("parameter_group_family")
             @config["parameter_group_name"] ||= @mu_name
+            @config["parameter_group_name"].downcase!
           end
 
           if args[:from_cloud_desc] and args[:from_cloud_desc].is_a?(Aws::RDS::Types::DBCluster)
@@ -510,8 +513,11 @@ dependencies
 
           if create
             MU.log "Creating a #{cluster ? "cluster" : "database" } parameter group #{@config["parameter_group_name"]}"
-
-            MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send(cluster ? :create_db_cluster_parameter_group : :create_db_parameter_group, params)
+            begin
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send(cluster ? :create_db_cluster_parameter_group : :create_db_parameter_group, params)
+            rescue Aws::RDS::Errors::DBParameterGroupAlreadyExists => e
+              MU.log e.message, MU::WARN
+            end
           end
 
 
@@ -546,11 +552,58 @@ dependencies
           end
         end
 
+        # If we've declared special option group options, ensure that we have
+        # an option group of the same name as our database instance with the
+        # correct settings.
+        # @return [String]: The name of the option group our database should use
+        def manageOptionGroup
+          if @config['option_group_options'] and !@config['option_group_options'].empty?
+            og = begin
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).describe_option_groups(
+                option_group_name: @mu_name
+              ).option_groups_list.first
+            rescue Aws::RDS::Errors::OptionGroupNotFoundFault
+              MU.log "Creating Option Group #{@mu_name} for #{@config['engine']} #{@config['major_version']} database #{@mu_name}"
+
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).create_option_group(
+                engine_name: @config['engine'],
+                major_engine_version: @config['major_version'],
+                option_group_name: @mu_name,
+                option_group_description: @deploy_id
+              ).option_group
+            end
+            need = []
+            ext_options = og.options.map { |o| o.option_name }
+            @config['option_group_options'].each { |opt|
+              if !ext_options.include?(opt)
+                need << { option_name: opt }
+              end
+            }
+
+            if !need.empty?
+              MU.log "Modifying Option Group #{@mu_name}", MU::NOTICE, need
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_option_group(
+                option_group_name: @mu_name,
+                apply_immediately: true,
+                options_to_include: need
+              )
+            end
+
+            @mu_name
+          # Otherwise if we've declared a non-default option group, ensure that
+          # we're using it.
+          elsif @config['option_group']
+            @config['option_group']
+          else
+            nil
+          end
+        end
+
         # Called automatically by {MU::Deploy#createResources}
         def groom
           cloud_desc(use_cache: false)
           manageSubnetGroup if @vpc
-          manageDbParameterGroup(@config["create_cluster"], create: false)
+          manageDbParameterGroup(@config["create_cluster"])
 
           noun = @config['create_cluster'] ? "cluster" : "instance"
 
@@ -575,6 +628,55 @@ dependencies
             mods[:vpc_security_group_ids] = @config["vpc_security_group_ids"]
           end
 
+          option_group_name = manageOptionGroup
+          if cloud_desc.option_group_memberships.first.option_group_name != option_group_name
+            mods[:option_group_name] = option_group_name
+          end
+
+          # If we've declared special option group options, ensure that we have
+          # an option group of the same name as our database instance with the
+          # correct settings.
+          if @config['option_group_options'] and !@config['option_group_options'].empty?
+            og = begin
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).describe_option_groups(
+                option_group_name: @mu_name
+              ).option_groups_list.first
+            rescue Aws::RDS::Errors::OptionGroupNotFoundFault
+              MU.log "Creating Option Group #{@mu_name} for #{@config['engine']} #{@config['major_version']} database #{@mu_name}"
+
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).create_option_group(
+                engine_name: @config['engine'],
+                major_engine_version: @config['major_version'],
+                option_group_name: @mu_name,
+                option_group_description: @deploy_id
+              ).option_group
+            end
+            need = []
+            ext_options = og.options.map { |o| o.option_name }
+            @config['option_group_options'].each { |opt|
+              if !ext_options.include?(opt)
+                need << { option_name: opt }
+              end
+            }
+
+            if !need.empty?
+              MU.log "Modifying Option Group #{@mu_name}", MU::NOTICE, need
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).modify_option_group(
+                option_group_name: @mu_name,
+                apply_immediately: true,
+                options_to_include: need
+              )
+            end
+
+            if cloud_desc.option_group_memberships.first.option_group_name != @mu_name
+              mods[:option_group_name] = @mu_name
+            end
+
+          # Otherwise if we've declared a non-default option group, ensure that
+          # we're using it.
+          elsif @config['option_group'] and cloud_desc.option_group_memberships.first.option_group_name != @config['option_group']
+            mods[:option_group_name] = @config['option_group']
+          end
 
           if @config['cloudwatch_logs'] and cloud_desc.enabled_cloudwatch_logs_exports.sort != @config['cloudwatch_logs'].sort
             mods[:cloudwatch_logs_export_configuration] = {
@@ -614,7 +716,9 @@ dependencies
             mods[:apply_immediately] = true
             mods[:allow_major_version_upgrade] = true
             wait_until_available
-            MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send("modify_db_#{noun}".to_sym, mods)
+            MU.retrier([Aws::RDS::Errors::InvalidDBInstanceState, Aws::RDS::Errors::InvalidParameterValue], wait: 60, max: 15) {
+              MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send("modify_db_#{noun}".to_sym, mods)
+            }
             wait_until_available
           end
 
@@ -812,7 +916,7 @@ dependencies
 
           threads = threaded_resource_purge(:describe_db_subnet_groups, :db_subnet_groups, :db_subnet_group_name, "subgrp", region, credentials, ignoremaster, known: flags['known'], deploy_id: deploy_id) { |id|
             MU.log "Deleting RDS subnet group #{id}"
-            MU.retrier([Aws::RDS::Errors::InvalidDBSubnetGroupStateFault], wait: 30, max: 5, ignoreme: [Aws::RDS::Errors::DBSubnetGroupNotFoundFault]) {
+            MU.retrier([Aws::RDS::Errors::InvalidDBSubnetGroupStateFault], wait: 60, max: 10, ignoreme: [Aws::RDS::Errors::DBSubnetGroupNotFoundFault]) {
               MU::Cloud::AWS.rds(region: region, credentials: credentials).delete_db_subnet_group(db_subnet_group_name: id) if !noop
             }
           }
@@ -820,7 +924,7 @@ dependencies
           ["db", "db_cluster"].each { |type|
             threads.concat threaded_resource_purge("describe_#{type}_parameter_groups".to_sym, "#{type}_parameter_groups".to_sym, "#{type}_parameter_group_name".to_sym, (type == "db" ? "pg" : "cluster-pg"), region, credentials, ignoremaster, known: flags['known'], deploy_id: deploy_id) { |id|
               MU.log "Deleting RDS #{type} parameter group #{id}"
-              MU.retrier([Aws::RDS::Errors::InvalidDBParameterGroupState], wait: 30, max: 5, ignoreme: [Aws::RDS::Errors::DBParameterGroupNotFound]) {
+              MU.retrier([Aws::RDS::Errors::InvalidDBParameterGroupState], wait: 60, max: 10, ignoreme: [Aws::RDS::Errors::DBParameterGroupNotFound]) {
                 MU::Cloud::AWS.rds(region: region, credentials: credentials).send("delete_#{type}_parameter_group", { "#{type}_parameter_group_name".to_sym => id }) if !noop
               }
             }
@@ -862,7 +966,6 @@ dependencies
             }
           }
 
-
           schema = {
             "db_parameter_group_parameters" => rds_parameters_primitive,
             "cluster_parameter_group_parameters" => rds_parameters_primitive,
@@ -881,8 +984,21 @@ dependencies
               "type" => "string",
               "default" => "gp2"
             },
+            "option_group" => {
+              "description" => "The name of a non-default RDS option group to use with this instance/cluster (this will be ignored if +option_group_options+ is declared).",
+              "type" => "string"
+            },
+            "option_group_options" => {
+              "description" => "One or more option group options to enable, with default settings. If specified, +option_group+ will be overridden.",
+              "type" => "array",
+              "items" => {
+                "type" => "string",
+                "description" => "Option group identifiers supported by the relevant database engine, such as +MARIADB_AUDIT_PLUGIN+ for MariaDB/MySQL. See https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_WorkingWithOptionGroups.html for current options."
+              }
+            },
             "cloudwatch_logs" => {
               "type" => "array",
+              "description" => "Enable output of logs to CloudWatch",
               "items" => {
                 "type" => "string",
                 "enum" => ["audit", "error", "general", "slowquery", "profiler", "postgresql", "alert", "listener", "trace", "upgrade", "agent"]
@@ -950,10 +1066,13 @@ dependencies
                 "versions" => [],
                 "families" => [],
                 "features" => {},
+                "options" => {},
                 "raw" => {}
               }
               engines[version.engine]['versions'] << version.engine_version
               engines[version.engine]['families'] << version.db_parameter_group_family
+              engines[version.engine]['options'][version.major_engine_version] ||= MU::Cloud::AWS.rds().describe_option_group_options(engine_name: version.engine, major_engine_version: version.major_engine_version).option_group_options.map { |o| o.name }
+
               engines[version.engine]['raw'][version.engine_version] = version
               [:supports_read_replica, :supports_log_exports_to_cloudwatch_logs].each { |feature|
                 if version.respond_to?(feature) and version.send(feature) == true
@@ -976,7 +1095,7 @@ dependencies
           @@engine_cache[credentials][region] = engines
           return engine ? @@engine_cache[credentials][region][engine] : @@engine_cache[credentials][region]
         end
-        private_class_method :get_supported_engines
+#        private_class_method :get_supported_engines
 
         # Make sure any source database/cluster/snapshot we've asked for exists
         # and is valid.
@@ -1039,6 +1158,28 @@ dependencies
         end
         private_class_method :validate_master_password
 
+        def self.valid_option_group_options?(db)
+          return true if !db['option_group_options'] or db['option_group_options'].empty?
+
+          engine = get_supported_engines(db['region'], db['credentials'], engine: db['engine'])
+          if engine.nil? or !engine['options'] or !engine['raw'][db['engine_version']]
+            return true # we can't be sure, so let the API sort it out later
+          end
+          major_version = engine['raw'][db['engine_version']].major_engine_version
+          return true if !engine['options'][major_version]
+
+          ok = true
+          db['option_group_options'].each { |opt|
+            if !engine['options'][major_version].include?(opt)
+              MU.log "#{opt} not a valid Option Group entry for #{db['engine']} #{major_version}", MU::ERR, details: engine['options'][major_version]
+              ok = false
+            end
+          }
+
+          ok
+        end
+        private_class_method :valid_option_group_options?
+
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::databases}, bare and unvalidated.
         # @param db [Hash]: The resource to process and validate
         # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a ember
@@ -1054,6 +1195,17 @@ dependencies
 
           ok = false if !valid_cloudwatch_logs?(db)
 
+          ok = false if !valid_option_group_options?(db)
+
+          engine = self.get_supported_engines(
+            db['region'],
+            db['credentials'],
+            engine: db['engine']
+          )
+          if engine and engine['raw'] and engine['raw'][db['engine_version']]
+            db['major_version'] = engine['raw'][db['engine_version']].major_engine_version
+          end
+
           db["license_model"] ||=
             if ["postgres", "postgresql", "aurora-postgresql"].include?(db["engine"])
               "postgresql-license"
@@ -1346,6 +1498,7 @@ dependencies
 
           MU.retrier([Aws::RDS::Errors::InvalidParameterValue, Aws::RDS::Errors::DBSubnetGroupNotFoundFault], max: 10, wait: 15) {
             if %w{existing_snapshot new_snapshot}.include?(@config["creation_style"])
+              params[:option_group_name] = manageOptionGroup
               clean_parent_opts.call
               MU.log "Creating database #{noun} #{@cloud_id} from snapshot #{@config["snapshot_id"]}"
               MU::Cloud::AWS.rds(region: @region, credentials: @credentials).send("restore_db_#{noun}_from_#{noun == "instance" ? "db_" : ""}snapshot".to_sym, params)

data/modules/mu/providers/aws/dnszone.rb

@@ -588,12 +588,17 @@ module MU
             ip = nil
 
             records = []
-            lookup = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(
-              hosted_zone_id: mu_zone.id,
-              start_record_name: "#{dns_name}.platform-mu",
-              start_record_type: record_type,
-              max_items: 1
-            ).resource_record_sets
+            begin
+              lookup = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(
+                hosted_zone_id: mu_zone.id,
+                start_record_name: "#{dns_name}.platform-mu",
+                start_record_type: record_type,
+                max_items: 1
+              ).resource_record_sets
+            rescue Aws::Route53::Errors::InvalidInput => e
+              MU.log "Failed to look up record during #{delete ? "delete" : "add"}: "+e.message, MU::ERR, details: { "hosted_zone_id" => mu_zone.id, "start_record_name" => "#{dns_name}.platform-mu", "start_record_type" => record_type }
+              return nil
+            end
 
             lookup.each { |record|
               if record.name.match(/^#{dns_name}\.platform-mu/i) and record.type == record_type

data/modules/mu/providers/aws/endpoint.rb

@@ -13,14 +13,54 @@ module MU
 
         # Called automatically by {MU::Deploy#createResources}
         def create
-          resp = MU::Cloud::AWS.apig(region: @region, credentials: @credentials).create_rest_api(
+          desc = {
             name: @mu_name,
             description: @deploy.deploy_id,
             endpoint_configuration: {
-              types: ["REGIONAL"] # XXX expose in BoK ["REGIONAL", "EDGE", "PRIVATE"]
+              types: [@config['endpoint_type']]
             },
             tags: @tags
-          )
+          }
+
+# XXX NLB? what the fork?
+#          if @vpc
+#            MU::Cloud::AWS.apig(region: @region, credentials: @credentials).create_vpc_link(
+#              name: @mu_name,
+#              target_arns: [Required] The ARN of the network load balancer of the VPC targeted by the VPC link. The network load balancer must be owned by the same AWS account of the API owner.
+#              tags: @tags
+#            )  
+#          end
+
+# XXX this is incomplete; need to cover non-VPC case, IP ranges, and fall back on account number if all else fails
+          if @config['endpoint_type'] == "PRIVATE"
+            desc[:policy] = JSON.generate(
+              {
+                "Version" => "2012-10-17",
+                "Statement" => [
+                  {
+                    "Effect" => "Deny",
+                    "Principal" => "*",
+                    "Action" => "execute-api:Invoke",
+                    "Resource" => "arn:aws:execute-api:#{@region}:#{MU::Cloud::AWS.credToAcct(@credentials)}:*/*/*/*",
+                    "Condition" => {
+                      "StringNotEquals" => {
+                        "aws =>sourceVpc": @vpc.cloud_id
+                      }
+                    }
+                  },
+                  {
+                    "Effect" => "Allow",
+                    "Principal" => "*",
+                    "Action" => "execute-api:Invoke",
+                    "Resource" => "arn:aws:execute-api:#{@region}:#{MU::Cloud::AWS.credToAcct(@credentials)}:*/*/*/*"
+                  }
+                ]
+              }
+            )
+          end
+
+          resp = MU::Cloud::AWS.apig(region: @region, credentials: @credentials).create_rest_api(desc)
+
           @cloud_id = resp.id
           generate_methods(false)
         end
@@ -629,6 +669,11 @@ MU::Cloud::AWS.apig(region: @region, credentials: @credentials).get_resource(
         def self.schema(_config)
           toplevel_required = []
           schema = {
+            "endpoint_type" => {
+              "default" => "REGIONAL",
+              "type" => "string",
+              "enum" => ["REGIONAL", "EDGE", "PRIVATE"]
+            },
             "domain_names" => {
               "type" => "array",
               "items" => {
@@ -791,7 +836,7 @@ MU::Cloud::AWS.apig(region: @region, credentials: @credentials).get_resource(
                       "type" => {
                         "type" => "string",
                         "description" => "A Mu resource type, for integrations with a sibling resource (e.g. a function), or the string +aws_generic+, which we can use in combination with +aws_generic_action+ to integrate with arbitrary AWS services.",
-                        "enum" => ["aws_generic"].concat(MU::Cloud.resource_types.values.map { |t| t[:cfg_plural] }.sort)
+                        "enum" => ["aws_generic", "mock"].concat(MU::Cloud.resource_types.values.map { |t| t[:cfg_plural] }.sort)
                       },
                       "aws_generic_action" => {
                         "type" => "string",
@@ -863,6 +908,23 @@ MU::Cloud::AWS.apig(region: @region, credentials: @credentials).get_resource(
           "arn:#{MU::Cloud::AWS.isGovCloud?(@region) ? "aws-us-gov" : "aws"}:execute-api:#{@region}:#{MU::Cloud::AWS.credToAcct(@credentials)}:#{@cloud_id}"
         end
 
+        # Go fish for the account-wide CloudWatch Logs role that grants APIG
+        # permissions to generate logs. This appears to have disappeared from
+        # the web console.
+        def self.findCloudWatchLogsRole(credentials = nil)
+          roles = MU::Cloud.resourceClass("AWS", "Role").find(credentials: credentials)
+          roles.each_pair { |id, r|
+            next if r.is_a?(Aws::IAM::Types::Policy)
+            attached_policies = MU::Cloud::AWS.iam(credentials: @credentials).list_attached_role_policies(
+              role_name: r.role_name
+            ).attached_policies
+            if attached_policies.size == 1 and attached_policies.first.policy_arn == "arn:aws:iam::aws:policy/service-role/AmazonAPIGatewayPushToCloudWatchLogs"
+              return r.arn
+            end
+          }
+          nil
+        end
+
 
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::endpoints}, bare and unvalidated.
         # @param endpoint [Hash]: The resource to process and validate
@@ -893,9 +955,22 @@ MU::Cloud::AWS.apig(region: @region, credentials: @credentials).get_resource(
 
           if endpoint['access_logs']
             resp = MU::Cloud::AWS.apig(credentials: endpoint['credentials'], region: endpoint['region']).get_account
+
             if !resp.cloudwatch_role_arn
-              MU.log "Endpoint '#{endpoint['name']}' is configured to use CloudWatch Logs, but the account-wide API Gateway log role is not configured", MU::ERR, details: "https://aws.amazon.com/premiumsupport/knowledge-center/api-gateway-cloudwatch-logs/"
-              ok = false
+              logs_role = findCloudWatchLogsRole
+              if logs_role
+                MU.log "Updating API Gateway account-wide to add log role #{logs_role}", MU::NOTICE
+                MU::Cloud::AWS.apig(credentials: endpoint['credentials'], region: endpoint['region']).update_account(
+                  patch_operations: [
+                    op: "replace",
+                    path: "/cloudwatchRoleArn",
+                    value: logs_role
+                  ]
+                )
+              else
+                MU.log "Endpoint '#{endpoint['name']}' is configured to use CloudWatch Logs, but the account-wide API Gateway log role is not configured", MU::ERR, details: "https://aws.amazon.com/premiumsupport/knowledge-center/api-gateway-cloudwatch-logs/"
+                ok = false
+              end
             else
               roles = MU::Cloud::AWS::Role.find(cloud_id: resp.cloudwatch_role_arn, credentials: endpoint['credentials'], region: endpoint['region'])
               if roles.empty?