cloud-mu 3.5.0 → 3.6.3

data/modules/mu/providers/aws/loadbalancer.rb

@@ -20,6 +20,7 @@ module MU
 
         @lb = nil
         attr_reader :targetgroups
+        attr_reader :is_lambda
 
         # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
         # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
@@ -161,7 +162,7 @@ module MU
           parent_thread_id = Thread.current.object_id
           generic_mu_dns = nil
           dnsthread = Thread.new {
-            if !MU::Cloud::AWS.isGovCloud?
+            if MU::Cloud::AWS.hosted? and !MU::Cloud::AWS.isGovCloud?
               MU.dupGlobals(parent_thread_id)
               generic_mu_dns = MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: @mu_name, target: "#{lb.dns_name}.", cloudclass: MU::Cloud::LoadBalancer, sync_wait: @config['dns_sync_wait'])
             end
@@ -181,6 +182,7 @@ module MU
           end
 
           @targetgroups = {}
+          @is_lambda = false
           if !@config['healthcheck'].nil? and @config['classic']
             MU.log "Configuring custom health check for ELB #{@mu_name}", details: @config['healthcheck']
             MU::Cloud::AWS.elb(region: @region, credentials: @credentials).configure_health_check(
@@ -203,10 +205,13 @@ module MU
                   :protocol => tg['proto'],
                   :vpc_id => @vpc.cloud_id,
                   :port => tg['port'],
-                  :target_type  => 'instance'
+                  :target_type  => tg['target_type'] || "instance"
                 }
-                if tg['target_type'] && tg['target_type'] != 'instance'
-                  tg_descriptor[:target_type] = tg['target_type']
+                if tg['target_type'] == "lambda"
+                  @is_lambda = true
+                  tg_descriptor.delete(:protocol)
+                  tg_descriptor.delete(:port)
+                  tg_descriptor.delete(:vpc_id)
                 end
                 if tg['httpcode']
                   tg_descriptor[:matcher] = {
@@ -426,17 +431,19 @@ module MU
                 timeout = 0
                 MU.log "Disabling connection draining on #{lb.dns_name}"
               end
-              @targetgroups.values.each { |tg|
-                MU::Cloud::AWS.elb2(region: @region, credentials: @credentials).modify_target_group_attributes(
-                  target_group_arn: tg.target_group_arn,
-                  attributes: [
-                    {
-                      key: "deregistration_delay.timeout_seconds",
-                      value: timeout.to_s
-                    }
-                  ]
-                )
-              }
+              if !@is_lambda
+                @targetgroups.values.each { |tg|
+                  MU::Cloud::AWS.elb2(region: @region, credentials: @credentials).modify_target_group_attributes(
+                    target_group_arn: tg.target_group_arn,
+                    attributes: [
+                      {
+                        key: "deregistration_delay.timeout_seconds",
+                        value: timeout.to_s
+                      }
+                    ]
+                  )
+                }
+              end
             end
           end
 
@@ -569,6 +576,11 @@ module MU
           notify
         end
 
+        # Called automatically by {MU::Deploy#createResources}
+        def groom
+          MU.log "LoadBalancer #{@config['name']} is at #{cloud_desc.dns_name}", MU::SUMMARY
+        end
+
         # Canonical Amazon Resource Number for this resource
         # @return [String]
         def arn
@@ -603,7 +615,6 @@ module MU
                   @targetgroups[tg_name] = MU::Cloud::AWS.elb2(region: @region, credentials: @credentials).describe_target_groups(target_group_arns: [tg_arn]).target_groups.first
                 }
               else
-                pp @config['targetgroups']
                 MU::Cloud::AWS.elb2(region: @region, credentials: @credentials).describe_target_groups(load_balancer_arn: @cloud_desc_cache.load_balancer_arn).target_groups.each { |tg|
                   tg_name = tg.target_group_name
                   if @config['targetgroups']
@@ -644,31 +655,31 @@ module MU
 
         # Register a Server node with an existing LoadBalancer.
         #
-        # @param instance_id [String] A node to register.
-        # @param targetgroups [Array<String>] The target group(s) of which this node should be made a member. Not applicable to classic LoadBalancers. If not supplied, the node will be registered to all available target groups on this LoadBalancer.
-        def registerNode(instance_id, targetgroups: nil)
+        # @param id [String] A node or function to register.
+        # @param backends [Array<String>] The target group(s) of which this node should be made a member. Not applicable to classic LoadBalancers. If not supplied, the node will be registered to all available target groups on this LoadBalancer.
+        def registerTarget(id, backends: nil)
           if @config['classic'] or !@config.has_key?("classic")
-            MU.log "Registering #{instance_id} to ELB #{@cloud_id}"
+            MU.log "Registering #{id} to ELB #{@cloud_id}"
             MU::Cloud::AWS.elb(region: @region, credentials: @credentials).register_instances_with_load_balancer(
               load_balancer_name: @cloud_id,
               instances: [
-                {instance_id: instance_id}
+                {instance_id: id}
               ]
             )
           else
-            if targetgroups.nil? or !targetgroups.is_a?(Array) or targetgroups.size == 0
+            if backends.nil? or !backends.is_a?(Array) or backends.size == 0
               if @targetgroups.nil?
                 cloud_desc
                 return if @targetgroups.nil?
               end
-              targetgroups = @targetgroups.keys
+              backends = @targetgroups.keys
             end
-            targetgroups.each { |tg|
-              MU.log "Registering #{instance_id} to Target Group #{tg}"
+            backends.each { |tg|
+              MU.log "Registering #{id} to Target Group #{tg}"
               MU::Cloud::AWS.elb2(region: @region, credentials: @credentials).register_targets(
                 target_group_arn: @targetgroups[tg].target_group_arn,
                 targets: [
-                  {id: instance_id}
+                  {id: id}
                 ]
               )
             }
@@ -753,7 +764,7 @@ module MU
                 end
               end
               if matched
-                if !MU::Cloud::AWS.isGovCloud?
+                if MU::Cloud::AWS.hosted? and !MU::Cloud::AWS.isGovCloud?
                   MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: lb.load_balancer_name, target: lb.dns_name, cloudclass: MU::Cloud::LoadBalancer, delete: true) if !noop
                 end
                 if classic
@@ -834,7 +845,7 @@ module MU
                     "type" => "string",
                     "enum" => ["HTTP", "HTTPS", "TCP", "SSL"],
                   },
-                  "target_type " => {
+                  "target_type" => {
                     "type" => "string",
                     "enum" => ["instance", "ip", "lambda"],
                   }

data/modules/mu/providers/aws/log.rb

@@ -244,7 +244,8 @@ module MU
         def self.find(**args)
           found = {}
           if !args[:cloud_id].nil? and !args[:cloud_id].match(/^arn:/i)
-            found[args[:cloud_id]] = MU::Cloud::AWS::Log.getLogGroupByName(args[:cloud_id], region: args[:region], credentials: args[:credentials])
+            exists = MU::Cloud::AWS::Log.getLogGroupByName(args[:cloud_id], region: args[:region], credentials: args[:credentials])
+            found[args[:cloud_id]] = exists if exists
           else
             next_token = nil
             begin

data/modules/mu/providers/aws/role.rb

@@ -184,6 +184,14 @@ module MU
                 desc
               end
 
+            rescue Aws::IAM::Errors::AccessDenied => e
+              if e.message =~ /Cannot create versions for policies outside your own account/
+                MU.log "Deleting and recreating cross-account policy #{policy_name}", MU::NOTICE
+                purgePolicy(arn, credentials)
+                retry
+              else
+                raise e
+              end
             rescue Aws::IAM::Errors::NoSuchEntity
               MU.log "Creating IAM policy #{policy_name}", details: policy.values.first
               desc = MU::Cloud::AWS.iam(credentials: credentials).create_policy(
@@ -328,7 +336,7 @@ end
                 version_id: p.default_version_id
               ).policy_version
 
-              doc = JSON.parse CGI.unescape_www_form_component old.document
+              doc = JSON.parse URI.decode_www_form_component old.document
               need_update = false
 
               doc["Statement"].each { |s|
@@ -539,7 +547,6 @@ end
           found = {}
 
           if args[:cloud_id]
-
             begin
               # managed policies get fetched by ARN, roles by plain name. Ok!
               if args[:cloud_id].match(/^arn:.*?:policy\//)
@@ -560,7 +567,6 @@ end
               end
             rescue ::Aws::IAM::Errors::NoSuchEntity
             end
-
           else
             resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_roles
             resp.roles.each { |role|
@@ -1118,7 +1124,7 @@ end
           if role['policies']
             role['policies'].each { |policy|
               policy['targets'].each { |target|
-                if target['type']
+                if target['type'] and _configurator.haveLitterMate?(target['identifier'], target['type'])
                   MU::Config.addDependency(role, target['identifier'], target['type'], my_phase: "groom")
                 end
               }
@@ -1135,7 +1141,7 @@ end
         # @param policies [Array<Hash>]: One or more policy chunks
         # @param deploy_obj [MU::MommaCat]: Deployment object to use when looking up sibling Mu resources
         # @return [Array<Hash>]
-        def self.genPolicyDocument(policies, deploy_obj: nil, bucket_style: false, version: "2012-10-17", doc_id: nil)
+        def self.genPolicyDocument(policies, deploy_obj: nil, bucket_style: false, version: "2012-10-17", doc_id: nil, credentials: nil)
           if policies
             name = nil
             doc = {
@@ -1211,7 +1217,19 @@ end
                       type: target["type"]
                     )
                     if sibling
-                      id = sibling.cloudobj.arn
+                      sibling = sibling.cloudobj
+                    else
+                      found = MU::MommaCat.findStray(
+                        "AWS",
+                        target["type"],
+                        cloud_id: target["identifier"],
+                        credentials: credentials,
+                        dummy_ok: true
+                      )
+                      sibling = found.first
+                    end
+                    if sibling
+                      id = sibling.arn
                       id.sub!(/:([^:]+)$/, ":"+'\1'+target["path"]) if target["path"]
                       statement["Resource"] << id
                       if id.match(/:log-group:/)
@@ -1269,7 +1287,7 @@ end
         # Convert entries from the cloud-neutral @config['policies'] list into
         # AWS syntax.
         def convert_policies_to_iam
-          MU::Cloud::AWS::Role.genPolicyDocument(@config['policies'], deploy_obj: @deploy)
+          MU::Cloud::AWS::Role.genPolicyDocument(@config['policies'], deploy_obj: @deploy, credentials: @credentials)
         end
 
         def get_tag_params(strip_std = false)

data/modules/mu/providers/aws/server.rb

@@ -276,6 +276,11 @@ module MU
           :key_name => @deploy.ssh_key_name,
           :instance_type => @config["size"],
           :disable_api_termination => true,
+          :metadata_options => {
+            :http_tokens => "optional",
+            :http_endpoint => "enabled",
+            :instance_metadata_tags => "enabled"
+          },
           :min_count => 1,
           :max_count => 1
         }
@@ -348,7 +353,10 @@ module MU
             raise MuError.new e.message, details: mysubnet_ids
           end
           instance_descriptor[:subnet_id] = (mysubnet_ids - bad_subnets).sample
-          MU.log "One or more subnets does not support this instance type, attempting with #{instance_descriptor[:subnet_id]} instead", MU::WARN, details: bad_subnets
+          if instance_descriptor[:subnet_id].nil?
+            raise MuError.new "Specified subnet#{bad_subnets.size > 1 ? "s do" : " does"} not support instance type #{instance_descriptor[:instance_type]}", details: bad_subnets
+          end
+          MU.log "One or more subnets does not support instance type #{instance_descriptor[:instance_type]}, attempting with #{instance_descriptor[:subnet_id]} instead", MU::WARN, details: bad_subnets
           retry
         rescue Aws::EC2::Errors::InvalidRequest => e
           MU.log e.message, MU::ERR, details: instance_descriptor
@@ -587,7 +595,6 @@ module MU
           searches << {
             filters: [
               base_filter,
-              {name: ip_type, values: [ip]},
               {name: "tag:#{args[:tag_key]}", values: [args[:tag_value]]},
             ]
           }
@@ -855,7 +862,7 @@ module MU
               raise MuError, "#{@mu_name} is configured to use LoadBalancers, but none have been loaded by dependencies()"
             end
             @loadbalancers.each { |lb|
-              lb.registerNode(@cloud_id)
+              lb.registerTarget(@cloud_id)
             }
           end
           MU.log %Q{Server #{@config['name']} private IP is #{@deploydata["private_ip_address"]}#{@deploydata["public_ip_address"] ? ", public IP is "+@deploydata["public_ip_address"] : ""}}, MU::SUMMARY
@@ -893,7 +900,8 @@ module MU
             end
           rescue MU::Groomer::RunError => e
             raise e if !@config['create_image'].nil? and !@config['image_created']
-            MU.log "Proceeding after failed initial Groomer run, but #{@mu_name} may not behave as expected!", MU::WARN, details: e.message
+            MU.log "Proceeding after failed initial Groomer run, but #{@mu_name} may not behave as expected!", MU::WARN, details: e.inspect
+            pp e.backtrace
           rescue StandardError => e
             raise e if !@config['create_image'].nil? and !@config['image_created']
             MU.log "Caught #{e.inspect} on #{@mu_name} in an unexpected place (after @groomer.run on Full Initial Run)", MU::ERR
@@ -1250,7 +1258,7 @@ module MU
             resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_addresses
           end
           resp.addresses.each { |address|
-            return address if (address.network_interface_id.nil? or address.network_interface_id.empty?) or !@eips_used.include?(address.public_ip)
+            return address if (address.network_interface_id.nil? or address.network_interface_id.empty?) and !@eips_used.include?(address.public_ip)
           }
           if !ip.nil?
             mode = classic ? "EC2 Classic" : "VPC"
@@ -1295,7 +1303,10 @@ module MU
         # @param size [String]: Size (in gb) of the new volume
         # @param type [String]: Cloud storage type of the volume, if applicable
         # @param delete_on_termination [Boolean]: Value of delete_on_termination flag to set
-        def addVolume(dev, size, type: "gp2", delete_on_termination: false)
+        def addVolume(dev: nil, size: 0, type: "gp3", delete_on_termination: false)
+          if dev.nil? or size == 0
+            raise MuError, "Must specify a device name and a size for addVolume"
+          end
 
           if setDeleteOntermination(dev, delete_on_termination)
             MU.log "A volume #{dev} already attached to #{self}, skipping", MU::NOTICE
@@ -1401,7 +1412,7 @@ module MU
               if @eips_used.include?(ip)
                 is_free = false
                 resp.addresses.each { |address|
-                  if address.public_ip == ip and (address.instance_id.nil? and address.network_interface_id.nil?) or address.instance_id == instance_id
+                  if address.public_ip == ip and (address.instance_id.nil? and address.association.nil?) or address.instance_id == instance_id
                     @eips_used.delete(ip)
                     is_free = true
                   end
@@ -1434,8 +1445,11 @@ module MU
                 allocation_ids: [elastic_ip.allocation_id]
               )
               first_addr = resp.addresses.first
-              if first_addr and first_addr.instance_id != instance_id
-                raise MuError, "Tried to associate #{elastic_ip.public_ip} with #{instance_id}, but it's already associated with #{first_addr.instance_id}!"
+              if first_addr and !first_addr.association_id.nil? and first_addr.instance_id != instance_id
+                ifaces = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_network_interfaces(
+                  filters: [{name: "association.allocation-id", values: [elastic_ip.allocation_id]}]
+                ).data.network_interfaces
+                raise MuError.new "Tried to associate #{elastic_ip.public_ip} with #{instance_id}, but it's already associated with #{first_addr.instance_id}!", details: ifaces
               end
             end
           }
@@ -1747,6 +1761,16 @@ module MU
                   "key_is" => "platform",
                   "value_is" => "amazon",
                   "set" => "ec2-user"
+                },
+                {
+                  "key_is" => "platform",
+                  "value_is" => "amazon2",
+                  "set" => "ec2-user"
+                },
+                {
+                  "key_is" => "platform",
+                  "value_is" => "amazon2023",
+                  "set" => "ec2-user"
                 }
               ]
             }
@@ -2098,9 +2122,9 @@ module MU
               end
             end
 
-            win_admin_password ||= MU.generateWindowsPassword
-            ec2config_password ||= MU.generateWindowsPassword
-            sshd_password ||= MU.generateWindowsPassword
+            win_admin_password ||= MU.generatePassword
+            ec2config_password ||= MU.generatePassword
+            sshd_password ||= MU.generatePassword
 
             # We're creating the vault here so when we run
             # MU::Cloud::Server.initialSSHTasks and we need to set the Windows