cloud-mu 3.2.0 → 3.5.0

data/modules/mu/providers/aws/storage_pool.rb

@@ -29,7 +29,7 @@ module MU
         # @return [String]: The cloud provider's identifier for this storage pool.
         def create
           MU.log "Creating storage pool #{@mu_name}"
-          resp = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).create_file_system(
+          resp = MU::Cloud::AWS.efs(region: @region, credentials: @credentials).create_file_system(
             creation_token: @mu_name,
             performance_mode: @config['storage_type']
           )
@@ -37,7 +37,7 @@ module MU
           attempts = 0
           loop do
             MU.log "Waiting for #{@mu_name}: #{resp.file_system_id} to become available" if attempts % 5 == 0
-            storage_pool = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_file_systems(
+            storage_pool = MU::Cloud::AWS.efs(region: @region, credentials: @credentials).describe_file_systems(
               creation_token: @mu_name
             ).file_systems.first
             break if storage_pool.life_cycle_state == "available"
@@ -47,7 +47,7 @@ module MU
             raise MuError, "timed out waiting for #{resp.mount_target_id }" if attempts >= 20
           end
 
-          addStandardTags(cloud_id: resp.file_system_id, region: @config['region'], credentials: @config['credentials'])
+          addStandardTags(cloud_id: resp.file_system_id, region: @region, credentials: @credentials)
           @cloud_id = resp.file_system_id
 
           if @config['mount_points'] && !@config['mount_points'].empty?
@@ -82,8 +82,8 @@ module MU
                   ip_address: target['ip_address'],
                   subnet_id: target['vpc']['subnet_id'],
                   security_groups: sgs,
-                  credentials: @config['credentials'],
-                  region: @config['region']
+                  credentials: @credentials,
+                  region: @region
                 )
                 target['cloud_id'] = mount_target.mount_target_id
               }
@@ -100,7 +100,7 @@ module MU
         # Canonical Amazon Resource Number for this resource
         # @return [String]
         def arn
-          "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":elasticfilesystem:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":file-system/"+@cloud_id
+          "arn:"+(MU::Cloud::AWS.isGovCloud?(@region) ? "aws-us-gov" : "aws")+":elasticfilesystem:"+@region+":"+MU::Cloud::AWS.credToAcct(@credentials)+":file-system/"+@cloud_id
         end
 
         # Locate an existing storage pool and return an array containing matching AWS resource descriptors for those that match.
@@ -254,14 +254,14 @@ module MU
 
         # Register a description of this storage pool with this deployment's metadata.
         def notify
-          storage_pool = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_file_systems(
+          storage_pool = MU::Cloud::AWS.efs(region: @region, credentials: @credentials).describe_file_systems(
             creation_token: @mu_name
           ).file_systems.first
 
           targets = {}
 
           if @config['mount_points'] && !@config['mount_points'].empty?
-            mount_targets = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_mount_targets(
+            mount_targets = MU::Cloud::AWS.efs(region: @region, credentials: @credentials).describe_mount_targets(
               file_system_id: storage_pool.file_system_id
             ).mount_targets
 
@@ -274,6 +274,10 @@ module MU
               subnet_obj = mp_vpc.subnets.select { |s|
                 s.name == mp["vpc"]["subnet_name"] or s.cloud_id == mp["vpc"]["subnet_id"]
               }.first
+              if !subnet_obj
+                MU.log "Failed to find live subnet matching configured mount_point", MU::WARN, details: mp["vpc"]
+                next
+              end
               mount_target = nil
               mount_targets.each { |t|
                 subnet_cidr_obj = NetAddr::IPv4Net.parse(subnet_obj.ip_block)
@@ -283,6 +287,10 @@ module MU
                   break
                 end
               }
+              if !mount_target
+                MU.log "Failed to find live mount_target corresponding to configured mount_point", MU::WARN, details: mp
+                next
+              end
 
               targets[mp["name"]] = {
                 "owner_id" => mount_target.owner_id,
@@ -294,7 +302,7 @@ module MU
                 "availability_zone" => subnet.availability_zone,
                 "state" => mount_target.life_cycle_state,
                 "ip_address" => mount_target.ip_address,
-                "endpoint" => "#{subnet.availability_zone}.#{mount_target.file_system_id}.efs.#{@config['region']}.amazonaws.com",
+                "endpoint" => "#{subnet.availability_zone}.#{mount_target.file_system_id}.efs.#{@region}.amazonaws.com",
                 "network_interface_id" => mount_target.network_interface_id
               }
             }
@@ -333,7 +341,7 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region in which to operate
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           MU.log "AWS::StoragePool.cleanup: need to support flags['known']", MU::DEBUG, details: flags
 
           supported_regions = %w{us-west-2 us-east-1 eu-west-1}
@@ -358,7 +366,7 @@ module MU
                 found_muid = false
                 found_master = false
                 tags.each { |tag|
-                  found_muid = true if tag.key == "MU-ID" && tag.value == MU.deploy_id
+                  found_muid = true if tag.key == "MU-ID" && tag.value == deploy_id
                   found_master = true if tag.key == "MU-MASTER-IP" && tag.value == MU.mu_public_ip
                 }
                 next if !found_muid

data/modules/mu/providers/aws/user.rb

@@ -190,16 +190,16 @@ module MU
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
           MU.log "AWS::User.cleanup: need to support flags['known']", MU::DEBUG, details: flags
 
           # XXX this doesn't belong here; maybe under roles, maybe as its own stupid first-class resource
           resp = MU::Cloud::AWS.iam(credentials: credentials).list_policies(
-            path_prefix: "/"+MU.deploy_id+"/"
+            path_prefix: "/"+deploy_id+"/"
           )
           if resp and resp.policies
             resp.policies.each { |policy|
-              MU.log "Deleting policy /#{MU.deploy_id}/#{policy.policy_name}"
+              MU.log "Deleting policy /#{deploy_id}/#{policy.policy_name}"
               if !noop
                 attachments = begin
                   MU::Cloud::AWS.iam(credentials: credentials).list_entities_for_policy(
@@ -277,7 +277,7 @@ MU.log e.inspect, MU::ERR, details: policy
             has_ourdeploy = false
             has_ourmaster = false
             tags.each { |tag|
-              if tag.key == "MU-ID" and tag.value == MU.deploy_id
+              if tag.key == "MU-ID" and tag.value == deploy_id
                 has_ourdeploy = true
               elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip
                 has_ourmaster = true
@@ -430,7 +430,7 @@ MU.log e.inspect, MU::ERR, details: policy
           if resp and resp.policy_names and resp.policy_names.size > 0
             resp.policy_names.each { |pol_name|
               pol = MU::Cloud::AWS.iam(credentials: @credentials).get_user_policy(user_name: @cloud_id, policy_name: pol_name)
-              doc = JSON.parse(URI.decode(pol.policy_document))
+              doc = JSON.parse(CGI.unescape(pol.policy_document))
               bok["inline_policies"] = MU::Cloud.resourceClass("AWS", "Role").doc2MuPolicies(pol.policy_name, doc, bok["inline_policies"])
             }
           end

data/modules/mu/providers/aws/userdata/linux.erb

@@ -42,7 +42,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
 <% if !$mu.skipApplyUpdates %>
     set +e
     if [ ! -f /.mu-installer-ran-updates ];then
-      service ssh stop
+      echo "Applying package updates" > /etc/nologin
       apt-get --fix-missing -y upgrade 
       touch /.mu-installer-ran-updates
       if [ $? -eq 0 ]
@@ -58,7 +58,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
       else
         echo "FAILED PACKAGE UPDATE" >&2
       fi
-      service ssh start
+      rm -f /etc/nologin
     fi
 <% end %>
   elif [ -x /usr/bin/yum ];then
@@ -94,7 +94,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
 <% if !$mu.skipApplyUpdates %>
     set +e
     if [ ! -f /.mu-installer-ran-updates ];then
-      service sshd stop
+      echo "Applying package updates" > /etc/nologin
       kernel_update=`yum list updates | grep kernel`
       yum -y update
       touch /.mu-installer-ran-updates
@@ -108,7 +108,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
       else
         echo "FAILED PACKAGE UPDATE" >&2
       fi
-      service sshd start
+      rm -f /etc/nologin
     fi
 <% end %>
   fi
@@ -116,6 +116,7 @@ else
   /bin/logger "***** Unable to verify internet connectivity, skipping package updates from userdata"
   touch /.mu-installer-ran-updates
 fi
+rm -f /etc/nologin
 
 AWSCLI='command -v aws'
 PIP='command -v pip'

data/modules/mu/providers/aws/vpc.rb

@@ -35,7 +35,7 @@ module MU
         # Called automatically by {MU::Deploy#createResources}
         def create
           MU.log "Creating VPC #{@mu_name}", details: @config
-          resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_vpc(cidr_block: @config['ip_block']).vpc
+          resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_vpc(cidr_block: @config['ip_block']).vpc
           @cloud_id = resp.vpc_id
           @config['vpc_id'] = @cloud_id
 
@@ -45,10 +45,10 @@ module MU
             begin
               MU.log "Waiting for VPC #{@mu_name} (#{@cloud_id}) to be available", MU::NOTICE
               sleep 5
-              resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpcs(vpc_ids: [@cloud_id]).vpcs.first
+              resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_vpcs(vpc_ids: [@cloud_id]).vpcs.first
             end while resp.state != "available"
             # There's a default route table that comes with. Let's tag it.
-            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
+            resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_route_tables(
               filters: [
                 {
                   name: "vpc-id",
@@ -63,13 +63,13 @@ module MU
 
           if @config['create_internet_gateway']
             MU.log "Creating Internet Gateway #{@mu_name}"
-            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_internet_gateway
+            resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_internet_gateway
             internet_gateway_id = resp.internet_gateway.internet_gateway_id
             sleep 5
 
             tag_me(internet_gateway_id)
 
-            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).attach_internet_gateway(vpc_id: @cloud_id, internet_gateway_id: internet_gateway_id)
+            MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).attach_internet_gateway(vpc_id: @cloud_id, internet_gateway_id: internet_gateway_id)
             @config['internet_gateway_id'] = internet_gateway_id
           end
 
@@ -93,7 +93,7 @@ module MU
               config[:policy_document] = statement.to_json
             end
 
-            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_vpc_endpoint(config).vpc_endpoint
+            resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_vpc_endpoint(config).vpc_endpoint
             endpoint_id = resp.vpc_endpoint_id
             MU.log "Creating VPC endpoint #{endpoint_id}"
             attempts = 0
@@ -102,7 +102,7 @@ module MU
               MU.log "Waiting for VPC endpoint #{endpoint_id} to become available" if attempts % 5 == 0
               sleep 10
               begin
-                resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_endpoints(vpc_endpoint_ids: [endpoint_id]).vpc_endpoints.first
+                resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_vpc_endpoints(vpc_endpoint_ids: [endpoint_id]).vpc_endpoints.first
               rescue Aws::EmptyStructure, NoMethodError
                 sleep 5
                 retry
@@ -119,7 +119,7 @@ module MU
             logrole = @deploy.findLitterMate(name: @config['name']+"logrole", type: "roles")
 
             MU.log "Enabling traffic logging on VPC #{@mu_name} to log group #{loggroup.mu_name}"
-            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_flow_logs(
+            MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_flow_logs(
               resource_ids: [@cloud_id],
               resource_type: "VPC",
               traffic_type: "ALL",
@@ -150,7 +150,7 @@ module MU
                     MU.log "Creating route for #{route['destination_network']} through NAT gatway #{gateway['id']}", details: route_config
                     MU.retrier([Aws::EC2::Errors::InvalidNatGatewayIDNotFound], wait: 10, max: 5) {
                       begin
-                        resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(route_config)
+                        resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_route(route_config)
                       rescue Aws::EC2::Errors::RouteAlreadyExists
                         MU.log "Attempt to create duplicate route to #{route['destination_network']} for #{gateway['id']} in #{rtb['route_table_id']}", MU::WARN
                       end
@@ -163,14 +163,14 @@ module MU
 
           if @config['enable_dns_support']
             MU.log "Enabling DNS support in #{@mu_name}"
-            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_vpc_attribute(
+            MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).modify_vpc_attribute(
                 vpc_id: @cloud_id,
                 enable_dns_support: {value: @config['enable_dns_support']}
             )
           end
           if @config['enable_dns_hostnames']
             MU.log "Enabling DNS hostnames in #{@mu_name}"
-            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_vpc_attribute(
+            MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).modify_vpc_attribute(
                 vpc_id: @cloud_id,
                 enable_dns_hostnames: {value: @config['enable_dns_hostnames']}
             )
@@ -196,20 +196,20 @@ module MU
               dhcpopts << {key: "netbios-name-servers", values: @config['dhcp']['netbios_servers']}
             end
 
-            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_dhcp_options(
+            resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_dhcp_options(
                 dhcp_configurations: dhcpopts
             )
             dhcpopt_id = resp.dhcp_options.dhcp_options_id
             tag_me(dhcpopt_id)
 
-            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).associate_dhcp_options(dhcp_options_id: dhcpopt_id, vpc_id: @cloud_id)
+            MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).associate_dhcp_options(dhcp_options_id: dhcpopt_id, vpc_id: @cloud_id)
           end
           notify
 
-          if !MU::Cloud::AWS.isGovCloud?(@config['region'])
-            mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu", credentials: @config['credentials']).values.first
+          if !MU::Cloud::AWS.isGovCloud?(@region)
+            mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu", credentials: @credentials).values.first
             if !mu_zone.nil?
-              MU::Cloud.resourceClass("AWS", "DNSZone").toggleVPCAccess(id: mu_zone.id, vpc_id: @cloud_id, region: @config['region'], credentials: @config['credentials'])
+              MU::Cloud.resourceClass("AWS", "DNSZone").toggleVPCAccess(id: mu_zone.id, vpc_id: @cloud_id, region: @region, credentials: @credentials)
             end
           end
 					loadSubnets
@@ -220,7 +220,7 @@ module MU
         # Canonical Amazon Resource Number for this resource
         # @return [String]
         def arn
-          "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":vpc/"+@cloud_id
+          "arn:"+(MU::Cloud::AWS.isGovCloud?(@region) ? "aws-us-gov" : "aws")+":ec2:"+@region+":"+MU::Cloud::AWS.credToAcct(@credentials)+":vpc/"+@cloud_id
         end
 
         # Describe this VPC
@@ -263,7 +263,7 @@ module MU
                   route_config[:instance_id] = nat_instance.cloud_id
 
                   MU.log "Creating route for #{route['destination_network']} through NAT host #{nat_instance.cloud_id}", details: route_config
-                  MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(route_config)
+                  MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_route(route_config)
                 end
               }
 
@@ -327,9 +327,9 @@ module MU
         def toKitten(**_args)
           bok = {
             "cloud" => "AWS",
-            "credentials" => @config['credentials'],
+            "credentials" => @credentials,
             "cloud_id" => @cloud_id,
-            "region" => @config['region']
+            "region" => @region
           }
 
           if !cloud_desc
@@ -352,7 +352,7 @@ module MU
 
           bok['create_bastion'] = false # XXX figure out a way to detect this
 
-          logs = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @credentials).describe_flow_logs(filter: [{ "name" => "resource-id", "values" => [@cloud_id] }])
+          logs = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_flow_logs(filter: [{ "name" => "resource-id", "values" => [@cloud_id] }])
           if logs and logs.flow_logs and !logs.flow_logs.empty?
             bok['enable_traffic_logging'] = true
             bok['traffic_type_to_log'] = logs.flow_logs.first.traffic_type.downcase
@@ -362,13 +362,13 @@ module MU
             end
           end
 
-          nats = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @credentials).describe_nat_gateways(filter: [{ "name" => "vpc-id", "values" => [@cloud_id] }])
+          nats = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_nat_gateways(filter: [{ "name" => "vpc-id", "values" => [@cloud_id] }])
           if nats and nats.nat_gateways and !nats.nat_gateways.empty?
             bok['create_nat_gateway'] = true
             bok['nat_gateway_multi_az'] = true if nats.nat_gateways.size > 1
           end
 
-          rtbs = MU::Cloud::AWS::VPC.get_route_tables(vpc_ids: [@cloud_id], region: @config['region'], credentials: @credentials)
+          rtbs = MU::Cloud::AWS::VPC.get_route_tables(vpc_ids: [@cloud_id], region: @region, credentials: @credentials)
 
           associations = {}
           if rtbs and !rtbs.empty?
@@ -454,13 +454,13 @@ module MU
         def loadSubnets
           return [] if !@cloud_id
 
-          resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(
+          resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_subnets(
             filters: [
               { name: "vpc-id", values: [@cloud_id] }
             ]
           )
           if resp.nil? or resp.subnets.nil? or resp.subnets.empty?
-            MU.log "Got empty results when trying to list subnets in #{@cloud_id}", MU::WARN
+            MU.log "Got empty results when trying to list subnets in #{@cloud_id} (#{@region})", MU::WARN
             return []
           end
 
@@ -473,8 +473,8 @@ module MU
             if !@config.nil? and @config.has_key?("subnets")
               @config['subnets'].each { |subnet|
                 subnet['mu_name'] ||= @mu_name+"-"+subnet['name']
-                subnet['region'] = @config['region']
-                subnet['credentials'] = @config['credentials']
+                subnet['region'] = @region
+                subnet['credentials'] = @credentials
                 resp.subnets.each { |desc|
                   if desc.cidr_block == subnet["ip_block"]
                     subnet["tags"] = MU.structToHash(desc.tags)
@@ -500,15 +500,32 @@ module MU
             # Of course we might be loading up a dummy subnet object from a
             # foreign or non-Mu-created VPC and subnet. So make something up.
             if @subnets.empty?
+              nets_by_block = {}
+
+              # Attempt to dig the canonical resource name out of
+              # deployment metadata, if it exists
+              if @deploy and @deploy.deployment and
+                 @deploy.deployment['vpcs'] and
+                 @deploy.deployment['vpcs'][@config['name']] and
+                 @deploy.deployment['vpcs'][@config['name']]['subnets']
+                @deploy.deployment['vpcs'][@config['name']]['subnets'].each { |s|
+                  nets_by_block[s["ip_block"]] = s
+                }
+              end
+
               resp.subnets.each { |desc|
                 subnet = {
                   "ip_block" => desc.cidr_block,
                   "tags" => MU.structToHash(desc.tags),
                   "cloud_id" => desc.subnet_id,
-                  'region' => @config['region'],
-                  'credentials' => @config['credentials'],
+                  'region' => @region,
+                  'credentials' => @credentials,
                 }
-                subnet['name'] = subnet["ip_block"].gsub(/[\.\/]/, "_")
+                if nets_by_block[desc.cidr_block] and
+                   nets_by_block[desc.cidr_block]["name"]
+                  subnet['name'] = nets_by_block[desc.cidr_block]["name"]
+                end
+                subnet['name'] ||= subnet["ip_block"].gsub(/[\.\/]/, "_")
                 subnet['mu_name'] = @mu_name+"-"+subnet['name']
                 @subnets << MU::Cloud::AWS::VPC::Subnet.new(self, subnet)
               }
@@ -571,7 +588,7 @@ module MU
               @config['cloud'],
               "server",
               name: nat_name,
-              region: @config['region'],
+              region: @region,
               cloud_id: nat_cloud_id,
               deploy_id: deploy_id,
               tag_key: nat_tag_key,
@@ -822,11 +839,11 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           MU.log "AWS::VPC.cleanup: need to support flags['known']", MU::DEBUG, details: flags
 
           tagfilters = [
-            {name: "tag:MU-ID", values: [MU.deploy_id]}
+            {name: "tag:MU-ID", values: [deploy_id]}
           ]
           if !ignoremaster
             tagfilters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
@@ -838,7 +855,7 @@ module MU
             vpcs = resp if !resp.empty?
           }
 
-#          resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
+#          resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_vpc_peering_connections(
 #            filters: [
 #              {
 #                name: "requester-vpc-info.vpc-id",
@@ -873,10 +890,11 @@ module MU
           purge_subnets(noop, tagfilters, region: region, credentials: credentials)
           purge_vpcs(noop, tagfilters, region: region, credentials: credentials)
           purge_dhcpopts(noop, tagfilters, region: region, credentials: credentials)
+          purge_eips(noop, tagfilters, region: region, credentials: credentials)
 
 #          unless noop
 #            MU::Cloud::AWS.iam.list_roles.roles.each{ |role|
-#              match_string = "#{MU.deploy_id}.*TRAFFIC-LOG"
+#              match_string = "#{deploy_id}.*TRAFFIC-LOG"
 #            }
 #          end
         end
@@ -1019,7 +1037,7 @@ module MU
                 subnet_routes[table['name']].each { |subnet|
                   nat_routes[subnet] = route['nat_host_name']
                 }
-                MU::Config.addDependency(vpc, route['nat_host_name'], "server", no_create_wait: true)
+                MU::Config.addDependency(vpc, route['nat_host_name'], "server", my_phase: "groom")
               elsif route['gateway'] == '#NAT'
                 vpc['create_nat_gateway'] = true
                 private_rtbs << table['name']
@@ -1118,7 +1136,7 @@ module MU
           if subnets and subnets.size > 0
             filters << { :name => "association.subnet-id", :values => subnets }
           end
-          tables = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
+          tables = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_route_tables(
             filters: filters
           )
           cidrs = []
@@ -1319,7 +1337,7 @@ module MU
             id: @cloud_id,
             cloud: "AWS",
             credentials: @credentials,
-            region: @config['region'],
+            region: @region,
             type: "vpcs",
             subnet_pref: subnet_pref
           )
@@ -1330,6 +1348,9 @@ module MU
         def peerWith(peer)
           peer_ref = MU::Config::Ref.get(peer['vpc'])
           peer_obj = peer_ref.kitten
+          if !peer_obj
+            raise MuError.new "#{@mu_name}: Failed to locate my peer VPC", details: peer_ref.to_h
+          end
           peer_id = peer_ref.kitten.cloud_id
           if peer_id == @cloud_id
             MU.log "#{@mu_name} attempted to peer with itself (#{@cloud_id})", MU::ERR, details: peer
@@ -1353,7 +1374,7 @@ module MU
 
           # See if the peering connection exists before we bother
           # creating it.
-          resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
+          resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_vpc_peering_connections(
             filters: [
               {
                 name: "requester-vpc-info.vpc-id",
@@ -1369,8 +1390,8 @@ module MU
           peering_id = if !resp or !resp.vpc_peering_connections or
              resp.vpc_peering_connections.empty?
 
-            MU.log "Setting peering connection from VPC #{@config['name']} (#{@cloud_id} in account #{MU::Cloud::AWS.credToAcct(@config['credentials'])}) to #{peer_id} in account #{peer['account']}", details: peer
-            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_vpc_peering_connection(
+            MU.log "Setting peering connection from VPC #{@config['name']} (#{@cloud_id} in account #{MU::Cloud::AWS.credToAcct(@credentials)}) to #{peer_id} in account #{peer['account']}", details: peer
+            resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_vpc_peering_connection(
               vpc_id: @cloud_id,
               peer_vpc_id: peer_id,
               peer_owner_id: peer['account'],
@@ -1386,13 +1407,13 @@ module MU
           tag_me(peering_id, peering_name)
 
           # Create routes to our new friend.
-          MU::Cloud::AWS::VPC.listAllSubnetRouteTables(@cloud_id, region: @config['region'], credentials: @config['credentials']).each { |rtb_id|
+          MU::Cloud::AWS::VPC.listAllSubnetRouteTables(@cloud_id, region: @region, credentials: @credentials).each { |rtb_id|
             my_route_config = {
               :route_table_id => rtb_id,
               :destination_cidr_block => peer_obj.cloud_desc.cidr_block,
               :vpc_peering_connection_id => peering_id
             }
-            rtbdesc = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
+            rtbdesc = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_route_tables(
               route_table_ids: [rtb_id]
             ).route_tables.first
             already_exists = false
@@ -1408,18 +1429,18 @@ module MU
             }
             next if already_exists
 
-            MU.log "Creating peering route to #{peer_obj.cloud_desc.cidr_block} in #{peer['vpc']['region']} from VPC #{@config['name']} in #{@config['region']}"
-            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(my_route_config)
+            MU.log "Creating peering route to #{peer_obj.cloud_desc.cidr_block} in #{peer['vpc']['region']} from VPC #{@config['name']} in #{@region}"
+            resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_route(my_route_config)
           } # MU::Cloud::AWS::VPC.listAllSubnetRouteTables
 
           can_auto_accept = ((!peer_obj.nil? and !peer_obj.deploydata.nil? and peer_obj.deploydata['auto_accept_peers']) or $MU_CFG['allow_invade_foreign_vpcs'])
 
-          cnxn = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
+          cnxn = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_vpc_peering_connections(
             vpc_peering_connection_ids: [peering_id]
           ).vpc_peering_connections.first
 
           loop_if = Proc.new {
-            cnxn = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
+            cnxn = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_vpc_peering_connections(
               vpc_peering_connection_ids: [peering_id]
             ).vpc_peering_connections.first
             ((can_auto_accept and cnxn.status.code == "pending-acceptance") or (cnxn.status.code != "active" and cnxn.status.code != "pending-acceptance"))
@@ -1448,9 +1469,9 @@ module MU
             end
 
             if ["failed", "rejected", "expired", "deleted"].include?(cnxn.status.code)
-              MU.log "VPC peering connection from VPC #{@config['name']} (#{@cloud_id} in #{@config['region']}) to #{peer_id} in #{peer_obj.config['region']} #{cnxn.status.code}: #{cnxn.status.message}", MU::ERR
+              MU.log "VPC peering connection from VPC #{@config['name']} (#{@cloud_id} in #{@region}) to #{peer_id} in #{peer_obj.config['region']} #{cnxn.status.code}: #{cnxn.status.message}", MU::ERR
               begin
-                MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).delete_vpc_peering_connection(
+                MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).delete_vpc_peering_connection(
                   vpc_peering_connection_id: peering_id
                 )
               rescue Aws::EC2::Errors::InvalidStateTransition
@@ -1466,8 +1487,8 @@ module MU
         def tag_me(resource_id = @cloud_id, name = @mu_name)
           MU::Cloud::AWS.createStandardTags(
             resource_id,
-            region: @config['region'],
-            credentials: @config['credentials'],
+            region: @region,
+            credentials: @credentials,
             optional: @config['optional_tags'],
             nametag: name,
             othertags: @config['tags']
@@ -1481,8 +1502,8 @@ module MU
         def createRouteTable(rtb)
           vpc_id = @cloud_id
           vpc_name = @config['name']
-          MU.setVar("curRegion", @config['region']) if !@config['region'].nil?
-          resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route_table(vpc_id: vpc_id).route_table
+          MU.setVar("curRegion", @region) if !@region.nil?
+          resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_route_table(vpc_id: vpc_id).route_table
           route_table_id = rtb['route_table_id'] = resp.route_table_id
           sleep 5
 
@@ -1503,7 +1524,7 @@ module MU
               unless route['gateway'] == '#NAT'
                 # Need to change the order of how things are created to create the route here
                 MU.log "Creating route for #{route['destination_network']}", details: route_config
-                resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(route_config)
+                resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_route(route_config)
               end
             end
           }
@@ -1611,6 +1632,40 @@ module MU
         end
         private_class_method :purge_nat_gateways
 
+        # Remove all Elastic IPs from the currently loaded deployment.
+        # @param noop [Boolean]: If true, will only print what would be done
+        # @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
+        # @param region [String]: The cloud provider region
+        # @return [void]
+        def self.purge_eips(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion, credentials: nil)
+          eips = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_addresses(
+            filters: tagfilters
+          ).addresses
+
+          threads = []
+
+          if !eips.empty?
+            eips.each { |eip|
+              MU.log "Releasing EIP #{eip.public_ip} (#{eip.allocation_id})"
+              next if noop
+              if eip.association_id
+                MU.log "Tags tell me I should release EIP #{eip.public_ip} (#{eip.allocation_id}), but it appears to be associated with something", MU::WARN, details: eip
+                next
+              end
+              threads << Thread.new {
+                MU::Cloud::AWS.ec2(credentials: credentials, region: region).release_address(allocation_id: eip.allocation_id)
+              }
+            }
+          end
+
+          threads.each { |t|
+            t.join
+          }
+
+          return nil
+        end
+        private_class_method :purge_eips
+
         # Remove all VPC endpoints associated with the VPC of the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param vpc_id [String]: The cloud provider's unique VPC identifier