cloud-mu 3.1.5 → 3.3.2

data/modules/mu/{clouds → providers}/aws/server_pool.rb

@@ -120,7 +120,7 @@ module MU
                   if !@deploy.nocleanup
                     Thread.new {
                       MU.dupGlobals(parent_thread_id)
-                      MU::Cloud::AWS::Server.terminateInstance(id: member.instance_id)
+                      MU::Cloud.resourceClass("AWS", "Server").terminateInstance(id: member.instance_id)
                     }
                   end
                 end
@@ -193,9 +193,10 @@ module MU
         # @return [Array<MU::Cloud::Server>]
         def listNodes
           nodes = []
-          me = MU::Cloud::AWS::ServerPool.find(cloud_id: cloud_id)
-          if me and me.first and me.first.instances
-            me.first.instances.each { |instance|
+          me = MU::Cloud::AWS::ServerPool.find(cloud_id: cloud_id).values.first
+          pp me
+          if me and me.instances
+            me.instances.each { |instance|
               found = MU::MommaCat.findStray("AWS", "server", cloud_id: instance.instance_id, region: @config["region"], dummy_ok: true)
               nodes.concat(found)
             }
@@ -425,6 +426,7 @@ module MU
         # @return [OpenStruct]
         def cloud_desc(use_cache: true)
           return @cloud_desc_cache if @cloud_desc_cache and use_cache
+          return nil if !@cloud_id
           @cloud_desc_cache = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_auto_scaling_groups(
             auto_scaling_group_names: [@mu_name]
           ).auto_scaling_groups.first
@@ -531,14 +533,25 @@ module MU
         if cloud_desc.vpc_zone_identifier and
            !cloud_desc.vpc_zone_identifier.empty?
           nets = cloud_desc.vpc_zone_identifier.split(/,/)
-          resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @credentials).describe_subnets(subnet_ids: nets).subnets.first
-          bok['vpc'] = MU::Config::Ref.get(
-            id: resp.vpc_id,
-            cloud: "AWS",
-            credentials: @credentials,
-            type: "vpcs",
-            subnets: nets.map { |s| { "subnet_id" => s } }
-          )
+          begin
+            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @credentials).describe_subnets(subnet_ids: nets).subnets.first
+            bok['vpc'] = MU::Config::Ref.get(
+              id: resp.vpc_id,
+              cloud: "AWS",
+              credentials: @credentials,
+              type: "vpcs",
+              subnets: nets.map { |s| { "subnet_id" => s } }
+            )
+          rescue Aws::EC2::Errors::InvalidSubnetIDNotFound => e
+            if e.message.match(/The subnet ID '(subnet-[a-f0-9]+)' does not exist/)
+              nets.delete(Regexp.last_match[1])
+              if nets.empty?
+                MU.log "Autoscale Group #{@cloud_id} was configured for a VPC, but the configuration held no valid subnets", MU::WARN, details: cloud_desc.vpc_zone_identifier.split(/,/)
+              end
+            else
+              raise e
+            end
+          end
         end
 
 #        MU.log @cloud_id, MU::NOTICE, details: cloud_desc
@@ -813,26 +826,7 @@ module MU
                 }
               }
             },
-            "ingress_rules" => {
-              "items" => {
-                "properties" => {
-                  "sgs" => {
-                    "type" => "array",
-                    "items" => {
-                      "description" => "Other AWS Security Groups; resources that are associated with this group will have this rule applied to their traffic",
-                      "type" => "string"
-                    }
-                  },
-                  "lbs" => {
-                    "type" => "array",
-                    "items" => {
-                      "description" => "AWS Load Balancers which will have this rule applied to their traffic",
-                      "type" => "string"
-                    }
-                  }
-                }
-              }
-            }
+            "ingress_rules" => MU::Cloud.resourceClass("AWS", "FirewallRule").ingressRuleAddtlSchema
           }
           [toplevel_required, schema]
         end
@@ -905,7 +899,7 @@ module MU
             launch = pool["basis"]["launch_config"]
             launch['iam_policies'] ||= pool['iam_policies']
 
-            launch['size'] = MU::Cloud::AWS::Server.validateInstanceType(launch["size"], pool["region"])
+            launch['size'] = MU::Cloud.resourceClass("AWS", "Server").validateInstanceType(launch["size"], pool["region"])
             ok = false if launch['size'].nil?
             if !launch['generate_iam_role']
               if !launch['iam_role'] and pool['cloud'] != "CloudFormation"
@@ -949,11 +943,7 @@ module MU
 
               role['credentials'] = pool['credentials'] if pool['credentials']
               configurator.insertKitten(role, "roles")
-              pool["dependencies"] ||= []
-              pool["dependencies"] << {
-                "type" => "role",
-                "name" => pool["name"]
-              }
+              MU::Config.addDependency(pool, pool['name'], "role")
             end
             launch["ami_id"] ||= launch["image_id"]
             if launch["server"].nil? and launch["instance_id"].nil? and launch["ami_id"].nil?
@@ -967,7 +957,7 @@ module MU
               end
             end
             if launch["server"] != nil
-              pool["dependencies"] << {"type" => "server", "name" => launch["server"]}
+              MU::Config.addDependency(pool, launch["server"], "server", phase: "groom")
 # XXX I dunno, maybe toss an error if this isn't done already
 #              servers.each { |server|
 #                if server["name"] == launch["server"]
@@ -1073,7 +1063,7 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           MU.log "AWS::ServerPool.cleanup: need to support flags['known']", MU::DEBUG, details: flags
 
           filters = [{name: "key", values: ["MU-ID"]}]
@@ -1096,7 +1086,7 @@ module MU
             if asg.key == "MU-MASTER-IP" and asg.value != MU.mu_public_ip and !ignoremaster
               no_purge << asg.resource_id
             end
-            if asg.key == "MU-ID" and asg.value == MU.deploy_id
+            if asg.key == "MU-ID" and asg.value == deploy_id
               maybe_purge << asg.resource_id
             end
           }
@@ -1123,7 +1113,7 @@ module MU
               end
             end
 
-#            MU::Cloud::AWS::Server.removeIAMProfile(resource_id)
+#            MU::Cloud.resourceClass("AWS", "Server").removeIAMProfile(resource_id)
 
             # Generally there should be a launch_configuration of the same name
             # XXX search for these independently, too?
@@ -1164,14 +1154,14 @@ module MU
             @config['basis']['launch_config']["ami_id"] = @deploy.deployment["images"][@config['basis']['launch_config']["server"]]["image_id"]
             MU.log "Using AMI '#{@config['basis']['launch_config']["ami_id"]}' from sibling server #{@config['basis']['launch_config']["server"]} in ServerPool #{@mu_name}"
           elsif !@config['basis']['launch_config']["instance_id"].nil?
-            @config['basis']['launch_config']["ami_id"] = MU::Cloud::AWS::Server.createImage(
+            @config['basis']['launch_config']["ami_id"] = MU::Cloud.resourceClass("AWS", "Server").createImage(
               name: @mu_name,
               instance_id: @config['basis']['launch_config']["instance_id"],
               credentials: @config['credentials'],
               region: @config['region']
             )[@config['region']]
           end
-          MU::Cloud::AWS::Server.waitForAMI(@config['basis']['launch_config']["ami_id"], credentials: @config['credentials'])
+          MU::Cloud.resourceClass("AWS", "Server").waitForAMI(@config['basis']['launch_config']["ami_id"], credentials: @config['credentials'])
 
           oldlaunch = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_launch_configurations(
             launch_configuration_names: [@mu_name]
@@ -1226,12 +1216,12 @@ module MU
                   vol.delete("encrypted")
                 end
               end
-              mapping, _cfm_mapping = MU::Cloud::AWS::Server.convertBlockDeviceMapping(vol)
+              mapping, _cfm_mapping = MU::Cloud.resourceClass("AWS", "Server").convertBlockDeviceMapping(vol)
               storage << mapping
             }
           end
 
-          storage.concat(MU::Cloud::AWS::Server.ephemeral_mappings)
+          storage.concat(MU::Cloud.resourceClass("AWS", "Server").ephemeral_mappings)
 
           if @config['basis']['launch_config']['generate_iam_role']
             role = @deploy.findLitterMate(name: @config['name'], type: "roles")
@@ -1426,20 +1416,12 @@ module MU
                     # XXX probably have to query API to get the DNS name of this one
                 }
               elsif lb["concurrent_load_balancer"]
-                raise MuError, "No loadbalancers exist! I need one named #{lb['concurrent_load_balancer']}" if !@deploy.deployment["loadbalancers"]
-                found = false
-                @deploy.deployment["loadbalancers"].each_pair { |lb_name, deployed_lb|
-                  if lb_name == lb['concurrent_load_balancer']
-                    lbs << deployed_lb["awsname"] # XXX check for classic
-                    if deployed_lb.has_key?("targetgroups")
-                      deployed_lb["targetgroups"].values.each { |tg_arn|
-                        tg_arns << tg_arn
-                      }
-                    end
-                    found = true
-                  end
-                }
-                raise MuError, "I need a loadbalancer named #{lb['concurrent_load_balancer']}, but none seems to have been created!" if !found
+                lb = @deploy.findLitterMate(name: lb['concurrent_load_balancer'], type: "loadbalancers")
+                raise MuError, "No loadbalancers exist! I need one named #{lb['concurrent_load_balancer']}" if !lb
+                lbs << lb.mu_name
+                if lb.targetgroups
+                  tg_arns = lb.targetgroups.values.map { |tg| tg.target_group_arn }
+                end
               end
             }
             if tg_arns.size > 0

data/modules/mu/{clouds → providers}/aws/storage_pool.rb

@@ -67,7 +67,7 @@ module MU
                 if target['vpc']["subnet_name"]
                   subnet_obj = vpc.getSubnet(name: target['vpc']["subnet_name"])
                   if subnet_obj.nil?
-                    raise MuError, "Failed to locate subnet from #{subnet} in StoragePool #{@config['name']}:#{target['name']}"
+                    raise MuError, "Failed to locate subnet from #{target['vpc']["subnet_name"]} in StoragePool #{@config['name']}:#{target['name']}"
                   end
                   target['vpc']['subnet_id'] = subnet_obj.cloud_id
                 end
@@ -261,49 +261,29 @@ module MU
           targets = {}
 
           if @config['mount_points'] && !@config['mount_points'].empty?
+            mount_targets = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_mount_targets(
+              file_system_id: storage_pool.file_system_id
+            ).mount_targets
+
             @config['mount_points'].each { |mp|
               subnet = nil
               dependencies
-              mp_vpc = if mp['vpc'] and mp['vpc']['vpc_name']
-                @deploy.findLitterMate(type: "vpc", name: mp['vpc']['vpc_name'], credentials: @config['credentials'])
-              elsif mp['vpc']
-                MU::MommaCat.findStray(
-                  @config['cloud'],
-                  "vpcs",
-                  deploy_id: mp['vpc']["deploy_id"],
-                  credentials: @config['credentials'],
-                  mu_name: mp['vpc']["mu_name"],
-                  cloud_id: mp['vpc']['vpc_id'],
-                  region: @config['region'],
-                  dummy_ok: false
-                ).first
-# XXX non-sibling, findStray version
-              end
+              mp_vpc = MU::Config::Ref.get(mp['vpc']).kitten
 
-              mount_targets = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_mount_targets(
-                file_system_id: storage_pool.file_system_id
-              ).mount_targets
 
-#              subnet_obj = mp_vpc.subnets.select { |s|
-#                s.name == mp["vpc"]["subnet_name"] or s.cloud_id == mp["vpc"]["subnet_id"]
-#              }.first
+              subnet_obj = mp_vpc.subnets.select { |s|
+                s.name == mp["vpc"]["subnet_name"] or s.cloud_id == mp["vpc"]["subnet_id"]
+              }.first
               mount_target = nil
-              mp_vpc.subnets.each { |subnet_obj|
-                mount_targets.map { |t|
-                  subnet_cidr_obj = NetAddr::IPv4Net.parse(subnet_obj.ip_block)
-                  if subnet_cidr_obj.contains(NetAddr::IPv4.parse(t.ip_address))
-                    mount_target = t
-                    subnet = subnet_obj.cloud_desc
-                  end
-                }
-                break if mount_target
+              mount_targets.each { |t|
+                subnet_cidr_obj = NetAddr::IPv4Net.parse(subnet_obj.ip_block)
+                if subnet_cidr_obj.contains(NetAddr::IPv4.parse(t.ip_address))
+                  mount_target = t
+                  subnet = subnet_obj.cloud_desc
+                  break
+                end
               }
 
-              # mount_target = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_mount_targets(
-                # mount_target_id: mp["cloud_id"]
-              # ).mount_targets.first
-
-
               targets[mp["name"]] = {
                 "owner_id" => mount_target.owner_id,
                 "cloud_id" => mount_target.mount_target_id,
@@ -353,7 +333,7 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region in which to operate
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           MU.log "AWS::StoragePool.cleanup: need to support flags['known']", MU::DEBUG, details: flags
 
           supported_regions = %w{us-west-2 us-east-1 eu-west-1}
@@ -378,7 +358,7 @@ module MU
                 found_muid = false
                 found_master = false
                 tags.each { |tag|
-                  found_muid = true if tag.key == "MU-ID" && tag.value == MU.deploy_id
+                  found_muid = true if tag.key == "MU-ID" && tag.value == deploy_id
                   found_master = true if tag.key == "MU-MASTER-IP" && tag.value == MU.mu_public_ip
                 }
                 next if !found_muid
@@ -493,6 +473,9 @@ module MU
 
           if pool['mount_points'] && !pool['mount_points'].empty?
             pool['mount_points'].each{ |mp|
+              if mp['vpc'] and mp['vpc']['name']
+                MU::Config.addDependency(pool, mp['vpc']['name'], "vpc")
+              end
               if mp['ingress_rules']
                 fwname = "storage-#{mp['name']}"
                 acl = {

data/modules/mu/{clouds → providers}/aws/user.rb

@@ -109,7 +109,7 @@ module MU
           # Create these if necessary, then append them to the list of
           # attachable_policies
           if @config['raw_policies']
-            pol_arns = MU::Cloud::AWS::Role.manageRawPolicies(
+            pol_arns = MU::Cloud.resourceClass("AWS", "Role").manageRawPolicies(
               @config['raw_policies'],
               basename: @deploy.getResourceName(@config['name']),
               credentials: @credentials
@@ -135,7 +135,7 @@ module MU
             attached_policies.each { |a|
               if !configured_policies.include?(a.policy_arn)
                 MU.log "Removing IAM policy #{a.policy_arn} from user #{@mu_name}", MU::NOTICE
-                MU::Cloud::AWS::Role.purgePolicy(a.policy_arn, @credentials)
+                MU::Cloud.resourceClass("AWS", "Role").purgePolicy(a.policy_arn, @credentials)
               else
                 configured_policies.delete(a.policy_arn)
               end
@@ -151,7 +151,7 @@ module MU
           end
 
           if @config['inline_policies']
-            docs = MU::Cloud::AWS::Role.genPolicyDocument(@config['inline_policies'], deploy_obj: @deploy)
+            docs = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['inline_policies'], deploy_obj: @deploy)
             docs.each { |doc|
               MU.log "Putting user policy #{doc.keys.first} to user #{@cloud_id} "
               MU::Cloud::AWS.iam(credentials: @credentials).put_user_policy(
@@ -190,16 +190,16 @@ module MU
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
           MU.log "AWS::User.cleanup: need to support flags['known']", MU::DEBUG, details: flags
 
           # XXX this doesn't belong here; maybe under roles, maybe as its own stupid first-class resource
           resp = MU::Cloud::AWS.iam(credentials: credentials).list_policies(
-            path_prefix: "/"+MU.deploy_id+"/"
+            path_prefix: "/"+deploy_id+"/"
           )
           if resp and resp.policies
             resp.policies.each { |policy|
-              MU.log "Deleting policy /#{MU.deploy_id}/#{policy.policy_name}"
+              MU.log "Deleting policy /#{deploy_id}/#{policy.policy_name}"
               if !noop
                 attachments = begin
                   MU::Cloud::AWS.iam(credentials: credentials).list_entities_for_policy(
@@ -277,7 +277,7 @@ MU.log e.inspect, MU::ERR, details: policy
             has_ourdeploy = false
             has_ourmaster = false
             tags.each { |tag|
-              if tag.key == "MU-ID" and tag.value == MU.deploy_id
+              if tag.key == "MU-ID" and tag.value == deploy_id
                 has_ourdeploy = true
               elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip
                 has_ourmaster = true
@@ -431,7 +431,7 @@ MU.log e.inspect, MU::ERR, details: policy
             resp.policy_names.each { |pol_name|
               pol = MU::Cloud::AWS.iam(credentials: @credentials).get_user_policy(user_name: @cloud_id, policy_name: pol_name)
               doc = JSON.parse(URI.decode(pol.policy_document))
-              bok["inline_policies"] = MU::Cloud::AWS::Role.doc2MuPolicies(pol.policy_name, doc, bok["inline_policies"])
+              bok["inline_policies"] = MU::Cloud.resourceClass("AWS", "Role").doc2MuPolicies(pol.policy_name, doc, bok["inline_policies"])
             }
           end
 
@@ -465,7 +465,7 @@ MU.log e.inspect, MU::ERR, details: policy
         def self.schema(_config)
           toplevel_required = []
           polschema = MU::Config::Role.schema["properties"]["policies"]
-          polschema.deep_merge!(MU::Cloud::AWS::Role.condition_schema)
+          polschema.deep_merge!(MU::Cloud.resourceClass("AWS", "Role").condition_schema)
 
           schema = {
             "inline_policies" => polschema,
@@ -517,7 +517,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-USER-FOO+"
           # If we're attaching some managed policies, make sure all of the ones
           # that should already exist do indeed exist
           if user['attachable_policies']
-            ok = false if !MU::Cloud::AWS::Role.validateAttachablePolicies(
+            ok = false if !MU::Cloud.resourceClass("AWS", "Role").validateAttachablePolicies(
               user['attachable_policies'],
               credentials: user['credentials'],
               region: user['region']
@@ -530,7 +530,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-USER-FOO+"
               if configurator.haveLitterMate?(group, "groups")
                 need_dependency = true
               else
-                found = MU::Cloud::AWS::Group.find(cloud_id: group)
+                found = MU::Cloud.resourceClass("AWS", "Group").find(cloud_id: group)
                 if found.nil? or found.empty? or (configurator.updating and
                    found.values.first.group.path == "/"+configurator.updating+"/")
                   groupdesc = {
@@ -542,11 +542,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-USER-FOO+"
               end
 
               if need_dependency
-                user["dependencies"] ||= []
-                user["dependencies"] << {
-                  "type" => "group",
-                  "name" => group
-                }
+                MU::Config.addDependency(user, group, "group")
               end
             }
           end

data/modules/mu/{clouds → providers}/aws/userdata/linux.erb

@@ -42,7 +42,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
 <% if !$mu.skipApplyUpdates %>
     set +e
     if [ ! -f /.mu-installer-ran-updates ];then
-      service ssh stop
+      echo "Applying package updates" > /etc/nologin
       apt-get --fix-missing -y upgrade 
       touch /.mu-installer-ran-updates
       if [ $? -eq 0 ]
@@ -58,7 +58,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
       else
         echo "FAILED PACKAGE UPDATE" >&2
       fi
-      service ssh start
+      rm -f /etc/nologin
     fi
 <% end %>
   elif [ -x /usr/bin/yum ];then
@@ -94,7 +94,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
 <% if !$mu.skipApplyUpdates %>
     set +e
     if [ ! -f /.mu-installer-ran-updates ];then
-      service sshd stop
+      echo "Applying package updates" > /etc/nologin
       kernel_update=`yum list updates | grep kernel`
       yum -y update
       touch /.mu-installer-ran-updates
@@ -108,7 +108,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
       else
         echo "FAILED PACKAGE UPDATE" >&2
       fi
-      service sshd start
+      rm -f /etc/nologin
     fi
 <% end %>
   fi
@@ -116,6 +116,7 @@ else
   /bin/logger "***** Unable to verify internet connectivity, skipping package updates from userdata"
   touch /.mu-installer-ran-updates
 fi
+rm -f /etc/nologin
 
 AWSCLI='command -v aws'
 PIP='command -v pip'