cloud-mu 3.1.3 → 3.3.0

data/modules/mu/{clouds → providers}/aws/server_pool.rb

@@ -113,14 +113,14 @@ module MU
                 }
               rescue MU::Groomer::RunError => e
                 MU.log "Proceeding after failed initial Groomer run, but #{member.instance_id} may not behave as expected!", MU::WARN, details: e.inspect
-              rescue Exception => e
+              rescue StandardError => e
                 if !member.nil? and !done
                   MU.log "Aborted before I could finish setting up #{@config['name']}, cleaning it up. Stack trace will print once cleanup is complete.", MU::WARN if !@deploy.nocleanup
                   MU::MommaCat.unlockAll
                   if !@deploy.nocleanup
                     Thread.new {
                       MU.dupGlobals(parent_thread_id)
-                      MU::Cloud::AWS::Server.terminateInstance(id: member.instance_id)
+                      MU::Cloud.resourceClass("AWS", "Server").terminateInstance(id: member.instance_id)
                     }
                   end
                 end
@@ -193,9 +193,10 @@ module MU
         # @return [Array<MU::Cloud::Server>]
         def listNodes
           nodes = []
-          me = MU::Cloud::AWS::ServerPool.find(cloud_id: cloud_id)
-          if me and me.first and me.first.instances
-            me.first.instances.each { |instance|
+          me = MU::Cloud::AWS::ServerPool.find(cloud_id: cloud_id).values.first
+          pp me
+          if me and me.instances
+            me.instances.each { |instance|
               found = MU::MommaCat.findStray("AWS", "server", cloud_id: instance.instance_id, region: @config["region"], dummy_ok: true)
               nodes.concat(found)
             }
@@ -293,7 +294,7 @@ module MU
             MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).create_or_update_tags(tag_conf)
             current.instances.each { |instance|
               tag_conf[:tags].each { |t|
-                MU::MommaCat.createTag(instance.instance_id, t[:key], t[:value], region: @config['region'], credentials: @config['credentials'])
+                MU::Cloud::AWS.createTag(instance.instance_id, t[:key], t[:value], region: @config['region'], credentials: @config['credentials'])
               }
             }
           end
@@ -305,13 +306,11 @@ module MU
           asg_options[:min_size] = @config["min_size"]
           asg_options[:max_size] = @config["max_size"]
           asg_options[:new_instances_protected_from_scale_in] = (@config['scale_in_protection'] == "all")
-          tg_arns = []
           if asg_options[:target_group_arns]
             MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).attach_load_balancer_target_groups(
               auto_scaling_group_name: @mu_name,
               target_group_arns: asg_options[:target_group_arns]
             )
-            tg_arns = asg_options[:target_group_arns].dup
             asg_options.delete(:target_group_arns)
           end
 
@@ -365,7 +364,6 @@ module MU
                 policy_params[:target_tracking_configuration].delete(:preferred_target_group)
                 if policy_params[:target_tracking_configuration][:predefined_metric_specification] and
                    policy_params[:target_tracking_configuration][:predefined_metric_specification][:predefined_metric_type] == "ALBRequestCountPerTarget"
-                  lb_path = nil
                   lb = @deploy.deployment["loadbalancers"].values.first
                   if @deploy.deployment["loadbalancers"].size > 1
                     MU.log "Multiple load balancers attached to Autoscale group #{@mu_name}, guessing wildly which one to use for TargetTrackingScaling policy", MU::WARN
@@ -415,7 +413,7 @@ module MU
               }
               if !policy_already_correct
                 MU.log "Putting scaling policy #{policy_name} for #{@mu_name}", MU::NOTICE, details: policy_params
-                resp = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).put_scaling_policy(policy_params)
+                MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).put_scaling_policy(policy_params)
               end
 
             }
@@ -423,12 +421,16 @@ module MU
 
         end
 
+        @cloud_desc_cache = nil
         # Retrieve the AWS descriptor for this Autoscale group
         # @return [OpenStruct]
-        def cloud_desc
-          MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_auto_scaling_groups(
+        def cloud_desc(use_cache: true)
+          return @cloud_desc_cache if @cloud_desc_cache and use_cache
+          return nil if !@cloud_id
+          @cloud_desc_cache = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_auto_scaling_groups(
             auto_scaling_group_names: [@mu_name]
           ).auto_scaling_groups.first
+          @cloud_desc_cache
         end
 
         # Canonical Amazon Resource Number for this resource
@@ -486,7 +488,7 @@ module MU
       # Reverse-map our cloud description into a runnable config hash.
       # We assume that any values we have in +@config+ are placeholders, and
       # calculate our own accordingly based on what's live in the cloud.
-      def toKitten(rootparent: nil, billing: nil, habitats: nil)
+      def toKitten(**_args)
         bok = {
           "cloud" => "AWS",
           "credentials" => @config['credentials'],
@@ -504,7 +506,7 @@ module MU
             bok['tags'] ||= []
             bok['tags'] << { "key" => tag.key, "value" => tag.value }
           }
-          realname = MU::Adoption.tagsToName(bok['tags'])
+          realname = MU::Adoption.tagsToName(bok['tags'], basename: @cloud_id)
           if realname
             bok['name'] = realname
             bok['name'].gsub!(/[^a-zA-Z0-9_\-]/, "_")
@@ -512,25 +514,56 @@ module MU
         end
         bok['name'] ||= @cloud_id
 
-#        if cloud_desc.vpc_id
-#          bok['vpc'] = MU::Config::Ref.get(
-#            id: cloud_desc.vpc_id,
-#            cloud: "AWS",
-#            credentials: @credentials,
-#            type: "vpcs",
-#          )
-#        end
+        bok['min_size'] = cloud_desc.min_size
+        bok['max_size'] = cloud_desc.max_size
 
-        MU.log @cloud_id, MU::NOTICE, details: cloud_desc
+        if cloud_desc.launch_configuration_name
+          launch = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @credentials).describe_launch_configurations(
+            launch_configuration_names: [cloud_desc.launch_configuration_name]
+          ).launch_configurations.first
+          bok['basis'] = {
+            "launch_config" => {
+              "image_id" => launch.image_id,
+              "name" => bok['name'],
+              "size" => launch.instance_type
+            }
+          }
+        end
+
+        if cloud_desc.vpc_zone_identifier and
+           !cloud_desc.vpc_zone_identifier.empty?
+          nets = cloud_desc.vpc_zone_identifier.split(/,/)
+          begin
+            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @credentials).describe_subnets(subnet_ids: nets).subnets.first
+            bok['vpc'] = MU::Config::Ref.get(
+              id: resp.vpc_id,
+              cloud: "AWS",
+              credentials: @credentials,
+              type: "vpcs",
+              subnets: nets.map { |s| { "subnet_id" => s } }
+            )
+          rescue Aws::EC2::Errors::InvalidSubnetIDNotFound => e
+            if e.message.match(/The subnet ID '(subnet-[a-f0-9]+)' does not exist/)
+              nets.delete(Regexp.last_match[1])
+              if nets.empty?
+                MU.log "Autoscale Group #{@cloud_id} was configured for a VPC, but the configuration held no valid subnets", MU::WARN, details: cloud_desc.vpc_zone_identifier.split(/,/)
+              end
+            else
+              raise e
+            end
+          end
+        end
+
+#        MU.log @cloud_id, MU::NOTICE, details: cloud_desc
 
         bok
       end
 
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
 
           term_policies = MU::Cloud::AWS.credConfig ? MU::Cloud::AWS.autoscale.describe_termination_policy_types.termination_policy_types : ["AllocationStrategy", "ClosestToNextInstanceHour", "Default", "NewestInstance", "OldestInstance", "OldestLaunchConfiguration", "OldestLaunchTemplate"]
@@ -793,26 +826,7 @@ module MU
                 }
               }
             },
-            "ingress_rules" => {
-              "items" => {
-                "properties" => {
-                  "sgs" => {
-                    "type" => "array",
-                    "items" => {
-                      "description" => "Other AWS Security Groups; resources that are associated with this group will have this rule applied to their traffic",
-                      "type" => "string"
-                    }
-                  },
-                  "lbs" => {
-                    "type" => "array",
-                    "items" => {
-                      "description" => "AWS Load Balancers which will have this rule applied to their traffic",
-                      "type" => "string"
-                    }
-                  }
-                }
-              }
-            }
+            "ingress_rules" => MU::Cloud.resourceClass("AWS", "FirewallRule").ingressRuleAddtlSchema
           }
           [toplevel_required, schema]
         end
@@ -846,7 +860,7 @@ module MU
                 next if !s[time]
                 begin
                   Time.parse(s[time])
-                rescue Exception => e
+                rescue StandardError => e
                   MU.log "Failed to parse #{time} '#{s[time]}' in scheduled action for AutoScale group #{pool['name']}: #{e.message}", MU::ERR
                   ok = false
                 end
@@ -885,7 +899,7 @@ module MU
             launch = pool["basis"]["launch_config"]
             launch['iam_policies'] ||= pool['iam_policies']
 
-            launch['size'] = MU::Cloud::AWS::Server.validateInstanceType(launch["size"], pool["region"])
+            launch['size'] = MU::Cloud.resourceClass("AWS", "Server").validateInstanceType(launch["size"], pool["region"])
             ok = false if launch['size'].nil?
             if !launch['generate_iam_role']
               if !launch['iam_role'] and pool['cloud'] != "CloudFormation"
@@ -929,11 +943,7 @@ module MU
 
               role['credentials'] = pool['credentials'] if pool['credentials']
               configurator.insertKitten(role, "roles")
-              pool["dependencies"] ||= []
-              pool["dependencies"] << {
-                "type" => "role",
-                "name" => pool["name"]
-              }
+              MU::Config.addDependency(pool, pool['name'], "role")
             end
             launch["ami_id"] ||= launch["image_id"]
             if launch["server"].nil? and launch["instance_id"].nil? and launch["ami_id"].nil?
@@ -947,7 +957,7 @@ module MU
               end
             end
             if launch["server"] != nil
-              pool["dependencies"] << {"type" => "server", "name" => launch["server"]}
+              MU::Config.addDependency(pool, launch["server"], "server", phase: "groom")
 # XXX I dunno, maybe toss an error if this isn't done already
 #              servers.each { |server|
 #                if server["name"] == launch["server"]
@@ -1053,7 +1063,9 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+          MU.log "AWS::ServerPool.cleanup: need to support flags['known']", MU::DEBUG, details: flags
+
           filters = [{name: "key", values: ["MU-ID"]}]
           if !ignoremaster
             filters << {name: "key", values: ["MU-MASTER-IP"]}
@@ -1074,7 +1086,7 @@ module MU
             if asg.key == "MU-MASTER-IP" and asg.value != MU.mu_public_ip and !ignoremaster
               no_purge << asg.resource_id
             end
-            if asg.key == "MU-ID" and asg.value == MU.deploy_id
+            if asg.key == "MU-ID" and asg.value == deploy_id
               maybe_purge << asg.resource_id
             end
           }
@@ -1101,7 +1113,7 @@ module MU
               end
             end
 
-#            MU::Cloud::AWS::Server.removeIAMProfile(resource_id)
+#            MU::Cloud.resourceClass("AWS", "Server").removeIAMProfile(resource_id)
 
             # Generally there should be a launch_configuration of the same name
             # XXX search for these independently, too?
@@ -1134,7 +1146,6 @@ module MU
           instance_secret = Password.random(50)
           @deploy.saveNodeSecret("default", instance_secret, "instance_secret")
 
-          nodes_name = @deploy.getResourceName(@config['basis']["launch_config"]["name"])
           if !@config['basis']['launch_config']["server"].nil?
             #XXX this isn't how we find these; use findStray or something
             if @deploy.deployment["images"].nil? or @deploy.deployment["images"][@config['basis']['launch_config']["server"]].nil?
@@ -1143,14 +1154,14 @@ module MU
             @config['basis']['launch_config']["ami_id"] = @deploy.deployment["images"][@config['basis']['launch_config']["server"]]["image_id"]
             MU.log "Using AMI '#{@config['basis']['launch_config']["ami_id"]}' from sibling server #{@config['basis']['launch_config']["server"]} in ServerPool #{@mu_name}"
           elsif !@config['basis']['launch_config']["instance_id"].nil?
-            @config['basis']['launch_config']["ami_id"] = MU::Cloud::AWS::Server.createImage(
+            @config['basis']['launch_config']["ami_id"] = MU::Cloud.resourceClass("AWS", "Server").createImage(
               name: @mu_name,
               instance_id: @config['basis']['launch_config']["instance_id"],
               credentials: @config['credentials'],
               region: @config['region']
             )[@config['region']]
           end
-          MU::Cloud::AWS::Server.waitForAMI(@config['basis']['launch_config']["ami_id"], credentials: @config['credentials'])
+          MU::Cloud.resourceClass("AWS", "Server").waitForAMI(@config['basis']['launch_config']["ami_id"], credentials: @config['credentials'])
 
           oldlaunch = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_launch_configurations(
             launch_configuration_names: [@mu_name]
@@ -1205,12 +1216,12 @@ module MU
                   vol.delete("encrypted")
                 end
               end
-              mapping, cfm_mapping = MU::Cloud::AWS::Server.convertBlockDeviceMapping(vol)
+              mapping, _cfm_mapping = MU::Cloud.resourceClass("AWS", "Server").convertBlockDeviceMapping(vol)
               storage << mapping
             }
           end
 
-          storage.concat(MU::Cloud::AWS::Server.ephemeral_mappings)
+          storage.concat(MU::Cloud.resourceClass("AWS", "Server").ephemeral_mappings)
 
           if @config['basis']['launch_config']['generate_iam_role']
             role = @deploy.findLitterMate(name: @config['name'], type: "roles")
@@ -1322,7 +1333,7 @@ module MU
             MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).create_launch_configuration(launch_options)
           rescue Aws::AutoScaling::Errors::ValidationError => e
             if lc_attempts > 3
-              MU.log "Got error while creating #{@mu_name} Launch Config#{@config['credentials'] ? " with credentials #{@config['credentials']}" : ""}: #{e.message}, retrying in 10s", MU::WARN, details: launch_options.reject { |k,v | k == :user_data }
+              MU.log "Got error while creating #{@mu_name} Launch Config#{@config['credentials'] ? " with credentials #{@config['credentials']}" : ""}: #{e.message}, retrying in 10s", MU::WARN, details: launch_options.reject { |k,_v | k == :user_data }
             end
             sleep 5
             lc_attempts += 1
@@ -1411,7 +1422,7 @@ module MU
                   if lb_name == lb['concurrent_load_balancer']
                     lbs << deployed_lb["awsname"] # XXX check for classic
                     if deployed_lb.has_key?("targetgroups")
-                      deployed_lb["targetgroups"].each_pair { |tg_name, tg_arn|
+                      deployed_lb["targetgroups"].values.each { |tg_arn|
                         tg_arns << tg_arn
                       }
                     end
@@ -1464,7 +1475,6 @@ module MU
 
 
           if @config['basis']["server"]
-            nodes_name = @deploy.getResourceName(@config['basis']["server"])
             srv_name = @config['basis']["server"]
 # XXX cloudformation bits
             if @deploy.deployment['servers'] != nil and
@@ -1473,7 +1483,6 @@ module MU
             end
           elsif @config['basis']["instance_id"]
             # TODO should go fetch the name tag or something
-            nodes_name = @deploy.getResourceName(@config['basis']["instance_id"].gsub(/-/, ""))
 # XXX cloudformation bits
             asg_options[:instance_id] = @config['basis']["instance_id"]
           end

data/modules/mu/{clouds → providers}/aws/storage_pool.rb

@@ -67,7 +67,7 @@ module MU
                 if target['vpc']["subnet_name"]
                   subnet_obj = vpc.getSubnet(name: target['vpc']["subnet_name"])
                   if subnet_obj.nil?
-                    raise MuError, "Failed to locate subnet from #{subnet} in StoragePool #{@config['name']}:#{target['name']}"
+                    raise MuError, "Failed to locate subnet from #{target['vpc']["subnet_name"]} in StoragePool #{@config['name']}:#{target['name']}"
                   end
                   target['vpc']['subnet_id'] = subnet_obj.cloud_id
                 end
@@ -246,7 +246,7 @@ module MU
           end
 
           security_groups.uniq!
-          resp = MU::Cloud::AWS.efs(region: region).modify_mount_target_security_groups(
+          MU::Cloud::AWS.efs(region: region).modify_mount_target_security_groups(
             mount_target_id: cloud_id,
             security_groups: security_groups
           )
@@ -254,7 +254,6 @@ module MU
 
         # Register a description of this storage pool with this deployment's metadata.
         def notify
-          deploy_struct = {}
           storage_pool = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_file_systems(
             creation_token: @mu_name
           ).file_systems.first
@@ -262,49 +261,29 @@ module MU
           targets = {}
 
           if @config['mount_points'] && !@config['mount_points'].empty?
+            mount_targets = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_mount_targets(
+              file_system_id: storage_pool.file_system_id
+            ).mount_targets
+
             @config['mount_points'].each { |mp|
               subnet = nil
               dependencies
-              mp_vpc = if mp['vpc'] and mp['vpc']['vpc_name']
-                @deploy.findLitterMate(type: "vpc", name: mp['vpc']['vpc_name'], credentials: @config['credentials'])
-              elsif mp['vpc']
-                MU::MommaCat.findStray(
-                  @config['cloud'],
-                  "vpcs",
-                  deploy_id: mp['vpc']["deploy_id"],
-                  credentials: @config['credentials'],
-                  mu_name: mp['vpc']["mu_name"],
-                  cloud_id: mp['vpc']['vpc_id'],
-                  region: @config['region'],
-                  dummy_ok: false
-                ).first
-# XXX non-sibling, findStray version
-              end
+              mp_vpc = MU::Config::Ref.get(mp['vpc']).kitten
 
-              mount_targets = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_mount_targets(
-                file_system_id: storage_pool.file_system_id
-              ).mount_targets
 
-#              subnet_obj = mp_vpc.subnets.select { |s|
-#                s.name == mp["vpc"]["subnet_name"] or s.cloud_id == mp["vpc"]["subnet_id"]
-#              }.first
+              subnet_obj = mp_vpc.subnets.select { |s|
+                s.name == mp["vpc"]["subnet_name"] or s.cloud_id == mp["vpc"]["subnet_id"]
+              }.first
               mount_target = nil
-              mp_vpc.subnets.each { |subnet_obj|
-                mount_targets.map { |t|
-                  subnet_cidr_obj = NetAddr::IPv4Net.parse(subnet_obj.ip_block)
-                  if subnet_cidr_obj.contains(NetAddr::IPv4.parse(t.ip_address))
-                    mount_target = t
-                    subnet = subnet_obj.cloud_desc
-                  end
-                }
-                break if mount_target
+              mount_targets.each { |t|
+                subnet_cidr_obj = NetAddr::IPv4Net.parse(subnet_obj.ip_block)
+                if subnet_cidr_obj.contains(NetAddr::IPv4.parse(t.ip_address))
+                  mount_target = t
+                  subnet = subnet_obj.cloud_desc
+                  break
+                end
               }
 
-              # mount_target = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_mount_targets(
-                # mount_target_id: mp["cloud_id"]
-              # ).mount_targets.first
-
-
               targets[mp["name"]] = {
                 "owner_id" => mount_target.owner_id,
                 "cloud_id" => mount_target.mount_target_id,
@@ -354,7 +333,9 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region in which to operate
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+          MU.log "AWS::StoragePool.cleanup: need to support flags['known']", MU::DEBUG, details: flags
+
           supported_regions = %w{us-west-2 us-east-1 eu-west-1}
           if supported_regions.include?(region)
             begin 
@@ -367,7 +348,6 @@ module MU
             end
 
             our_pools = []
-            our_replication_group_ids = []
 
             if !storage_pools.empty?
               storage_pools.each{ |pool|
@@ -378,7 +358,7 @@ module MU
                 found_muid = false
                 found_master = false
                 tags.each { |tag|
-                  found_muid = true if tag.key == "MU-ID" && tag.value == MU.deploy_id
+                  found_muid = true if tag.key == "MU-ID" && tag.value == deploy_id
                   found_master = true if tag.key == "MU-MASTER-IP" && tag.value == MU.mu_public_ip
                 }
                 next if !found_muid
@@ -445,9 +425,9 @@ module MU
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
             "ingress_rules" => {
@@ -493,6 +473,9 @@ module MU
 
           if pool['mount_points'] && !pool['mount_points'].empty?
             pool['mount_points'].each{ |mp|
+              if mp['vpc'] and mp['vpc']['name']
+                MU::Config.addDependency(pool, mp['vpc']['name'], "vpc")
+              end
               if mp['ingress_rules']
                 fwname = "storage-#{mp['name']}"
                 acl = {
@@ -515,7 +498,6 @@ module MU
           ok
         end
 
-        private
       end #class
     end #class
   end