cloud-mu 3.1.6 → 3.2.0

data/modules/mu/{clouds → providers}/aws/search_domain.rb

@@ -156,8 +156,8 @@ module MU
             begin
               resp = MU::Cloud::AWS.iam(credentials: credentials).list_roles(marker: marker)
               resp.roles.each{ |role|
-                # XXX Maybe we should have a more generic way to delete IAM profiles and policies. The call itself should be moved from MU::Cloud::AWS::Server.
-#                MU::Cloud::AWS::Server.removeIAMProfile(role.role_name) if role.role_name.match(/^#{Regexp.quote(MU.deploy_id)}/)
+                # XXX Maybe we should have a more generic way to delete IAM profiles and policies. The call itself should be moved from MU::Cloud.resourceClass("AWS", "Server").
+#                MU::Cloud.resourceClass("AWS", "Server").removeIAMProfile(role.role_name) if role.role_name.match(/^#{Regexp.quote(MU.deploy_id)}/)
               }
               marker = resp.marker
             end while resp.is_truncated
@@ -378,9 +378,9 @@ module MU
 
           if dom['slow_logs']
             if configurator.haveLitterMate?(dom['slow_logs'], "log")
-              dom['dependencies'] << { "name" => dom['slow_logs'], "type" => "log" }
+              MU::Config.addDependency(dom, dom['slow_logs'], "log")
             else
-              log_group = MU::Cloud::AWS::Log.find(cloud_id: dom['slow_logs'], region: dom['region']).values.first
+              log_group = MU::Cloud.resourceClass("AWS", "Log").find(cloud_id: dom['slow_logs'], region: dom['region']).values.first
               if !log_group
                 MU.log "Specified slow_logs CloudWatch log group '#{dom['slow_logs']}' in SearchDomain '#{dom['name']}' doesn't appear to exist", MU::ERR
                 ok = false
@@ -395,7 +395,7 @@ module MU
               "credentials" => dom['credentials']
             }
             ok = false if !configurator.insertKitten(log_group, "logs")
-            dom['dependencies'] << { "name" => dom['slow_logs'], "type" => "log" }
+            MU::Config.addDependency(dom, dom['slow_logs'], "log")
           end
 
           if dom['advanced_options']
@@ -456,12 +456,7 @@ module MU
                 ]
               }
               configurator.insertKitten(roledesc, "roles")
-
-              dom['dependencies'] ||= []
-              dom['dependencies'] << {
-                "type" => "role",
-                "name" => dom['name']+"cognitorole"
-              }
+              MU::Config.addDependency(dom, dom['name']+"cognitorole", "role")
             end
 
           end
@@ -525,7 +520,7 @@ module MU
               arn = @config['slow_logs']
             else
               log_group = @deploy.findLitterMate(type: "log", name: @config['slow_logs'])
-              log_group = MU::Cloud::AWS::Log.find(cloud_id: log_group.mu_name, region: log_group.cloudobj.config['region']).values.first
+              log_group = MU::Cloud.resourceClass("AWS", "Log").find(cloud_id: log_group.mu_name, region: log_group.cloudobj.config['region']).values.first
               if log_group.nil? or log_group.arn.nil?
                 raise MuError, "Failed to retrieve ARN of sibling LogGroup '#{@config['slow_logs']}'"
               end
@@ -552,7 +547,7 @@ module MU
               params[:log_publishing_options]["SEARCH_SLOW_LOGS"] = {}
               params[:log_publishing_options]["SEARCH_SLOW_LOGS"][:enabled] = true
               params[:log_publishing_options]["SEARCH_SLOW_LOGS"][:cloud_watch_logs_log_group_arn] = arn
-              MU::Cloud::AWS::Log.allowService("es.amazonaws.com", arn, @config['region'])
+              MU::Cloud.resourceClass("AWS", "Log").allowService("es.amazonaws.com", arn, @config['region'])
             end
           end
 

data/modules/mu/{clouds → providers}/aws/server.rb

@@ -145,7 +145,7 @@ module MU
               raise MuError, "My second argument should be a hash of variables to pass into ERB templates"
             end
             $mu = OpenStruct.new(template_variables)
-            userdata_dir = File.expand_path(MU.myRoot+"/modules/mu/clouds/aws/userdata")
+            userdata_dir = File.expand_path(MU.myRoot+"/modules/mu/providers/aws/userdata")
             platform = "linux" if %w{centos centos6 centos7 ubuntu ubuntu14 rhel rhel7 rhel71 amazon}.include? platform
             platform = "windows" if %w{win2k12r2 win2k12 win2k8 win2k8r2 win2k16}.include? platform
             erbfile = "#{userdata_dir}/#{platform}.erb"
@@ -299,7 +299,7 @@ module MU
             raise MuError, "Got null subnet id out of #{@config['vpc']}"
           end
           MU.log "Deploying #{@mu_name} into VPC #{@vpc.cloud_id} Subnet #{subnet.cloud_id}"
-          punchAdminNAT
+          allowBastionAccess
           instance_descriptor[:subnet_id] = subnet.cloud_id
         end
 
@@ -399,13 +399,13 @@ module MU
       # Figure out what's needed to SSH into this server.
       # @return [Array<String>]: nat_ssh_key, nat_ssh_user, nat_ssh_host, canonical_ip, ssh_user, ssh_key_name, alternate_names
       def getSSHConfig
-        describe(cloud_id: @cloud_id)
+        cloud_desc(use_cache: false) # make sure we're current
 # XXX add some awesome alternate names from metadata and make sure they end
 # up in MU::MommaCat's ssh config wangling
         return nil if @config.nil? or @deploy.nil?
 
         nat_ssh_key = nat_ssh_user = nat_ssh_host = nil
-        if !@config["vpc"].nil? and !MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
+        if !@config["vpc"].nil? and !MU::Cloud.resourceClass("AWS", "VPC").haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
           if !@nat.nil?
             if @nat.is_a?(Struct) && @nat.nat_gateway_id && @nat.nat_gateway_id.start_with?("nat-")
               raise MuError, "Configured to use NAT Gateway, but I have no route to instance. Either use Bastion, or configure VPC peering"
@@ -444,8 +444,7 @@ module MU
       # administravia for a new instance.
       def postBoot(instance_id = nil)
         @cloud_id ||= instance_id
-        node, _config, deploydata = describe(cloud_id: @cloud_id)
-        @mu_name ||= node
+        _node, _config, deploydata = describe(cloud_id: @cloud_id)
 
         raise MuError, "Couldn't find instance #{@mu_name} (#{@cloud_id})" if !cloud_desc
         return false if !MU::MommaCat.lock(@cloud_id+"-orchestrate", true)
@@ -482,7 +481,7 @@ module MU
           end
         }
 
-        punchAdminNAT
+        allowBastionAccess
 
         setAlarms
 
@@ -615,7 +614,7 @@ module MU
           return nil
         end
 
-        asgs = MU::Cloud::AWS::ServerPool.find(
+        asgs = MU::Cloud.resourceClass("AWS", "ServerPool").find(
           instance_id: @cloud_id,
           region: @config['region'],
           credentials: @credentials
@@ -725,15 +724,15 @@ module MU
 
           if int.groups.size > 0
 
-            require 'mu/clouds/aws/firewall_rule'
-            ifaces = MU::Cloud::AWS::FirewallRule.getAssociatedInterfaces(int.groups.map { |sg| sg.group_id }, credentials: @credentials, region: @config['region'])
+            require 'mu/providers/aws/firewall_rule'
+            ifaces = MU::Cloud.resourceClass("AWS", "FirewallRule").getAssociatedInterfaces(int.groups.map { |sg| sg.group_id }, credentials: @credentials, region: @config['region'])
             done_local_rules = false
             int.groups.each { |sg|
               if !done_local_rules and ifaces[sg.group_id].size == 1
-                sg_desc = MU::Cloud::AWS::FirewallRule.find(cloud_id: sg.group_id, credentials: @credentials, region: @config['region']).values.first
+                sg_desc = MU::Cloud.resourceClass("AWS", "FirewallRule").find(cloud_id: sg.group_id, credentials: @credentials, region: @config['region']).values.first
                 if sg_desc
-                  bok["ingress_rules"] = MU::Cloud::AWS::FirewallRule.rulesToBoK(sg_desc.ip_permissions)
-                  bok["ingress_rules"].concat(MU::Cloud::AWS::FirewallRule.rulesToBoK(sg_desc.ip_permissions_egress, egress: true))
+                  bok["ingress_rules"] = MU::Cloud.resourceClass("AWS", "FirewallRule").rulesToBoK(sg_desc.ip_permissions)
+                  bok["ingress_rules"].concat(MU::Cloud.resourceClass("AWS", "FirewallRule").rulesToBoK(sg_desc.ip_permissions_egress, egress: true))
                   done_local_rules = true
                   next
                 end
@@ -802,44 +801,13 @@ module MU
           end
           deploydata["region"] = @config['region'] if !@config['region'].nil?
           if !@named
-            MU::MommaCat.nameKitten(self)
+            MU::MommaCat.nameKitten(self, no_dns: true)
             @named = true
           end
 
           return deploydata
         end
 
-        # If the specified server is in a VPC, and has a NAT, make sure we'll
-        # be letting ssh traffic in from said NAT.
-        def punchAdminNAT
-          if @config['vpc'].nil? or 
-            (
-              !@config['vpc'].has_key?("nat_host_id") and
-              !@config['vpc'].has_key?("nat_host_tag") and
-              !@config['vpc'].has_key?("nat_host_ip") and
-              !@config['vpc'].has_key?("nat_host_name")
-            )
-            return nil
-          end
-
-          return nil if @nat.is_a?(Struct) && @nat.nat_gateway_id && @nat.nat_gateway_id.start_with?("nat-")
-
-          dependencies if @nat.nil?
-          if @nat.nil? or @nat.cloud_desc.nil?
-            raise MuError, "#{@mu_name} (#{MU.deploy_id}) is configured to use #{@config['vpc']} but I can't find the cloud descriptor for a matching NAT instance"
-          end
-          MU.log "Adding administrative holes for NAT host #{@nat.cloud_desc.private_ip_address} to #{@mu_name}"
-          if !@deploy.kittens['firewall_rules'].nil?
-            @deploy.kittens['firewall_rules'].values.each { |acl|
-              if acl.config["admin"]
-                acl.addRule([@nat.cloud_desc.private_ip_address], proto: "tcp")
-                acl.addRule([@nat.cloud_desc.private_ip_address], proto: "udp")
-                acl.addRule([@nat.cloud_desc.private_ip_address], proto: "icmp")
-              end
-            }
-          end
-        end
-
         # Called automatically by {MU::Deploy#createResources}
         def groom
           MU::MommaCat.lock(@cloud_id+"-groom")
@@ -851,7 +819,7 @@ module MU
             end
           end
 
-          punchAdminNAT
+          allowBastionAccess
 
           tagVolumes
 
@@ -883,12 +851,25 @@ module MU
 
           begin
             getIAMProfile
+
+            dbs = @deploy.findLitterMate(type: "database", return_all: true)
+            if dbs
+              dbs.each_pair { |sib_name, sib|
+                @groomer.groomer_class.grantSecretAccess(@mu_name, sib_name, "database_credentials")
+                if sib.config and sib.config['auth_vault']
+                  @groomer.groomer_class.grantSecretAccess(@mu_name, sib.config['auth_vault']['vault'], sib.config['auth_vault']['item'])
+                end
+              }
+            end
+
             if @config['groom'].nil? or @config['groom']
               @groomer.run(purpose: "Full Initial Run", max_retries: 15, reboot_first_fail: (windows? and @config['groomer'] != "Ansible"), timeout: @config['groomer_timeout'])
             end
           rescue MU::Groomer::RunError => e
+            raise e if !@config['create_image'].nil? and !@config['image_created']
             MU.log "Proceeding after failed initial Groomer run, but #{@mu_name} may not behave as expected!", MU::WARN, details: e.message
           rescue StandardError => e
+            raise e if !@config['create_image'].nil? and !@config['image_created']
             MU.log "Caught #{e.inspect} on #{@mu_name} in an unexpected place (after @groomer.run on Full Initial Run)", MU::ERR
           end
 
@@ -961,7 +942,7 @@ module MU
           # Our deploydata gets corrupted often with server pools, this will cause us to use the wrong IP to identify a node
           # which will cause us to create certificates, DNS records and other artifacts with incorrect information which will cause our deploy to fail.
           # The cloud_id is always correct so lets use 'cloud_desc' to get the correct IPs
-          if MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials']) or @deploydata["public_ip_address"].nil?
+          if MU::Cloud.resourceClass("AWS", "VPC").haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials']) or @deploydata["public_ip_address"].nil?
             @config['canonical_ip'] = cloud_desc.private_ip_address
             @deploydata["private_ip_address"] = cloud_desc.private_ip_address
             return cloud_desc.private_ip_address
@@ -1181,10 +1162,7 @@ module MU
             end
           end
 
-          if @cloud_id.nil?
-            describe
-            @cloud_id = cloud_desc.instance_id
-          end
+          @cloud_id ||= cloud_desc(use_cache: false).instance_id
           ssh_keydir = "#{Etc.getpwuid(Process.uid).dir}/.ssh"
           ssh_key_name = @deploy.ssh_key_name
 
@@ -1318,7 +1296,7 @@ module MU
 
           if @deploy
             MU::Cloud::AWS.createStandardTags(
-              resource_id,
+              creation.volume_id,
               region: @config['region'],
               credentials: @config['credentials'],
               optional: @config['optional_tags'],
@@ -1577,7 +1555,11 @@ module MU
           return if !instance
 
           id ||= instance.instance_id
-          MU::MommaCat.lock(".cleanup-"+id)
+          begin
+            MU::MommaCat.lock(".cleanup-"+id)
+          rescue Errno::ENOENT => e
+            MU.log "No lock for terminating instance #{id} due to missing metadata", MU::DEBUG
+          end
 
           ips, names = getAddresses(instance, region: region, credentials: credentials)
           targets = ips +names
@@ -1632,7 +1614,11 @@ module MU
           end
 
           MU.log "#{instance.instance_id}#{server_obj ? " ("+server_obj.mu_name+")" : ""} terminated" if !noop
-          MU::MommaCat.unlock(".cleanup-"+id)
+          begin
+            MU::MommaCat.unlock(".cleanup-"+id)
+          rescue Errno::ENOENT => e
+            MU.log "No lock for terminating instance #{id} due to missing metadata", MU::DEBUG
+          end
 
         end
 
@@ -1690,26 +1676,7 @@ module MU
                 "type" => "object"
               }
             },
-            "ingress_rules" => {
-              "items" => {
-                "properties" => {
-                  "sgs" => {
-                    "type" => "array",
-                    "items" => {
-                      "description" => "Other AWS Security Groups; resources that are associated with this group will have this rule applied to their traffic",
-                      "type" => "string"
-                    }
-                  },
-                  "lbs" => {
-                    "type" => "array",
-                    "items" => {
-                      "description" => "AWS Load Balancers which will have this rule applied to their traffic",
-                      "type" => "string"
-                    }
-                  }
-                }
-              }
-            },
+            "ingress_rules" => MU::Cloud.resourceClass("AWS", "FirewallRule").ingressRuleAddtlSchema,
             "ssh_user" => {
               "type" => "string",
               "default" => "root",
@@ -1777,8 +1744,7 @@ module MU
 
             MU::Cloud.availableClouds.each { |cloud|
               next if cloud == "AWS"
-              cloudbase = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-              foreign_types = (cloudbase.listInstanceTypes).values.first
+              foreign_types = (MU::Cloud.cloudClass(cloud).listInstanceTypes).values.first
               if foreign_types.size == 1
                 foreign_types = foreign_types.values.first
               end
@@ -1845,12 +1811,7 @@ module MU
           end
 
           configurator.insertKitten(role, "roles")
-
-          server["dependencies"] ||= []
-          server["dependencies"] << {
-            "type" => "role",
-            "name" => server["name"]
-          }
+          MU::Config.addDependency(server, server["name"], "role")
         end
 
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::servers}, bare and unvalidated.
@@ -1901,10 +1862,7 @@ module MU
             server["loadbalancers"].each { |lb|
               lb["name"] ||= lb["concurrent_load_balancer"]
               if lb["name"]
-                server["dependencies"] << {
-                  "type" => "loadbalancer",
-                  "name" => lb["name"]
-                }
+                MU::Config.addDependency(server, lb["name"], "loadbalancer")
               end
             }
           end
@@ -2019,6 +1977,13 @@ module MU
           configured_storage
         end
 
+        # Return all of the IP addresses, public and private, from all of our
+        # network interfaces.
+        # @return [Array<String>]
+        def listIPs
+          MU::Cloud::AWS::Server.getAddresses(cloud_desc).first
+        end
+
         private
 
         def bootstrapGroomer
@@ -2144,7 +2109,7 @@ module MU
             subnet = @vpc.getSubnet(cloud_id: cloud_desc.subnet_id)
 
             _nat_ssh_key, _nat_ssh_user, nat_ssh_host, _canonical_ip, _ssh_user, _ssh_key_name = getSSHConfig
-            if subnet.private? and !nat_ssh_host and !MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
+            if subnet.private? and !nat_ssh_host and !MU::Cloud.resourceClass("AWS", "VPC").haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
               raise MuError, "#{@mu_name} is in a private subnet (#{subnet}), but has no bastion host configured, and I have no other route to it"
             end
 
@@ -2236,15 +2201,15 @@ module MU
               alarm["dimensions"] = [{:name => "InstanceId", :value => @cloud_id}]
 
               if alarm["enable_notifications"]
-                topic_arn = MU::Cloud::AWS::Notification.createTopic(alarm["notification_group"], region: @config["region"], credentials: @config['credentials'])
-                MU::Cloud::AWS::Notification.subscribe(arn: topic_arn, protocol: alarm["notification_type"], endpoint: alarm["notification_endpoint"], region: @config["region"], credentials: @config["credentials"])
+                topic_arn = MU::Cloud.resourceClass("AWS", "Notification").createTopic(alarm["notification_group"], region: @config["region"], credentials: @config['credentials'])
+                MU::Cloud.resourceClass("AWS", "Notification").subscribe(arn: topic_arn, protocol: alarm["notification_type"], endpoint: alarm["notification_endpoint"], region: @config["region"], credentials: @config["credentials"])
                 alarm["alarm_actions"] = [topic_arn]
                 alarm["ok_actions"]  = [topic_arn]
               end
 
               alarm_name = alarm_obj ? alarm_obj.cloud_id : "#{@mu_name}-#{alarm['name']}".upcase
 
-              MU::Cloud::AWS::Alarm.setAlarm(
+              MU::Cloud.resourceClass("AWS", "Alarm").setAlarm(
                 name: alarm_name,
                 ok_actions: alarm["ok_actions"],
                 alarm_actions: alarm["alarm_actions"],

data/modules/mu/{clouds → providers}/aws/server_pool.rb

@@ -120,7 +120,7 @@ module MU
                   if !@deploy.nocleanup
                     Thread.new {
                       MU.dupGlobals(parent_thread_id)
-                      MU::Cloud::AWS::Server.terminateInstance(id: member.instance_id)
+                      MU::Cloud.resourceClass("AWS", "Server").terminateInstance(id: member.instance_id)
                     }
                   end
                 end
@@ -813,26 +813,7 @@ module MU
                 }
               }
             },
-            "ingress_rules" => {
-              "items" => {
-                "properties" => {
-                  "sgs" => {
-                    "type" => "array",
-                    "items" => {
-                      "description" => "Other AWS Security Groups; resources that are associated with this group will have this rule applied to their traffic",
-                      "type" => "string"
-                    }
-                  },
-                  "lbs" => {
-                    "type" => "array",
-                    "items" => {
-                      "description" => "AWS Load Balancers which will have this rule applied to their traffic",
-                      "type" => "string"
-                    }
-                  }
-                }
-              }
-            }
+            "ingress_rules" => MU::Cloud.resourceClass("AWS", "FirewallRule").ingressRuleAddtlSchema
           }
           [toplevel_required, schema]
         end
@@ -905,7 +886,7 @@ module MU
             launch = pool["basis"]["launch_config"]
             launch['iam_policies'] ||= pool['iam_policies']
 
-            launch['size'] = MU::Cloud::AWS::Server.validateInstanceType(launch["size"], pool["region"])
+            launch['size'] = MU::Cloud.resourceClass("AWS", "Server").validateInstanceType(launch["size"], pool["region"])
             ok = false if launch['size'].nil?
             if !launch['generate_iam_role']
               if !launch['iam_role'] and pool['cloud'] != "CloudFormation"
@@ -949,11 +930,7 @@ module MU
 
               role['credentials'] = pool['credentials'] if pool['credentials']
               configurator.insertKitten(role, "roles")
-              pool["dependencies"] ||= []
-              pool["dependencies"] << {
-                "type" => "role",
-                "name" => pool["name"]
-              }
+              MU::Config.addDependency(pool, pool['name'], "role")
             end
             launch["ami_id"] ||= launch["image_id"]
             if launch["server"].nil? and launch["instance_id"].nil? and launch["ami_id"].nil?
@@ -967,7 +944,7 @@ module MU
               end
             end
             if launch["server"] != nil
-              pool["dependencies"] << {"type" => "server", "name" => launch["server"]}
+              MU::Config.addDependency(pool, launch["server"], "server", phase: "groom")
 # XXX I dunno, maybe toss an error if this isn't done already
 #              servers.each { |server|
 #                if server["name"] == launch["server"]
@@ -1123,7 +1100,7 @@ module MU
               end
             end
 
-#            MU::Cloud::AWS::Server.removeIAMProfile(resource_id)
+#            MU::Cloud.resourceClass("AWS", "Server").removeIAMProfile(resource_id)
 
             # Generally there should be a launch_configuration of the same name
             # XXX search for these independently, too?
@@ -1164,14 +1141,14 @@ module MU
             @config['basis']['launch_config']["ami_id"] = @deploy.deployment["images"][@config['basis']['launch_config']["server"]]["image_id"]
             MU.log "Using AMI '#{@config['basis']['launch_config']["ami_id"]}' from sibling server #{@config['basis']['launch_config']["server"]} in ServerPool #{@mu_name}"
           elsif !@config['basis']['launch_config']["instance_id"].nil?
-            @config['basis']['launch_config']["ami_id"] = MU::Cloud::AWS::Server.createImage(
+            @config['basis']['launch_config']["ami_id"] = MU::Cloud.resourceClass("AWS", "Server").createImage(
               name: @mu_name,
               instance_id: @config['basis']['launch_config']["instance_id"],
               credentials: @config['credentials'],
               region: @config['region']
             )[@config['region']]
           end
-          MU::Cloud::AWS::Server.waitForAMI(@config['basis']['launch_config']["ami_id"], credentials: @config['credentials'])
+          MU::Cloud.resourceClass("AWS", "Server").waitForAMI(@config['basis']['launch_config']["ami_id"], credentials: @config['credentials'])
 
           oldlaunch = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_launch_configurations(
             launch_configuration_names: [@mu_name]
@@ -1226,12 +1203,12 @@ module MU
                   vol.delete("encrypted")
                 end
               end
-              mapping, _cfm_mapping = MU::Cloud::AWS::Server.convertBlockDeviceMapping(vol)
+              mapping, _cfm_mapping = MU::Cloud.resourceClass("AWS", "Server").convertBlockDeviceMapping(vol)
               storage << mapping
             }
           end
 
-          storage.concat(MU::Cloud::AWS::Server.ephemeral_mappings)
+          storage.concat(MU::Cloud.resourceClass("AWS", "Server").ephemeral_mappings)
 
           if @config['basis']['launch_config']['generate_iam_role']
             role = @deploy.findLitterMate(name: @config['name'], type: "roles")