cloud-mu 3.1.6 → 3.4.0

data/modules/mu/{clouds → providers}/aws/vpc.rb

@@ -18,7 +18,7 @@ module MU
 
       # Creation of Virtual Private Clouds and associated artifacts (routes, subnets, etc).
       class VPC < MU::Cloud::VPC
-        require 'mu/clouds/aws/vpc_subnet'
+        require 'mu/providers/aws/vpc_subnet'
 
         # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
         # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
@@ -209,7 +209,7 @@ module MU
           if !MU::Cloud::AWS.isGovCloud?(@config['region'])
             mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu", credentials: @config['credentials']).values.first
             if !mu_zone.nil?
-              MU::Cloud::AWS::DNSZone.toggleVPCAccess(id: mu_zone.id, vpc_id: @cloud_id, region: @config['region'], credentials: @config['credentials'])
+              MU::Cloud.resourceClass("AWS", "DNSZone").toggleVPCAccess(id: mu_zone.id, vpc_id: @cloud_id, region: @config['region'], credentials: @config['credentials'])
             end
           end
 					loadSubnets
@@ -822,11 +822,11 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           MU.log "AWS::VPC.cleanup: need to support flags['known']", MU::DEBUG, details: flags
 
           tagfilters = [
-            {name: "tag:MU-ID", values: [MU.deploy_id]}
+            {name: "tag:MU-ID", values: [deploy_id]}
           ]
           if !ignoremaster
             tagfilters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
@@ -838,9 +838,23 @@ module MU
             vpcs = resp if !resp.empty?
           }
 
+#          resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
+#            filters: [
+#              {
+#                name: "requester-vpc-info.vpc-id",
+#                values: [@cloud_id]
+#              },
+#              {
+#                name: "accepter-vpc-info.vpc-id",
+#                values: [peer_id.to_s]
+#              }
+#            ]
+#          )
+
           if !vpcs.empty?
             gwthreads = []
             vpcs.each { |vpc|
+              purge_peering_connections(noop, vpc.vpc_id, region: region, credentials: credentials)
               # NAT gateways don't have any tags, and we can't assign them a name. Lets find them based on a VPC ID
               gwthreads << Thread.new {
                 purge_nat_gateways(noop, vpc_id: vpc.vpc_id, region: region, credentials: credentials)
@@ -862,7 +876,7 @@ module MU
 
 #          unless noop
 #            MU::Cloud::AWS.iam.list_roles.roles.each{ |role|
-#              match_string = "#{MU.deploy_id}.*TRAFFIC-LOG"
+#              match_string = "#{deploy_id}.*TRAFFIC-LOG"
 #            }
 #          end
         end
@@ -916,11 +930,7 @@ module MU
             logdesc["tags"] = vpc["tags"] if !vpc["tags"].nil?
 #            logdesc["optional_tags"] = vpc["optional_tags"] if !vpc["optional_tags"].nil?
             configurator.insertKitten(logdesc, "logs")
-            vpc['dependencies'] ||= []
-            vpc['dependencies'] << {
-              "type" => "log",
-              "name" => vpc['name']+"loggroup"
-            }
+            MU::Config.addDependency(vpc, vpc['name']+"loggroup", "log")
 
             roledesc = {
               "name" => vpc['name']+"logrole",
@@ -958,11 +968,7 @@ module MU
             roledesc["tags"] = vpc["tags"] if !vpc["tags"].nil?
             roledesc["optional_tags"] = vpc["optional_tags"] if !vpc["optional_tags"].nil?
             configurator.insertKitten(roledesc, "roles")
-            vpc['dependencies'] ||= []
-            vpc['dependencies'] << {
-              "type" => "role",
-              "name" => vpc['name']+"logrole"
-            }
+            MU::Config.addDependency(vpc, vpc['name']+"logrole", "role")
           end
 
           subnet_routes = Hash.new
@@ -1013,10 +1019,7 @@ module MU
                 subnet_routes[table['name']].each { |subnet|
                   nat_routes[subnet] = route['nat_host_name']
                 }
-                vpc['dependencies'] << {
-                  "type" => "server",
-                  "name" => route['nat_host_name']
-                }
+                MU::Config.addDependency(vpc, route['nat_host_name'], "server", no_create_wait: true)
               elsif route['gateway'] == '#NAT'
                 vpc['create_nat_gateway'] = true
                 private_rtbs << table['name']
@@ -1225,7 +1228,7 @@ module MU
                   # suits me just fine
                 rescue Aws::EC2::Errors::AuthFailure => e
                   if !tried_lbs and iface.attachment.instance_owner_id == "amazon-elb"
-                    MU::Cloud::AWS::LoadBalancer.cleanup(
+                    MU::Cloud.resourceClass("AWS", "LoadBalancer").cleanup(
                       noop: noop,
                       region: region,
                       credentials: credentials,
@@ -1265,11 +1268,71 @@ module MU
           nil
         end
 
+        # Try to locate the default VPC for a region, and return a BoK-style
+        # config fragment for something that might want to live in it.
+        def self.defaultVpc(region, credentials)
+          cfg_fragment = nil
+          MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_vpcs.vpcs.each { |vpc|
+            if vpc.is_default
+              cfg_fragment = {
+                "id" => vpc.vpc_id,
+                "cloud" => "AWS",
+                "region" => region,
+                "credentials" => credentials
+              }
+              cfg_fragment['subnets'] = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_subnets(
+                filters: [
+                  {
+                    name: "vpc-id",
+                    values: [vpc.vpc_id]
+                  }
+                ]
+              ).subnets.map { |s| { "subnet_id" => s.subnet_id } }
+              break
+            end
+          }
+
+          cfg_fragment
+        end
+
+        # Return a {MU::Config::Ref} that indicates this VPC.
+        # @param subnet_ids [Array<String>]: Optional list of subnet ids with which to infer a +subnet_pref+ parameter.
+        # @return [MU::Config::Ref]
+        def getReference(subnet_ids = [])
+          have_private = have_public = false
+          subnets.each { |s|
+            next if subnet_ids and !subnet_ids.empty? and !subnet_ids.include?(s.cloud_id)
+            if s.private?
+              have_private = true
+            else
+              have_public = true
+            end
+          }
+          subnet_pref = if have_private == have_public
+            "any"
+          elsif have_private
+            "all_private"
+          elsif have_public
+            "all_public"
+          end
+          MU::Config::Ref.get(
+            id: @cloud_id,
+            cloud: "AWS",
+            credentials: @credentials,
+            region: @config['region'],
+            type: "vpcs",
+            subnet_pref: subnet_pref
+          )
+        end
+
         private
 
         def peerWith(peer)
           peer_ref = MU::Config::Ref.get(peer['vpc'])
           peer_obj = peer_ref.kitten
+          if !peer_obj
+            raise MuError.new "#{@mu_name}: Failed to locate my peer VPC", details: peer_ref.to_h
+          end
           peer_id = peer_ref.kitten.cloud_id
           if peer_id == @cloud_id
             MU.log "#{@mu_name} attempted to peer with itself (#{@cloud_id})", MU::ERR, details: peer
@@ -1683,6 +1746,61 @@ module MU
         end
         private_class_method :purge_dhcpopts
 
+        def self.purge_peering_connections(noop, vpc_id, region: MU.curRegion, credentials: nil)
+          my_peer_conns = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_peering_connections(
+            filters: [
+              {
+                name: "requester-vpc-info.vpc-id",
+                values: [vpc_id]
+              }
+            ]
+          ).vpc_peering_connections
+          my_peer_conns.concat(MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_peering_connections(
+            filters: [
+              {
+                name: "accepter-vpc-info.vpc-id",
+                values: [vpc_id]
+              }
+            ]
+          ).vpc_peering_connections)
+
+          my_peer_conns.each { |cnxn|
+            [cnxn.accepter_vpc_info.vpc_id, cnxn.requester_vpc_info.vpc_id].each { |peer_vpc|
+              MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_vpc, region: region, credentials: credentials).each { |rtb_id|
+                begin
+                  resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_route_tables(
+                    route_table_ids: [rtb_id]
+                  )
+                rescue Aws::EC2::Errors::InvalidRouteTableIDNotFound
+                  next
+                end
+                resp.route_tables.each { |rtb|
+                  rtb.routes.each { |route|
+                    if route.vpc_peering_connection_id == cnxn.vpc_peering_connection_id
+                      MU.log "Removing route #{route.destination_cidr_block} from route table #{rtb_id} in VPC #{peer_vpc}"
+                      MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_route(
+                          route_table_id: rtb_id,
+                          destination_cidr_block: route.destination_cidr_block
+                      ) if !noop
+                    end
+                  }
+                }
+              }
+            }
+            MU.log "Deleting VPC peering connection #{cnxn.vpc_peering_connection_id}"
+            begin
+              MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc_peering_connection(
+                vpc_peering_connection_id: cnxn.vpc_peering_connection_id
+              ) if !noop
+            rescue Aws::EC2::Errors::InvalidStateTransition
+              MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} not in removable (state #{cnxn.status.code})", MU::WARN
+            rescue Aws::EC2::Errors::OperationNotPermitted => e
+              MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} refuses to delete: #{e.message}", MU::WARN
+            end
+          }
+        end
+        private_class_method :purge_peering_connections
+
         # Remove all VPCs associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
@@ -1697,60 +1815,10 @@ module MU
           return if vpcs.nil? or vpcs.size == 0
 
           vpcs.each { |vpc|
-            my_peer_conns = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_peering_connections(
-              filters: [
-                {
-                  name: "requester-vpc-info.vpc-id",
-                  values: [vpc.vpc_id]
-                }
-              ]
-            ).vpc_peering_connections
-            my_peer_conns.concat(MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_peering_connections(
-              filters: [
-                {
-                  name: "accepter-vpc-info.vpc-id",
-                  values: [vpc.vpc_id]
-                }
-              ]
-            ).vpc_peering_connections)
-            my_peer_conns.each { |cnxn|
-
-              [cnxn.accepter_vpc_info.vpc_id, cnxn.requester_vpc_info.vpc_id].each { |peer_vpc|
-                MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_vpc, region: region, credentials: credentials).each { |rtb_id|
-                  begin
-                    resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_route_tables(
-                      route_table_ids: [rtb_id]
-                    )
-                  rescue Aws::EC2::Errors::InvalidRouteTableIDNotFound
-                    next
-                  end
-                  resp.route_tables.each { |rtb|
-                    rtb.routes.each { |route|
-                      if route.vpc_peering_connection_id == cnxn.vpc_peering_connection_id
-                        MU.log "Removing route #{route.destination_cidr_block} from route table #{rtb_id} in VPC #{peer_vpc}"
-                        MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_route(
-                            route_table_id: rtb_id,
-                            destination_cidr_block: route.destination_cidr_block
-                        ) if !noop
-                      end
-                    }
-                  }
-                }
-              }
-              MU.log "Deleting VPC peering connection #{cnxn.vpc_peering_connection_id}"
-              begin
-                MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc_peering_connection(
-                    vpc_peering_connection_id: cnxn.vpc_peering_connection_id
-                ) if !noop
-              rescue Aws::EC2::Errors::InvalidStateTransition
-                MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} not in removable (state #{cnxn.status.code})", MU::WARN
-              rescue Aws::EC2::Errors::OperationNotPermitted => e
-                MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} refuses to delete: #{e.message}", MU::WARN
-              end
-            }
+            purge_peering_connections(noop, vpc.vpc_id, region: region, credentials: credentials)
 
             on_retry = Proc.new {
-              MU::Cloud::AWS::FirewallRule.cleanup(
+              MU::Cloud.resourceClass("AWS", "FirewallRule").cleanup(
                 noop: noop,
                 region: region,
                 credentials: credentials,
@@ -1767,7 +1835,7 @@ module MU
             if !MU::Cloud::AWS.isGovCloud?(region)
               mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu", region: region, credentials: credentials).values.first
               if !mu_zone.nil?
-                MU::Cloud::AWS::DNSZone.toggleVPCAccess(id: mu_zone.id, vpc_id: vpc.vpc_id, remove: true, credentials: credentials)
+                MU::Cloud.resourceClass("AWS", "DNSZone").toggleVPCAccess(id: mu_zone.id, vpc_id: vpc.vpc_id, remove: true, credentials: credentials)
               end
             end
           }

data/modules/mu/{clouds → providers}/azure.rb

@@ -48,7 +48,7 @@ module MU
       end
 
       # List all Azure subscriptions available to our credentials
-      def self.listHabitats(credentials = nil)
+      def self.listHabitats(credentials = nil, use_cache: true)
         []
       end
 
@@ -284,6 +284,9 @@ module MU
           end
           raise e
         end
+        if !sdk_response
+          raise MuError, "Nil response from Azure API attempting list_locations(#{subscription})"
+        end
 
         sdk_response.value.each do | region |
           @@regions.push(region.name)

data/modules/mu/{clouds → providers}/azure/container_cluster.rb

@@ -218,11 +218,7 @@ module MU
               "Azure Kubernetes Service Cluster Admin Role"
             ]
           }
-          cluster['dependencies'] ||= []
-          cluster['dependencies'] << {
-            "type" => "user",
-            "name" => cluster["name"]+"user"
-          }
+          MU::Config.addDependency(cluster, cluster['name']+"user", "user")
 
           ok = false if !configurator.insertKitten(svcacct_desc, "users")
 

data/modules/mu/{clouds → providers}/azure/firewall_rule.rb

@@ -337,7 +337,14 @@ module MU
         # We assume that any values we have in +@config+ are placeholders, and
         # calculate our own accordingly based on what's live in the cloud.
         def toKitten(**args)
-          bok = {}
+
+          bok = {
+            "cloud" => "Azure",
+            "name" => cloud_desc.name,
+            "project" => @config['project'],
+            "credentials" => @config['credentials'],
+            "cloud_id" => @cloud_id.to_s
+          }
 
           bok
         end

data/modules/mu/{clouds → providers}/azure/server.rb

@@ -146,7 +146,7 @@ module MU
           return nil if @config.nil? or @deploy.nil?
 
           nat_ssh_key = nat_ssh_user = nat_ssh_host = nil
-          if !@config["vpc"].nil? and !MU::Cloud::Azure::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
+          if !@config["vpc"].nil? and !MU::Cloud.resourceClass("Azure", "VPC").haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
 
             if !@nat.nil? and @nat.mu_name != @mu_name
               if @nat.cloud_desc.nil?
@@ -189,7 +189,7 @@ module MU
           end
 
           _nat_ssh_key, _nat_ssh_user, nat_ssh_host, _canonical_ip, _ssh_user, _ssh_key_name = getSSHConfig
-          if !nat_ssh_host and !MU::Cloud::Azure::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
+          if !nat_ssh_host and !MU::Cloud.resourceClass("Azure", "VPC").haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
 # XXX check if canonical_ip is in the private ranges
 #            raise MuError, "#{node} has no NAT host configured, and I have no other route to it"
           end
@@ -384,7 +384,7 @@ module MU
           # Our deploydata gets corrupted often with server pools, this will cause us to use the wrong IP to identify a node
           # which will cause us to create certificates, DNS records and other artifacts with incorrect information which will cause our deploy to fail.
           # The cloud_id is always correct so lets use 'cloud_desc' to get the correct IPs
-          if MU::Cloud::Azure::VPC.haveRouteToInstance?(cloud_desc, credentials: @config['credentials']) or public_ips.size == 0
+          if MU::Cloud.resourceClass("Azure", "VPC").haveRouteToInstance?(cloud_desc, credentials: @config['credentials']) or public_ips.size == 0
             @config['canonical_ip'] = private_ips.first
             return private_ips.first
           else
@@ -393,6 +393,28 @@ module MU
           end
         end
 
+        # Return all of the IP addresses, public and private, from all of our
+        # network interfaces.
+        # @return [Array<String>]
+        def listIPs
+          ips = []
+          cloud_desc.network_profile.network_interfaces.each { |iface|
+            iface_id = Id.new(iface.is_a?(Hash) ? iface['id'] : iface.id)
+            iface_desc = MU::Cloud::Azure.network(credentials: @credentials).network_interfaces.get(@resource_group, iface_id.to_s)
+            iface_desc.ip_configurations.each { |ipcfg|
+              ips << ipcfg.private_ipaddress
+              if ipcfg.respond_to?(:public_ipaddress) and ipcfg.public_ipaddress
+                ip_id = Id.new(ipcfg.public_ipaddress.id)
+                ip_desc = MU::Cloud::Azure.network(credentials: @credentials).public_ipaddresses.get(@resource_group, ip_id.to_s)
+                if ip_desc
+                  ips << ip_desc.ip_address
+                end
+              end
+            }
+          }
+          ips
+        end
+
         # return [String]: A password string.
         def getWindowsAdminPassword
         end
@@ -430,7 +452,7 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
         end
 
         # Cloud-specific configuration properties.
@@ -441,7 +463,7 @@ module MU
           hosts_schema = MU::Config::CIDR_PRIMITIVE
           hosts_schema["pattern"] = "^(\\d+\\.\\d+\\.\\d+\\.\\d+\/[0-9]{1,2}|\\*)$"
           schema = {
-            "roles" => MU::Cloud::Azure::User.schema(config)[1]["roles"],
+            "roles" => MU::Cloud.resourceClass("Azure", "User").schema(config)[1]["roles"],
             "ingress_rules" => {
               "items" => {
                 "properties" => {
@@ -497,8 +519,7 @@ module MU
             foundmatch = false
             MU::Cloud.availableClouds.each { |cloud|
               next if cloud == "Azure"
-              cloudbase = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-              foreign_types = (cloudbase.listInstanceTypes).values.first
+              foreign_types = (MU::Cloud.cloudClass(cloud).listInstanceTypes).values.first
               if foreign_types.size == 1
                 foreign_types = foreign_types.values.first
               end
@@ -590,18 +611,8 @@ module MU
             if !configurator.insertKitten(vpc, "vpcs", true)
               ok = false
             end
-            server['dependencies'] ||= []
-
-            server['dependencies'] << {
-              "type" => "vpc",
-              "name" => server['name']+"vpc"
-            }
-# XXX what happens if there's no natstion here?
-            server['dependencies'] << {
-              "type" => "server",
-              "name" => server['name']+"vpc-natstion",
-              "phase" => "groom"
-            }
+            MU::Config.addDependency(server, server['name']+"vpc", "vpc")
+            MU::Config.addDependency(server, server['name']+"vpc-natstion", "server", phase: "groom")
             server['vpc'] = {
               "name" => server['name']+"vpc",
               "subnet_pref" => "private"
@@ -618,17 +629,14 @@ module MU
             "credentials" => server["credentials"],
             "roles" => server["roles"]
           }
-          server['dependencies'] ||= []
-          server['dependencies'] << {
-            "type" => "user",
-            "name" => server["name"]+"user"
-          }
+          MU::Config.addDependency(server, server['name']+"user", "user")
 
           ok = false if !configurator.insertKitten(svcacct_desc, "users")
 
           ok
         end
 
+        # stub
         def self.diskConfig(config, create = true, disk_as_url = true, credentials: nil)
         end