cloud-mu 3.1.6 → 3.2.0

data/modules/mu/{clouds → providers}/aws/storage_pool.rb

@@ -67,7 +67,7 @@ module MU
                 if target['vpc']["subnet_name"]
                   subnet_obj = vpc.getSubnet(name: target['vpc']["subnet_name"])
                   if subnet_obj.nil?
-                    raise MuError, "Failed to locate subnet from #{subnet} in StoragePool #{@config['name']}:#{target['name']}"
+                    raise MuError, "Failed to locate subnet from #{target['vpc']["subnet_name"]} in StoragePool #{@config['name']}:#{target['name']}"
                   end
                   target['vpc']['subnet_id'] = subnet_obj.cloud_id
                 end
@@ -261,49 +261,29 @@ module MU
           targets = {}
 
           if @config['mount_points'] && !@config['mount_points'].empty?
+            mount_targets = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_mount_targets(
+              file_system_id: storage_pool.file_system_id
+            ).mount_targets
+
             @config['mount_points'].each { |mp|
               subnet = nil
               dependencies
-              mp_vpc = if mp['vpc'] and mp['vpc']['vpc_name']
-                @deploy.findLitterMate(type: "vpc", name: mp['vpc']['vpc_name'], credentials: @config['credentials'])
-              elsif mp['vpc']
-                MU::MommaCat.findStray(
-                  @config['cloud'],
-                  "vpcs",
-                  deploy_id: mp['vpc']["deploy_id"],
-                  credentials: @config['credentials'],
-                  mu_name: mp['vpc']["mu_name"],
-                  cloud_id: mp['vpc']['vpc_id'],
-                  region: @config['region'],
-                  dummy_ok: false
-                ).first
-# XXX non-sibling, findStray version
-              end
+              mp_vpc = MU::Config::Ref.get(mp['vpc']).kitten
 
-              mount_targets = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_mount_targets(
-                file_system_id: storage_pool.file_system_id
-              ).mount_targets
 
-#              subnet_obj = mp_vpc.subnets.select { |s|
-#                s.name == mp["vpc"]["subnet_name"] or s.cloud_id == mp["vpc"]["subnet_id"]
-#              }.first
+              subnet_obj = mp_vpc.subnets.select { |s|
+                s.name == mp["vpc"]["subnet_name"] or s.cloud_id == mp["vpc"]["subnet_id"]
+              }.first
               mount_target = nil
-              mp_vpc.subnets.each { |subnet_obj|
-                mount_targets.map { |t|
-                  subnet_cidr_obj = NetAddr::IPv4Net.parse(subnet_obj.ip_block)
-                  if subnet_cidr_obj.contains(NetAddr::IPv4.parse(t.ip_address))
-                    mount_target = t
-                    subnet = subnet_obj.cloud_desc
-                  end
-                }
-                break if mount_target
+              mount_targets.each { |t|
+                subnet_cidr_obj = NetAddr::IPv4Net.parse(subnet_obj.ip_block)
+                if subnet_cidr_obj.contains(NetAddr::IPv4.parse(t.ip_address))
+                  mount_target = t
+                  subnet = subnet_obj.cloud_desc
+                  break
+                end
               }
 
-              # mount_target = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_mount_targets(
-                # mount_target_id: mp["cloud_id"]
-              # ).mount_targets.first
-
-
               targets[mp["name"]] = {
                 "owner_id" => mount_target.owner_id,
                 "cloud_id" => mount_target.mount_target_id,
@@ -493,6 +473,9 @@ module MU
 
           if pool['mount_points'] && !pool['mount_points'].empty?
             pool['mount_points'].each{ |mp|
+              if mp['vpc'] and mp['vpc']['name']
+                MU::Config.addDependency(pool, mp['vpc']['name'], "vpc")
+              end
               if mp['ingress_rules']
                 fwname = "storage-#{mp['name']}"
                 acl = {

data/modules/mu/{clouds → providers}/aws/user.rb

@@ -109,7 +109,7 @@ module MU
           # Create these if necessary, then append them to the list of
           # attachable_policies
           if @config['raw_policies']
-            pol_arns = MU::Cloud::AWS::Role.manageRawPolicies(
+            pol_arns = MU::Cloud.resourceClass("AWS", "Role").manageRawPolicies(
               @config['raw_policies'],
               basename: @deploy.getResourceName(@config['name']),
               credentials: @credentials
@@ -135,7 +135,7 @@ module MU
             attached_policies.each { |a|
               if !configured_policies.include?(a.policy_arn)
                 MU.log "Removing IAM policy #{a.policy_arn} from user #{@mu_name}", MU::NOTICE
-                MU::Cloud::AWS::Role.purgePolicy(a.policy_arn, @credentials)
+                MU::Cloud.resourceClass("AWS", "Role").purgePolicy(a.policy_arn, @credentials)
               else
                 configured_policies.delete(a.policy_arn)
               end
@@ -151,7 +151,7 @@ module MU
           end
 
           if @config['inline_policies']
-            docs = MU::Cloud::AWS::Role.genPolicyDocument(@config['inline_policies'], deploy_obj: @deploy)
+            docs = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['inline_policies'], deploy_obj: @deploy)
             docs.each { |doc|
               MU.log "Putting user policy #{doc.keys.first} to user #{@cloud_id} "
               MU::Cloud::AWS.iam(credentials: @credentials).put_user_policy(
@@ -431,7 +431,7 @@ MU.log e.inspect, MU::ERR, details: policy
             resp.policy_names.each { |pol_name|
               pol = MU::Cloud::AWS.iam(credentials: @credentials).get_user_policy(user_name: @cloud_id, policy_name: pol_name)
               doc = JSON.parse(URI.decode(pol.policy_document))
-              bok["inline_policies"] = MU::Cloud::AWS::Role.doc2MuPolicies(pol.policy_name, doc, bok["inline_policies"])
+              bok["inline_policies"] = MU::Cloud.resourceClass("AWS", "Role").doc2MuPolicies(pol.policy_name, doc, bok["inline_policies"])
             }
           end
 
@@ -465,7 +465,7 @@ MU.log e.inspect, MU::ERR, details: policy
         def self.schema(_config)
           toplevel_required = []
           polschema = MU::Config::Role.schema["properties"]["policies"]
-          polschema.deep_merge!(MU::Cloud::AWS::Role.condition_schema)
+          polschema.deep_merge!(MU::Cloud.resourceClass("AWS", "Role").condition_schema)
 
           schema = {
             "inline_policies" => polschema,
@@ -517,7 +517,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-USER-FOO+"
           # If we're attaching some managed policies, make sure all of the ones
           # that should already exist do indeed exist
           if user['attachable_policies']
-            ok = false if !MU::Cloud::AWS::Role.validateAttachablePolicies(
+            ok = false if !MU::Cloud.resourceClass("AWS", "Role").validateAttachablePolicies(
               user['attachable_policies'],
               credentials: user['credentials'],
               region: user['region']
@@ -530,7 +530,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-USER-FOO+"
               if configurator.haveLitterMate?(group, "groups")
                 need_dependency = true
               else
-                found = MU::Cloud::AWS::Group.find(cloud_id: group)
+                found = MU::Cloud.resourceClass("AWS", "Group").find(cloud_id: group)
                 if found.nil? or found.empty? or (configurator.updating and
                    found.values.first.group.path == "/"+configurator.updating+"/")
                   groupdesc = {
@@ -542,11 +542,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-USER-FOO+"
               end
 
               if need_dependency
-                user["dependencies"] ||= []
-                user["dependencies"] << {
-                  "type" => "group",
-                  "name" => group
-                }
+                MU::Config.addDependency(user, group, "group")
               end
             }
           end

data/modules/mu/{clouds → providers}/aws/vpc.rb

@@ -18,7 +18,7 @@ module MU
 
       # Creation of Virtual Private Clouds and associated artifacts (routes, subnets, etc).
       class VPC < MU::Cloud::VPC
-        require 'mu/clouds/aws/vpc_subnet'
+        require 'mu/providers/aws/vpc_subnet'
 
         # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
         # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
@@ -209,7 +209,7 @@ module MU
           if !MU::Cloud::AWS.isGovCloud?(@config['region'])
             mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu", credentials: @config['credentials']).values.first
             if !mu_zone.nil?
-              MU::Cloud::AWS::DNSZone.toggleVPCAccess(id: mu_zone.id, vpc_id: @cloud_id, region: @config['region'], credentials: @config['credentials'])
+              MU::Cloud.resourceClass("AWS", "DNSZone").toggleVPCAccess(id: mu_zone.id, vpc_id: @cloud_id, region: @config['region'], credentials: @config['credentials'])
             end
           end
 					loadSubnets
@@ -838,9 +838,23 @@ module MU
             vpcs = resp if !resp.empty?
           }
 
+#          resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
+#            filters: [
+#              {
+#                name: "requester-vpc-info.vpc-id",
+#                values: [@cloud_id]
+#              },
+#              {
+#                name: "accepter-vpc-info.vpc-id",
+#                values: [peer_id.to_s]
+#              }
+#            ]
+#          )
+
           if !vpcs.empty?
             gwthreads = []
             vpcs.each { |vpc|
+              purge_peering_connections(noop, vpc.vpc_id, region: region, credentials: credentials)
               # NAT gateways don't have any tags, and we can't assign them a name. Lets find them based on a VPC ID
               gwthreads << Thread.new {
                 purge_nat_gateways(noop, vpc_id: vpc.vpc_id, region: region, credentials: credentials)
@@ -916,11 +930,7 @@ module MU
             logdesc["tags"] = vpc["tags"] if !vpc["tags"].nil?
 #            logdesc["optional_tags"] = vpc["optional_tags"] if !vpc["optional_tags"].nil?
             configurator.insertKitten(logdesc, "logs")
-            vpc['dependencies'] ||= []
-            vpc['dependencies'] << {
-              "type" => "log",
-              "name" => vpc['name']+"loggroup"
-            }
+            MU::Config.addDependency(vpc, vpc['name']+"loggroup", "log")
 
             roledesc = {
               "name" => vpc['name']+"logrole",
@@ -958,11 +968,7 @@ module MU
             roledesc["tags"] = vpc["tags"] if !vpc["tags"].nil?
             roledesc["optional_tags"] = vpc["optional_tags"] if !vpc["optional_tags"].nil?
             configurator.insertKitten(roledesc, "roles")
-            vpc['dependencies'] ||= []
-            vpc['dependencies'] << {
-              "type" => "role",
-              "name" => vpc['name']+"logrole"
-            }
+            MU::Config.addDependency(vpc, vpc['name']+"logrole", "role")
           end
 
           subnet_routes = Hash.new
@@ -1013,10 +1019,7 @@ module MU
                 subnet_routes[table['name']].each { |subnet|
                   nat_routes[subnet] = route['nat_host_name']
                 }
-                vpc['dependencies'] << {
-                  "type" => "server",
-                  "name" => route['nat_host_name']
-                }
+                MU::Config.addDependency(vpc, route['nat_host_name'], "server", no_create_wait: true)
               elsif route['gateway'] == '#NAT'
                 vpc['create_nat_gateway'] = true
                 private_rtbs << table['name']
@@ -1225,7 +1228,7 @@ module MU
                   # suits me just fine
                 rescue Aws::EC2::Errors::AuthFailure => e
                   if !tried_lbs and iface.attachment.instance_owner_id == "amazon-elb"
-                    MU::Cloud::AWS::LoadBalancer.cleanup(
+                    MU::Cloud.resourceClass("AWS", "LoadBalancer").cleanup(
                       noop: noop,
                       region: region,
                       credentials: credentials,
@@ -1265,6 +1268,63 @@ module MU
           nil
         end
 
+        # Try to locate the default VPC for a region, and return a BoK-style
+        # config fragment for something that might want to live in it.
+        def self.defaultVpc(region, credentials)
+          cfg_fragment = nil
+          MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_vpcs.vpcs.each { |vpc|
+            if vpc.is_default
+              cfg_fragment = {
+                "id" => vpc.vpc_id,
+                "cloud" => "AWS",
+                "region" => region,
+                "credentials" => credentials
+              }
+              cfg_fragment['subnets'] = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_subnets(
+                filters: [
+                  {
+                    name: "vpc-id",
+                    values: [vpc.vpc_id]
+                  }
+                ]
+              ).subnets.map { |s| { "subnet_id" => s.subnet_id } }
+              break
+            end
+          }
+
+          cfg_fragment
+        end
+
+        # Return a {MU::Config::Ref} that indicates this VPC.
+        # @param subnet_ids [Array<String>]: Optional list of subnet ids with which to infer a +subnet_pref+ parameter.
+        # @return [MU::Config::Ref]
+        def getReference(subnet_ids = [])
+          have_private = have_public = false
+          subnets.each { |s|
+            next if subnet_ids and !subnet_ids.empty? and !subnet_ids.include?(s.cloud_id)
+            if s.private?
+              have_private = true
+            else
+              have_public = true
+            end
+          }
+          subnet_pref = if have_private == have_public
+            "any"
+          elsif have_private
+            "all_private"
+          elsif have_public
+            "all_public"
+          end
+          MU::Config::Ref.get(
+            id: @cloud_id,
+            cloud: "AWS",
+            credentials: @credentials,
+            region: @config['region'],
+            type: "vpcs",
+            subnet_pref: subnet_pref
+          )
+        end
+
         private
 
         def peerWith(peer)
@@ -1683,6 +1743,61 @@ module MU
         end
         private_class_method :purge_dhcpopts
 
+        def self.purge_peering_connections(noop, vpc_id, region: MU.curRegion, credentials: nil)
+          my_peer_conns = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_peering_connections(
+            filters: [
+              {
+                name: "requester-vpc-info.vpc-id",
+                values: [vpc_id]
+              }
+            ]
+          ).vpc_peering_connections
+          my_peer_conns.concat(MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_peering_connections(
+            filters: [
+              {
+                name: "accepter-vpc-info.vpc-id",
+                values: [vpc_id]
+              }
+            ]
+          ).vpc_peering_connections)
+
+          my_peer_conns.each { |cnxn|
+            [cnxn.accepter_vpc_info.vpc_id, cnxn.requester_vpc_info.vpc_id].each { |peer_vpc|
+              MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_vpc, region: region, credentials: credentials).each { |rtb_id|
+                begin
+                  resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_route_tables(
+                    route_table_ids: [rtb_id]
+                  )
+                rescue Aws::EC2::Errors::InvalidRouteTableIDNotFound
+                  next
+                end
+                resp.route_tables.each { |rtb|
+                  rtb.routes.each { |route|
+                    if route.vpc_peering_connection_id == cnxn.vpc_peering_connection_id
+                      MU.log "Removing route #{route.destination_cidr_block} from route table #{rtb_id} in VPC #{peer_vpc}"
+                      MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_route(
+                          route_table_id: rtb_id,
+                          destination_cidr_block: route.destination_cidr_block
+                      ) if !noop
+                    end
+                  }
+                }
+              }
+            }
+            MU.log "Deleting VPC peering connection #{cnxn.vpc_peering_connection_id}"
+            begin
+              MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc_peering_connection(
+                vpc_peering_connection_id: cnxn.vpc_peering_connection_id
+              ) if !noop
+            rescue Aws::EC2::Errors::InvalidStateTransition
+              MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} not in removable (state #{cnxn.status.code})", MU::WARN
+            rescue Aws::EC2::Errors::OperationNotPermitted => e
+              MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} refuses to delete: #{e.message}", MU::WARN
+            end
+          }
+        end
+        private_class_method :purge_peering_connections
+
         # Remove all VPCs associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
@@ -1697,60 +1812,10 @@ module MU
           return if vpcs.nil? or vpcs.size == 0
 
           vpcs.each { |vpc|
-            my_peer_conns = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_peering_connections(
-              filters: [
-                {
-                  name: "requester-vpc-info.vpc-id",
-                  values: [vpc.vpc_id]
-                }
-              ]
-            ).vpc_peering_connections
-            my_peer_conns.concat(MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_peering_connections(
-              filters: [
-                {
-                  name: "accepter-vpc-info.vpc-id",
-                  values: [vpc.vpc_id]
-                }
-              ]
-            ).vpc_peering_connections)
-            my_peer_conns.each { |cnxn|
-
-              [cnxn.accepter_vpc_info.vpc_id, cnxn.requester_vpc_info.vpc_id].each { |peer_vpc|
-                MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_vpc, region: region, credentials: credentials).each { |rtb_id|
-                  begin
-                    resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_route_tables(
-                      route_table_ids: [rtb_id]
-                    )
-                  rescue Aws::EC2::Errors::InvalidRouteTableIDNotFound
-                    next
-                  end
-                  resp.route_tables.each { |rtb|
-                    rtb.routes.each { |route|
-                      if route.vpc_peering_connection_id == cnxn.vpc_peering_connection_id
-                        MU.log "Removing route #{route.destination_cidr_block} from route table #{rtb_id} in VPC #{peer_vpc}"
-                        MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_route(
-                            route_table_id: rtb_id,
-                            destination_cidr_block: route.destination_cidr_block
-                        ) if !noop
-                      end
-                    }
-                  }
-                }
-              }
-              MU.log "Deleting VPC peering connection #{cnxn.vpc_peering_connection_id}"
-              begin
-                MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc_peering_connection(
-                    vpc_peering_connection_id: cnxn.vpc_peering_connection_id
-                ) if !noop
-              rescue Aws::EC2::Errors::InvalidStateTransition
-                MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} not in removable (state #{cnxn.status.code})", MU::WARN
-              rescue Aws::EC2::Errors::OperationNotPermitted => e
-                MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} refuses to delete: #{e.message}", MU::WARN
-              end
-            }
+            purge_peering_connections(noop, vpc.vpc_id, region: region, credentials: credentials)
 
             on_retry = Proc.new {
-              MU::Cloud::AWS::FirewallRule.cleanup(
+              MU::Cloud.resourceClass("AWS", "FirewallRule").cleanup(
                 noop: noop,
                 region: region,
                 credentials: credentials,
@@ -1767,7 +1832,7 @@ module MU
             if !MU::Cloud::AWS.isGovCloud?(region)
               mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu", region: region, credentials: credentials).values.first
               if !mu_zone.nil?
-                MU::Cloud::AWS::DNSZone.toggleVPCAccess(id: mu_zone.id, vpc_id: vpc.vpc_id, remove: true, credentials: credentials)
+                MU::Cloud.resourceClass("AWS", "DNSZone").toggleVPCAccess(id: mu_zone.id, vpc_id: vpc.vpc_id, remove: true, credentials: credentials)
               end
             end
           }

data/modules/mu/{clouds → providers}/azure.rb

@@ -48,7 +48,7 @@ module MU
       end
 
       # List all Azure subscriptions available to our credentials
-      def self.listHabitats(credentials = nil)
+      def self.listHabitats(credentials = nil, use_cache: true)
         []
       end
 
@@ -284,6 +284,9 @@ module MU
           end
           raise e
         end
+        if !sdk_response
+          raise MuError, "Nil response from Azure API attempting list_locations(#{subscription})"
+        end
 
         sdk_response.value.each do | region |
           @@regions.push(region.name)