cloud-mu 3.1.2 → 3.2.0

data/modules/mu/{clouds → providers}/google/vpc.rb

@@ -36,12 +36,10 @@ module MU
 
         # Called automatically by {MU::Deploy#createResources}
         def create
-
           networkobj = MU::Cloud::Google.compute(:Network).new(
             name: MU::Cloud::Google.nameStr(@mu_name),
             description: @deploy.deploy_id,
             auto_create_subnetworks: false
-#            i_pv4_range: @config['ip_block']
           )
           MU.log "Creating network #{@mu_name} (#{@config['ip_block']}) in project #{@project_id}", details: networkobj
 
@@ -58,7 +56,7 @@ module MU
                 subnet_name = @config['name']+subnet['name']
 
                 subnet_mu_name = @config['scrub_mu_isms'] ? @cloud_id+subnet_name.downcase : MU::Cloud::Google.nameStr(@deploy.getResourceName(subnet_name, max_length: 61))
-                MU.log "Creating subnetwork #{subnet_mu_name} (#{subnet['ip_block']}) in project #{@project_id}", details: subnet
+                MU.log "Creating subnetwork #{subnet_mu_name} (#{subnet['ip_block']}) in project #{@project_id} region #{subnet['availability_zone']}", details: subnet
                 subnetobj = MU::Cloud::Google.compute(:Subnetwork).new(
                   name: subnet_mu_name,
                   description: @deploy.deploy_id,
@@ -72,9 +70,17 @@ module MU
                 subnetdesc = nil
                 begin 
                   subnetdesc = MU::Cloud::Google.compute(credentials: @config['credentials']).get_subnetwork(@project_id, subnet['availability_zone'], subnet_mu_name)
+                  if !subnetdesc.nil?
+                    subnet_cfg = {}
+                    subnet_cfg["ip_block"] = subnet['ip_block']
+                    subnet_cfg["name"] = subnet_name
+                    subnet_cfg['mu_name'] = subnet_mu_name
+                    subnet_cfg["cloud_id"] = subnetdesc.self_link.gsub(/.*?\/([^\/]+)$/, '\1')
+                    subnet_cfg['az'] = subnet['availability_zone']
+                    @subnets << MU::Cloud::Google::VPC::Subnet.new(self, subnet_cfg, precache_description: false)
+                  end
                   sleep 1
                 end while subnetdesc.nil?
-  
               }
             }
             subnetthreads.each do |t|
@@ -82,7 +88,6 @@ module MU
             end
           end
 
-          route_table_ids = []
           if !@config['route_tables'].nil?
             @config['route_tables'].each { |rtb|
               rtb['routes'].each { |route|
@@ -108,7 +113,7 @@ module MU
         # Describe this VPC
         # @return [Hash]
         def notify
-          base = MU.structToHash(cloud_desc)
+          base = MU.structToHash(cloud_desc, stringify_keys: true)
           base["cloud_id"] = @cloud_id
           base["project_id"] = habitat_id
           base.merge!(@config.to_h)
@@ -120,8 +125,8 @@ module MU
 
         # Describe this VPC from the cloud platform's perspective
         # @return [Google::Apis::Core::Hashable]
-        def cloud_desc
-          if @cloud_desc_cache
+        def cloud_desc(use_cache: true)
+          if @cloud_desc_cache and use_cache
             return @cloud_desc_cache
           end
 
@@ -230,8 +235,8 @@ end
         # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
         # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching resources
         def self.find(**args)
-          args[:project] ||= args[:habitat]
-          args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
+          args = MU::Cloud::Google.findLocationArgs(args)
+
           resp = {}
           if args[:cloud_id] and args[:project]
             begin
@@ -240,7 +245,7 @@ end
               args[:cloud_id].to_s.sub(/^.*?\/([^\/]+)$/, '\1')
             )
             resp[args[:cloud_id]] = vpc if !vpc.nil?
-            rescue ::Google::Apis::ClientError => e
+            rescue ::Google::Apis::ClientError
               MU.log "VPC #{args[:cloud_id]} in project #{args[:project]} does not exist, or I do not have permission to view it", MU::WARN
             end
           else # XXX other criteria
@@ -296,14 +301,10 @@ end
              @deploy.deployment["vpcs"][@config['name']]["subnets"] and
              @deploy.deployment["vpcs"][@config['name']]["subnets"].size > 0
             @deploy.deployment["vpcs"][@config['name']]["subnets"].each { |desc|
-              subnet = {}
-              subnet["ip_block"] = desc['ip_block']
-              subnet["name"] = desc["name"]
+              subnet = desc.clone
               subnet['mu_name'] = @config['scrub_mu_isms'] ? @cloud_id+subnet['name'].downcase : MU::Cloud::Google.nameStr(@deploy.getResourceName(subnet['name'], max_length: 61))
-              subnet["cloud_id"] = desc['cloud_id']
               subnet["cloud_id"] ||= desc['self_link'].gsub(/.*?\/([^\/]+)$/, '\1')
               subnet["cloud_id"] ||= subnet['mu_name']
-              subnet['az'] = desc["az"]
               subnet['az'] ||= desc["region"].gsub(/.*?\/([^\/]+)$/, '\1')
               @subnets << MU::Cloud::Google::VPC::Subnet.new(self, subnet, precache_description: false)
             }
@@ -477,9 +478,8 @@ end
         # directly at child nodes in peered VPCs, the public internet, and the
         # like.
         # @param target_instance [OpenStruct]: The cloud descriptor of the instance to check.
-        # @param region [String]: The cloud provider region of the target subnet.
         # @return [Boolean]
-        def self.haveRouteToInstance?(target_instance, region: MU.curRegion, credentials: nil)
+        def self.haveRouteToInstance?(target_instance, credentials: nil)
           project ||= MU::Cloud::Google.defaultProject(credentials)
           return false if MU.myCloud != "Google"
 # XXX see if we reside in the same Network and overlap subnets
@@ -509,7 +509,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
         # @param target_subnets_key [String]: The subnet/subnets on the other side of the peered VPC.
         # @param instance_id [String]: The instance ID in the target subnet/subnets.
         # @return [Boolean]
-        def self.have_route_peered_vpc?(source_subnets_key, target_subnets_key, instance_id)
+        def self.can_route_to_master_peer?(source_subnets_key, target_subnets_key, instance_id)
         end
 
         # Retrieves the route tables of used by subnets
@@ -536,13 +536,17 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
         # Remove all VPC resources associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
-        # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
+        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
+          flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
+          return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
+          filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
+          if !ignoremaster and MU.mu_public_ip
+            filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
+          end
+          MU.log "Placeholder: Google VPC artifacts do not support labels, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: filter
 
-          purge_subnets(noop, project: flags['project'], credentials: credentials)
+          purge_subnets(noop, project: flags['habitat'], credentials: credentials)
           ["route", "network"].each { |type|
 # XXX tagged routes aren't showing up in list, and the networks that own them
 # fail to delete silently
@@ -551,7 +555,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
             begin
               MU::Cloud::Google.compute(credentials: credentials).delete(
                 type,
-                flags["project"],
+                flags["habitat"],
                 nil,
                 noop
               )
@@ -561,13 +565,13 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
                   MU.log e.message, MU::WARN
                   if e.message.match(/Failed to delete network (.+)/)
                     network_name = Regexp.last_match[1]
-                    fwrules = MU::Cloud::Google::FirewallRule.find(project: flags['project'], credentials: credentials)
-                    fwrules.reject! { |name, desc|
+                    fwrules = MU::Cloud.resourceClass("Google", "FirewallRule").find(project: flags['habitat'], credentials: credentials)
+                    fwrules.reject! { |_name, desc|
                       !desc.network.match(/.*?\/#{Regexp.quote(network_name)}$/)
                     }
                     fwrules.keys.each { |name|
                       MU.log "Attempting to delete firewall rule #{name} so that VPC #{network_name} can be removed", MU::NOTICE
-                      MU::Cloud::Google.compute(credentials: credentials).delete_firewall(flags['project'], name)
+                      MU::Cloud::Google.compute(credentials: credentials).delete_firewall(flags['habitat'], name)
                     }
                   end
                 end
@@ -586,7 +590,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
         # We assume that any values we have in +@config+ are placeholders, and
         # calculate our own accordingly based on what's live in the cloud.
         # XXX add flag to return the diff between @config and live cloud
-        def toKitten(rootparent: nil, billing: nil, habitats: nil)
+        def toKitten(**_args)
           return nil if cloud_desc.name == "default" # parent project builds these
           bok = {
             "cloud" => "Google",
@@ -595,8 +599,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
           }
           MU::Cloud::Google.listRegions.size
 
-          diff = {}
-          schema, valid = MU::Config.loadResourceSchema("VPC", cloud: "Google")
+          _schema, valid = MU::Config.loadResourceSchema("VPC", cloud: "Google")
           return [nil, nil] if !valid
 #          pp schema
 #          MU.log "++++++++++++++++++++++++++++++++"
@@ -609,6 +612,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
             bok['subnets'] = []
             regions_seen = []
             names_seen = []
+            @subnets.reject! { |x| x.cloud_desc.nil? }
             @subnets.map { |x| x.cloud_desc }.each { |s|
               subnet_name = s.name.dup
               names_seen << s.name.dup
@@ -630,7 +634,6 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
             end
           end
 
-          peer_names = []
           if cloud_desc.peerings and cloud_desc.peerings.size > 0
             bok['peers'] = []
             cloud_desc.peerings.each { |peer|
@@ -688,9 +691,9 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config = nil)
+        def self.schema(_config = nil)
           toplevel_required = []
           schema = {
             "regions" => {
@@ -736,7 +739,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
 
           # see if one of this thing's siblings declared a subnet_pref we can
           # use to guess which one we should marry ourselves to
-          configurator.kittens.each_pair { |type, siblings|
+          configurator.kittens.values.each { |siblings|
             siblings.each { |sibling|
               next if !sibling['dependencies']
               sibling['dependencies'].each { |dep|
@@ -900,7 +903,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
                 "destination_network"=>"0.0.0.0/0"
               }
             end
-            nat_count = 0
+
             # You know what, let's just guarantee that we'll have a route from
             # this master, always
             # XXX this confuses machines that don't have public IPs
@@ -943,9 +946,44 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
           createRoute(route, network: @url, tags: [MU::Cloud::Google.nameStr(server.mu_name)])
         end
 
+        # Looks at existing subnets, and attempts to find the next available
+        # IP block that's roughly similar to the ones we already have. This
+        # checks against secondary IP ranges, as well as each subnet's primary
+        # CIDR block.
+        # @param exclude [Array<String>]: One or more CIDRs to treat as unavailable, in addition to those allocated to existing subnets
+        # @return [String]
+        def getUnusedAddressBlock(exclude: [], max_bits: 28)
+          used_ranges = exclude.map { |cidr| NetAddr::IPv4Net.parse(cidr) }
+          subnets.each { |s|
+            used_ranges << NetAddr::IPv4Net.parse(s.cloud_desc.ip_cidr_range)
+            if s.cloud_desc.secondary_ip_ranges
+              used_ranges.concat(s.cloud_desc.secondary_ip_ranges.map { |r| NetAddr::IPv4Net.parse(r.ip_cidr_range) })
+            end
+          }
+# XXX sort used_ranges
+          candidate = used_ranges.first.next_sib
+
+          begin
+            if candidate.netmask.prefix_len > max_bits
+              candidate = candidate.resize(max_bits)
+            end
+            try_again = false
+            used_ranges.each { |cidr|
+              if !cidr.rel(candidate).nil?
+                candidate = candidate.next_sib
+                try_again = true
+                break
+              end
+            }
+            try_again = false if candidate.nil?
+          end while try_again
+
+          candidate.to_s
+        end
+
         private
 
-        def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project, publicroute = true, credentials: nil)
+        def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project, _publicroute = true, credentials: nil)
           private_acl = {
             "name" => vpc_name+"-rt",
             "cloud" => "Google",
@@ -973,6 +1011,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
 #          end
           configurator.insertKitten(private_acl, "firewall_rules", true)
         end
+        private_class_method :genStandardSubnetACLs
 
         # Helper method for manufacturing routes. Expect to be called from
         # {MU::Cloud::Google::VPC#create} or {MU::Cloud::Google::VPC#groom}.
@@ -1039,7 +1078,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
             rescue ::Google::Apis::ClientError, MU::MuError => e
               if e.message.match(/notFound/)
                 MU.log "Creating route #{routename} in project #{@project_id}", details: routeobj
-                resp = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_route(@project_id, routeobj)
+                MU::Cloud::Google.compute(credentials: @config['credentials']).insert_route(@project_id, routeobj)
               else
                 # TODO can't update GCP routes, would have to delete and re-create
               end
@@ -1047,44 +1086,12 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
           end
         end
 
-
-        # Remove all network gateways associated with the currently loaded deployment.
-        # @param noop [Boolean]: If true, will only print what would be done
-        # @param region [String]: The cloud provider region
-        # @return [void]
-        def self.purge_gateways(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion)
-        end
-
-        # Remove all NAT gateways associated with the VPC of the currently loaded deployment.
-        # @param noop [Boolean]: If true, will only print what would be done
-        # @param vpc_id [String]: The cloud provider's unique VPC identifier
-        # @param region [String]: The cloud provider region
-        # @return [void]
-        def self.purge_nat_gateways(noop = false, vpc_id: nil, region: MU.curRegion)
-        end
-
-        # Remove all VPC endpoints associated with the VPC of the currently loaded deployment.
-        # @param noop [Boolean]: If true, will only print what would be done
-        # @param vpc_id [String]: The cloud provider's unique VPC identifier
-        # @param region [String]: The cloud provider region
-        # @return [void]
-        def self.purge_endpoints(noop = false, vpc_id: nil, region: MU.curRegion)
-        end
-
-        # Remove all network interfaces associated with the currently loaded deployment.
-        # @param noop [Boolean]: If true, will only print what would be done
-        # @param tagfilters [Array<Hash>]: Labels to filter against when search for resources to purge
-        # @param region [String]: The cloud provider region
-        # @return [void]
-        def self.purge_interfaces(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion)
-        end
-
         # Remove all subnets associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
-        # @param tagfilters [Array<Hash>]: Labels to filter against when search for resources to purge
+        # @param _tagfilters [Array<Hash>]: Labels to filter against when search for resources to purge
         # @param regions [Array<String>]: The cloud provider regions to check
         # @return [void]
-        def self.purge_subnets(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], regions: MU::Cloud::Google.listRegions, project: nil, credentials: nil)
+        def self.purge_subnets(noop = false, _tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], regions: MU::Cloud::Google.listRegions, project: nil, credentials: nil)
           project ||= MU::Cloud::Google.defaultProject(credentials)
           parent_thread_id = Thread.current.object_id
           regionthreads = []
@@ -1098,7 +1105,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
                   r,
                   noop
                 )
-              rescue MU::Cloud::MuDefunctHabitat => e
+              rescue MU::Cloud::MuDefunctHabitat
                 Thread.exit
               end
             }
@@ -1107,8 +1114,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
             t.join
           end
         end
-
-        protected
+        private_class_method :purge_subnets
 
         # Subnets are almost a first-class resource. So let's kinda sorta treat
         # them like one. This should only be invoked on objects that already
@@ -1116,7 +1122,6 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
         class Subnet < MU::Cloud::Google::VPC
 
           attr_reader :cloud_id
-          attr_reader :url
           attr_reader :ip_block
           attr_reader :mu_name
           attr_reader :name
@@ -1146,13 +1151,32 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
           # Describe this VPC Subnet
           # @return [Hash]
           def notify
-            MU.structToHash(cloud_desc)
+            MU.structToHash(cloud_desc, stringify_keys: true)
+          end
+
+          # Return the +self_link+ to this subnet
+          def url
+            cloud_desc if !@url
+            @url
           end
 
+          @cloud_desc_cache = nil
           # Describe this VPC Subnet from the cloud platform's perspective
           # @return [Google::Apis::Core::Hashable]
-          def cloud_desc
-            @cloud_desc_cache ||= MU::Cloud::Google.compute(credentials: @parent.config['credentials']).get_subnetwork(@parent.habitat_id, @config['az'], @config['cloud_id'])
+          def cloud_desc(use_cache: true)
+            return @cloud_desc_cache if @cloud_desc_cache and use_cache
+
+            begin
+              @cloud_desc_cache = MU::Cloud::Google.compute(credentials: @parent.config['credentials']).get_subnetwork(@parent.habitat_id, @az, @cloud_id)
+            rescue ::Google::Apis::ClientError => e
+              if e.message.match(/notFound: /)
+                MU.log "Failed to fetch cloud description for Google subnet #{@cloud_id}", MU::WARN, details: { "project" => @parent.habitat_id, "region" => @az, "name" => @cloud_id }
+                return nil
+              else
+                raise e
+              end
+            end
+            @url ||= @cloud_desc_cache.self_link
             @cloud_desc_cache
           end
 

data/modules/tests/bucket.yml

@@ -6,7 +6,11 @@ buckets:
   policies:
   - name: testpermissions
     grant_to:
+<% if cloud == "Google" %>
     - identifier: egt.gcp.sandbox@gmail.com
+<% elsif cloud == "AWS" %>
+    - identifier: "arn:aws:iam::<%= MU::Cloud::AWS.account_number %>:root"
+<% end %>
     targets: # XXX this is redundant except for path:
     - type: bucket
       identifier: bucket

data/modules/tests/centos6.yaml

@@ -0,0 +1,11 @@
+# groomers: Chef
+---
+appname: smoketest
+servers: 
+- name: centos6
+  platform: centos6
+  size: m3.medium
+  run_list:
+  - recipe[mu-tools::apply_security]
+  - recipe[mu-tools::updates]
+  - recipe[mu-tools::split_var_partitions]

data/modules/tests/centos7.yaml

@@ -0,0 +1,11 @@
+# groomers: Chef
+---
+appname: smoketest
+servers: 
+- name: centos7
+  platform: centos7
+  size: m3.medium
+  run_list:
+  - recipe[mu-tools::apply_security]
+  - recipe[mu-tools::updates]
+  - recipe[mu-tools::split_var_partitions]

data/modules/tests/centos8.yaml

@@ -0,0 +1,12 @@
+# groomers: Chef
+# clouds: Azure, Google
+---
+appname: smoketest
+servers: 
+- name: centos8
+  platform: centos8
+  size: m3.medium
+  run_list:
+  - recipe[mu-tools::apply_security]
+  - recipe[mu-tools::updates]
+  - recipe[mu-tools::split_var_partitions]