cloud-mu 3.1.3 → 3.1.4

data/modules/mu/clouds/google/group.rb

@@ -139,12 +139,17 @@ module MU
         # Remove all groups associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
-        # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
-          my_domains = MU::Cloud::Google.getDomains(credentials)
+        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
+          MU::Cloud::Google.getDomains(credentials)
           my_org = MU::Cloud::Google.getOrg(credentials)
 
+          filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
+          if !ignoremaster and MU.mu_public_ip
+            filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
+          end
+          MU.log "Placeholder: Google Group artifacts do not support labels, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: filter
+
           if my_org
             groups = MU::Cloud::Google.admin_directory(credentials: credentials).list_groups(customer: MU::Cloud::Google.customerID(credentials)).groups
             if groups
@@ -199,7 +204,7 @@ module MU
         # Reverse-map our cloud description into a runnable config hash.
         # We assume that any values we have in +@config+ are placeholders, and
         # calculate our own accordingly based on what's live in the cloud.
-        def toKitten(rootparent: nil, billing: nil, habitats: nil)
+        def toKitten(**_args)
 
           bok = {
             "cloud" => "Google",
@@ -208,15 +213,15 @@ module MU
 
           bok['name'] = cloud_desc.name
           bok['cloud_id'] = cloud_desc.email
-          bok['members'] = members
-          bok['members'].each { |m|
-            m = MU::Config::Ref.get(
-              id: m,
-              cloud: "Google",
-              credentials: @config['credentials'],
-              type: "users"
-            )
-          }
+          bok['members'] = members.dup
+#          bok['members'] = members.map { |m|
+#            MU::Config::Ref.get(
+#              id: m,
+#              cloud: "Google",
+#              credentials: @config['credentials'],
+#              type: "users"
+#            )
+#          }
           group_roles = MU::Cloud::Google::Role.getAllBindings(@config['credentials'])["by_entity"]
           if group_roles["group"] and group_roles["group"][bok['cloud_id']] and
              group_roles["group"][bok['cloud_id']].size > 0
@@ -227,9 +232,9 @@ module MU
        end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
             "name" => {
@@ -322,7 +327,7 @@ If we are binding (rather than creating) a group and no roles are specified, we
           end
 
 
-          credcfg = MU::Cloud::Google.credConfig(group['credentials'])
+          MU::Cloud::Google.credConfig(group['credentials'])
 
           if group['external'] and group['members']
             MU.log "Cannot manage memberships for external group #{group['name']}", MU::ERR
@@ -360,8 +365,6 @@ If we are binding (rather than creating) a group and no roles are specified, we
           ok
         end
 
-        private
-
       end
     end
   end

data/modules/mu/clouds/google/habitat.rb

@@ -113,7 +113,7 @@ module MU
           @habitat_id = parent_id
           begin
             setProjectBilling
-          rescue Exception => e
+          rescue StandardError => e
             MU.log "Failed to set billing account #{@config['billing_acct']} on project #{@cloud_id}: #{e.message}", MU::ERR
             MU::Cloud::Google.resource_manager(credentials: @config['credentials']).delete_project(@cloud_id)
             raise e
@@ -167,10 +167,12 @@ module MU
           end
         end
 
+        @cached_cloud_desc = nil
         # Return the cloud descriptor for the Habitat
         # @return [Google::Apis::Core::Hashable]
-        def cloud_desc
-          @cached_cloud_desc ||= MU::Cloud::Google::Habitat.find(cloud_id: @cloud_id).values.first
+        def cloud_desc(use_cache: true)
+          return @cached_cloud_desc if @cached_cloud_desc and use_cache
+          @cached_cloud_desc = MU::Cloud::Google::Habitat.find(cloud_id: @cloud_id).values.first
           if @cached_cloud_desc and @cached_cloud_desc.parent
             @habitat_id ||= @cached_cloud_desc.parent.id
           end
@@ -209,7 +211,7 @@ module MU
                billing.billing_account_name.empty?
               return false
             end
-          rescue ::Google::Apis::ClientError => e
+          rescue ::Google::Apis::ClientError
             return false
           end
 
@@ -219,14 +221,15 @@ module MU
         # Remove all Google projects associated with the currently loaded deployment. Try to, anyway.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
-        # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
           resp = MU::Cloud::Google.resource_manager(credentials: credentials).list_projects
+
           if resp and resp.projects
             resp.projects.each { |p|
-              if p.labels and p.labels["mu-id"] == MU.deploy_id.downcase and
-                 p.lifecycle_state == "ACTIVE"
+              labelmatch = (p.labels and (p.labels["mu-id"] == MU.deploy_id.downcase and (ignoremaster or p.labels["mu-master-ip"] == MU.mu_public_ip.gsub(/\./, "_"))))
+              flagmatch = (flags and flags['known'] and flags['known'].include?(p.project_id))
+              if (labelmatch or flagmatch) and p.lifecycle_state == "ACTIVE"
                 MU.log "Deleting project #{p.project_id} (#{p.name})", details: p
                 if !noop
                   begin
@@ -291,7 +294,7 @@ module MU
         # Reverse-map our cloud description into a runnable config hash.
         # We assume that any values we have in +@config+ are placeholders, and
         # calculate our own accordingly based on what's live in the cloud.
-        def toKitten(rootparent: nil, billing: nil, habitats: nil)
+        def toKitten(**args)
           bok = {
             "cloud" => "Google",
             "credentials" => @config['credentials']
@@ -311,17 +314,17 @@ module MU
                 credentials: @config['credentials'],
                 type: "folders"
               )
-            elsif rootparent
+            elsif args[:rootparent]
               bok['parent'] = {
-                'id' => rootparent.is_a?(String) ? rootparent : rootparent.cloud_desc.name
+                'id' => args[:rootparent].is_a?(String) ? args[:rootparent] : args[:rootparent].cloud_desc.name
               }
             else
               # org parent is *probably* safe to infer from credentials
             end
           end
 
-          if billing
-            bok['billing_acct'] = billing
+          if args[:billing]
+            bok['billing_acct'] = args[:billing]
           else
             cur_billing = MU::Cloud::Google.billing(credentials: @config['credentials']).get_project_billing_info("projects/"+@cloud_id)
             if cur_billing and cur_billing.billing_account_name
@@ -334,9 +337,9 @@ module MU
 
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
             "billing_acct" => {

data/modules/mu/clouds/google/loadbalancer.rb

@@ -79,13 +79,13 @@ module MU
             end
             if @config['global']
               MU.log "Creating Global Forwarding Rule #{@mu_name}", MU::NOTICE, details: ruleobj
-              resp = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_global_forwarding_rule(
+              MU::Cloud::Google.compute(credentials: @config['credentials']).insert_global_forwarding_rule(
                 @project_id,
                 ruleobj
               )
             else
               MU.log "Creating regional Forwarding Rule #{@mu_name} in #{@config['region']}", MU::NOTICE, details: ruleobj
-              resp = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_forwarding_rule(
+              MU::Cloud::Google.compute(credentials: @config['credentials']).insert_forwarding_rule(
                 @project_id,
                 @config['region'],
                 ruleobj
@@ -149,6 +149,11 @@ module MU
         def self.cleanup(noop: false, ignoremaster: false, region: nil, credentials: nil, flags: {})
           flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
           return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
+          filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
+          if !ignoremaster and MU.mu_public_ip
+            filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
+          end
+          MU.log "Placeholder: Google LoadBalancer artifacts do not support labels, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: filter
 
           if region
             ["forwarding_rule", "region_backend_service"].each { |type|
@@ -174,9 +179,9 @@ module MU
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
             "named_ports" => {
@@ -202,9 +207,9 @@ module MU
 
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::loadbalancers}, bare and unvalidated.
         # @param lb [Hash]: The resource to process and validate
-        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+        # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
         # @return [Boolean]: True if validation succeeded, False otherwise
-        def self.validateConfig(lb, configurator)
+        def self.validateConfig(lb, _configurator)
           ok = true
           if lb['classic']
             MU.log "LoadBalancer 'classic' flag has no meaning in Google Cloud", MU::WARN
@@ -236,7 +241,6 @@ module MU
           end
 
           lb["listeners"].each { |l|
-            ruleobj = nil
             if lb["private"] and !["TCP", "UDP"].include?(l['lb_protocol'])
               MU.log "Only TCP and UDP listeners are valid for private LoadBalancers in Google Cloud", MU::ERR
               ok = false
@@ -255,7 +259,6 @@ module MU
           lb["targetgroups"].each { |tg|
             if tg["healthcheck"]
               target = tg["healthcheck"]['target'].match(/^([^:]+):(\d+)(.*)/)
-              proto = target[1]
               if tg["proto"] != target[1]
                 MU.log "LoadBalancer #{lb['name']} can't mix and match target group and health check protocols in Google Cloud", MU::ERR, details: tg
                 ok = false
@@ -286,14 +289,9 @@ module MU
         end
 
         # Locate an existing LoadBalancer or LoadBalancers and return an array containing matching Google resource descriptors for those that match.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region
-        # @param tag_key [String]: A tag key to search.
-        # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
-        # @param flags [Hash]: Optional flags
-        # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching LoadBalancers
-        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, flags: {}, credentials: nil)
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching LoadBalancers
+        def self.find(**args)
+          args = MU::Cloud::Google.findLocationArgs(args)
         end
 
         private

data/modules/mu/clouds/google/role.rb

@@ -32,6 +32,7 @@ module MU
           # If we're being reverse-engineered from a cloud descriptor, use that
           # to determine what sort of account we are.
           if args[:from_cloud_desc]
+            require 'google/apis/admin_directory_v1'
             @cloud_desc_cache = args[:from_cloud_desc]
             if args[:from_cloud_desc].class == ::Google::Apis::AdminDirectoryV1::Role
               @config['role_source'] = "directory"
@@ -87,7 +88,7 @@ module MU
             resp = if @config['role_source'] == "org"
               my_org = MU::Cloud::Google.getOrg(@config['credentials'])
               MU.log "Creating IAM organization role #{@mu_name} in #{my_org.display_name}", details: create_role_obj
-              resp = MU::Cloud::Google.iam(credentials: @credentials).create_organization_role(my_org.name, create_role_obj)
+              MU::Cloud::Google.iam(credentials: @credentials).create_organization_role(my_org.name, create_role_obj)
             elsif @config['role_source'] == "project"
               if !@project_id
                 raise MuError, "Role #{@mu_name} is supposed to be in project #{@config['project']}, but no such project was found"
@@ -133,12 +134,13 @@ module MU
           }
         end
 
+        @cloud_desc_cache = nil
         # Return the cloud descriptor for the Role
         # @return [Google::Apis::Core::Hashable]
-        def cloud_desc
-          return @cloud_desc_cache if @cloud_desc_cache
+        def cloud_desc(use_cache: true)
+          return @cloud_desc_cache if @cloud_desc_cache and use_cache
 
-          my_org = MU::Cloud::Google.getOrg(@config['credentials'])
+          MU::Cloud::Google.getOrg(@config['credentials'])
 
           @cloud_desc_cache = if @config['role_source'] == "directory"
             MU::Cloud::Google.admin_directory(credentials: @config['credentials']).get_role(@customer, @cloud_id)
@@ -238,7 +240,7 @@ module MU
             req_obj = MU::Cloud::Google.resource_manager(:SetIamPolicyRequest).new(
               policy: policy
             )
-            policy = if scope_type == "organizations"
+            if scope_type == "organizations"
               MU::Cloud::Google.resource_manager(credentials: credentials).set_organization_iam_policy(
                 scope_id,
                 req_obj
@@ -310,7 +312,7 @@ module MU
                   policy: policy
                 )
 
-                policy = if scope_type == "organizations"
+                if scope_type == "organizations"
                   MU::Cloud::Google.resource_manager(credentials: credentials).set_organization_iam_policy(
                     scope_id,
                     req_obj
@@ -340,8 +342,6 @@ module MU
         def self.bindFromConfig(entity_type, entity_id, cfg, credentials: nil, deploy: nil, debug: false)
           loglevel = debug ? MU::NOTICE : MU::DEBUG
 
-          bindings = []
-
           return if !cfg
           MU.log "Google::Role::bindFromConfig binding called for #{entity_type} #{entity_id}", loglevel, details: cfg
 
@@ -464,12 +464,17 @@ module MU
         # Remove all roles associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
-        # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
           customer = MU::Cloud::Google.customerID(credentials)
           my_org = MU::Cloud::Google.getOrg(credentials)
 
+          filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
+          if !ignoremaster and MU.mu_public_ip
+            filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
+          end
+          MU.log "Placeholder: Google Role artifacts do not support labels, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: filter
+
           if flags['known']
             flags['known'].each { |id|
               next if id.nil?
@@ -576,7 +581,7 @@ module MU
               }
             end
             if args[:cloud_id]
-              found.reject! { |k, v| k != role.name }
+              found.reject! { |k, _v| k != role.name }
             end
 
             # Now go get everything that's bound here
@@ -626,15 +631,17 @@ module MU
               }
             end
 
-            resp = begin
-              MU::Cloud::Google.iam(credentials: args[:credentials]).list_organization_roles(my_org.name)
-            rescue ::Google::Apis::ClientError => e
-              raise e if !e.message.match(/forbidden: /)
-            end
-            if resp and resp.roles
-              resp.roles.each { |role|
-                found[role.name] = role
-              }
+            if my_org
+              resp = begin
+                MU::Cloud::Google.iam(credentials: args[:credentials]).list_organization_roles(my_org.name)
+              rescue ::Google::Apis::ClientError => e
+                raise e if !e.message.match(/forbidden: /)
+              end
+              if resp and resp.roles
+                resp.roles.each { |role|
+                  found[role.name] = role
+                }
+              end
             end
           end
 
@@ -644,7 +651,8 @@ module MU
         # Reverse-map our cloud description into a runnable config hash.
         # We assume that any values we have in +@config+ are placeholders, and
         # calculate our own accordingly based on what's live in the cloud.
-        def toKitten(rootparent: nil, billing: nil, habitats: nil)
+        def toKitten(**args)
+
           bok = {
             "cloud" => "Google",
             "credentials" => @config['credentials'],
@@ -677,7 +685,7 @@ module MU
             end
             if !cloud_desc.role_privileges.nil? and !cloud_desc.role_privileges.empty?
               bok['import'] = []
-              ids, names, privs = MU::Cloud::Google::Role.privilege_service_to_name(@config['credentials'])
+              ids, _names, _privs = MU::Cloud::Google::Role.privilege_service_to_name(@config['credentials'])
               cloud_desc.role_privileges.each { |priv|
                 if !ids[priv.service_id]
                   MU.log "Role privilege defined for a service id with no name I can find, writing with raw id", MU::WARN, details: priv
@@ -696,7 +704,7 @@ module MU
               bok['name'] = name.gsub(/[^a-z0-9]/i, '-')
               bok['role_source'] = "canned"
             elsif cloud_desc.name.match(/^([^\/]+?)\/([^\/]+?)\/roles\/(.*)/)
-              junk, type, parent, name = Regexp.last_match.to_a
+              _junk, type, parent, name = Regexp.last_match.to_a
               bok['name'] = name.gsub(/[^a-z0-9]/i, '-')
               bok['role_source'] = type == "organizations" ? "org" : "project"
               if bok['role_source'] == "project"
@@ -723,8 +731,9 @@ module MU
                 bindings[scopetype].each_pair { |scope_id, entity_types|
                   # If we've been given a habitat filter, skip over bindings
                   # that don't match it.
-                  if scopetype == "projects" and habitats and
-                     !habitats.empty? and !habitats.include?(scope_id)
+                  if scopetype == "projects" and args[:habitats] and
+                     !args[:habitats].empty? and
+                     !args[:habitats].include?(scope_id)
                     next
                   end
 
@@ -983,9 +992,9 @@ module MU
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
             "name" => {
@@ -1050,7 +1059,7 @@ If this value is not specified, and the role name matches the name of an existin
         def self.validateConfig(role, configurator)
           ok = true
 
-          credcfg = MU::Cloud::Google.credConfig(role['credentials'])
+          MU::Cloud::Google.credConfig(role['credentials'])
 
           my_org = MU::Cloud::Google.getOrg(role['credentials'])
           if !role['role_source']
@@ -1059,7 +1068,7 @@ If this value is not specified, and the role name matches the name of an existin
               if !lookup_name.match(/^roles\//)
                 lookup_name = "roles/"+lookup_name
               end
-              canned = MU::Cloud::Google.iam(credentials: role['credentials']).get_role(lookup_name)
+              MU::Cloud::Google.iam(credentials: role['credentials']).get_role(lookup_name)
               MU.log "Role #{role['name']} appears to be a referenced to canned role #{role.name} (#{role.title})", MU::NOTICE
               role['role_source'] = "canned"
             rescue ::Google::Apis::ClientError
@@ -1121,13 +1130,16 @@ If this value is not specified, and the role name matches the name of an existin
           ok
         end
 
-        private
-
         @@service_id_to_name = {}
         @@service_id_to_privs = {}
         @@service_name_to_id = {}
         @@service_name_map_semaphore = Mutex.new
 
+        # Generate lookup tables mapping between hex service role identifiers,
+        # human-readable names of services, and the privileges associated with
+        # those roles.
+        # @param credentials [String]
+        # @return [Array<Hash,Hash,Hash>]
         def self.privilege_service_to_name(credentials = nil)
 
           customer = MU::Cloud::Google.customerID(credentials)
@@ -1176,6 +1188,11 @@ If this value is not specified, and the role name matches the name of an existin
           }
         end
 
+        # Convert a list of shorthand GSuite/Cloud Identity privileges into
+        # +RolePrivilege+ objects for consumption by other API calls
+        # @param roles [Array<String>]:
+        # @param credentials [String]:
+        # @return [Array<Google::Apis::AdminDirectoryV1::DirectoryService::Role::RolePrivilege>]
         def self.map_directory_privileges(roles, credentials: nil)
           rolepriv_objs = []
           notfound = []