cloud-mu 3.1.2 → 3.2.0

data/modules/mu/{clouds → providers}/google/server_pool.rb

@@ -64,10 +64,9 @@ module MU
             size = @config['basis']['launch_config']['size']
             @config['image_id'] = @config['basis']['launch_config']['image_id']
           end
-          az = @config['availability_zone']
-          if az.nil?
-            az = MU::Cloud::Google.listAZs(@config['region']).sample
-          end
+# XXX this should create a non-regional instance group
+#          az = @config['availability_zone']
+#          az ||= MU::Cloud::Google.listAZs(@config['region']).sample
 
           metadata = { # :items?
             "startup-script" => @userdata
@@ -90,8 +89,8 @@ module MU
             machine_type: size,
             service_accounts: [@service_acct],
             labels: labels,
-            disks: MU::Cloud::Google::Server.diskConfig(@config, false, false, credentials: @config['credentials']),
-            network_interfaces: MU::Cloud::Google::Server.interfaceConfig(@config, @vpc),
+            disks: MU::Cloud.resourceClass("Google", "Server").diskConfig(@config, false, false, credentials: @config['credentials']),
+            network_interfaces: MU::Cloud.resourceClass("Google", "Server").interfaceConfig(@config, @vpc),
             metadata: metadata,
             tags: MU::Cloud::Google.compute(:Tags).new(items: [MU::Cloud::Google.nameStr(@mu_name)])
           )
@@ -170,8 +169,7 @@ module MU
         # Locate an existing ServerPool or ServerPools and return an array containing matching Google resource descriptors for those that match.
         # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching ServerPools
         def self.find(**args)
-          args[:project] ||= args[:habitat]
-          args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
+          args = MU::Cloud::Google.findLocationArgs(args)
 
           regions = if args[:region]
             [args[:region]]
@@ -213,7 +211,7 @@ module MU
         # Reverse-map our cloud description into a runnable config hash.
         # We assume that any values we have in +@config+ are placeholders, and
         # calculate our own accordingly based on what's live in the cloud.
-        def toKitten(rootparent: nil, billing: nil, habitats: nil)
+        def toKitten(**_args)
           bok = {
             "cloud" => "Google",
             "credentials" => @credentials,
@@ -326,11 +324,11 @@ end
         def self.schema(config)
           toplevel_required = []
           schema = {
-            "ssh_user" => MU::Cloud::Google::Server.schema(config)[1]["ssh_user"],
-            "metadata" => MU::Cloud::Google::Server.schema(config)[1]["metadata"],
-            "service_account" => MU::Cloud::Google::Server.schema(config)[1]["service_account"],
-            "scopes" => MU::Cloud::Google::Server.schema(config)[1]["scopes"],
-            "network_tags" => MU::Cloud::Google::Server.schema(config)[1]["network_tags"],
+            "ssh_user" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["ssh_user"],
+            "metadata" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["metadata"],
+            "service_account" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["service_account"],
+            "scopes" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["scopes"],
+            "network_tags" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["network_tags"],
             "availability_zone" => {
               "type" => "string",
               "description" => "Target a specific availability zone for this pool, which will create zonal instance managers and scalers instead of regional ones."
@@ -362,7 +360,7 @@ end
         # @return [Boolean]: True if validation succeeded, False otherwise
         def self.validateConfig(pool, configurator)
           ok = true
-start = Time.now
+#start = Time.now
           pool['project'] ||= MU::Cloud::Google.defaultProject(pool['credentials'])
           if pool['service_account']
             pool['service_account']['cloud'] = "Google"
@@ -373,29 +371,7 @@ start = Time.now
               ok = false
             end
           else
-            user = {
-              "name" => pool['name'],
-              "cloud" => "Google",
-              "project" => pool["project"],
-              "credentials" => pool["credentials"],
-              "type" => "service"
-            }
-            if user["name"].length < 6
-              user["name"] += Password.pronounceable(6)
-            end
-            configurator.insertKitten(user, "users", true)
-            pool['dependencies'] ||= []
-            pool['service_account'] = MU::Config::Ref.get(
-              type: "users",
-              cloud: "Google",
-              name: pool["name"],
-              project: pool["project"],
-              credentials: pool["credentials"]
-            )
-            pool['dependencies'] << {
-              "type" => "user",
-              "name" => pool["name"]
-            }
+            pool = MU::Cloud::Google::User.genericServiceAccount(pool, configurator)
           end
 
           pool['named_ports'] ||= []
@@ -406,7 +382,7 @@ start = Time.now
           if pool['basis']['launch_config']
             launch = pool["basis"]["launch_config"]
 
-            launch['size'] = MU::Cloud::Google::Server.validateInstanceType(launch["size"], pool["region"])
+            launch['size'] = MU::Cloud.resourceClass("Google", "Server").validateInstanceType(launch["size"], pool["region"])
             ok = false if launch['size'].nil?
 
             if launch['image_id'].nil?
@@ -421,7 +397,7 @@ start = Time.now
 
             real_image = nil
             begin
-              real_image = MU::Cloud::Google::Server.fetchImage(launch['image_id'].to_s, credentials: pool['credentials'])
+              real_image = MU::Cloud.resourceClass("Google", "Server").fetchImage(launch['image_id'].to_s, credentials: pool['credentials'])
             rescue ::Google::Apis::ClientError => e
               MU.log e.inspect, MU::WARN
             end
@@ -456,14 +432,19 @@ start = Time.now
         # @param region [String]: The cloud provider region
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
+          flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
+          return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
+          filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
+          if !ignoremaster and MU.mu_public_ip
+            filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
+          end
+          MU.log "Placeholder: Google ServerPool artifacts do not support labels, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: filter
 
           if !flags["global"]
             ["region_autoscaler", "region_instance_group_manager"].each { |type|
               MU::Cloud::Google.compute(credentials: credentials).delete(
                 type,
-                flags["project"],
+                flags["habitat"],
                 region,
                 noop
               )
@@ -471,7 +452,7 @@ start = Time.now
           else
             MU::Cloud::Google.compute(credentials: credentials).delete(
               "instance_template",
-              flags["project"],
+              flags["habitat"],
               noop
             )
           end

data/modules/mu/{clouds → providers}/google/user.rb

@@ -26,10 +26,12 @@ module MU
           # If we're being reverse-engineered from a cloud descriptor, use that
           # to determine what sort of account we are.
           if args[:from_cloud_desc]
+            @cloud_desc_cache = args[:from_cloud_desc]
             MU::Cloud::Google.admin_directory
             MU::Cloud::Google.iam
             if args[:from_cloud_desc].class == ::Google::Apis::AdminDirectoryV1::User
               @config['type'] = "interactive"
+              @cloud_id = args[:from_cloud_desc].primary_email
             elsif args[:from_cloud_desc].class == ::Google::Apis::IamV1::ServiceAccount
               @config['type'] = "service"
               @config['name'] = args[:from_cloud_desc].display_name
@@ -48,6 +50,10 @@ module MU
             @config['name']
           end
 
+          if @config['type'] == "interactive" and @config['email']
+            @cloud_id ||= @config['email']
+          end
+
         end
 
         # Called automatically by {MU::Deploy#createResources}
@@ -58,7 +64,7 @@ module MU
               account_id: acct_id,
               service_account: MU::Cloud::Google.iam(:ServiceAccount).new(
                 display_name: @mu_name,
-                description: @config['scrub_mu_isms'] ? nil : @deploy.deploy_id
+                description: @config['scrub_mu_isms'] ? @config['description'] : @deploy.deploy_id
               )
             )
             if @config['use_if_exists']
@@ -90,7 +96,7 @@ module MU
             end
           elsif @config['external']
             @cloud_id = @config['email']
-            MU::Cloud::Google::Role.bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
+            MU::Cloud.resourceClass("Google", "Role").bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
           else
             if !@config['email']
               domains = MU::Cloud::Google.admin_directory(credentials: @credentials).list_domains(@customer)
@@ -122,10 +128,10 @@ module MU
         # Called automatically by {MU::Deploy#createResources}
         def groom
           if @config['external']
-            MU::Cloud::Google::Role.bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
+            MU::Cloud.resourceClass("Google", "Role").bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
           elsif @config['type'] == "interactive"
             need_update = false
-            MU::Cloud::Google::Role.bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
+            MU::Cloud.resourceClass("Google", "Role").bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
 
             if @config['force_password_change'] and !cloud_desc.change_password_at_next_login
               MU.log "Forcing #{@mu_name} to change their password at next login", MU::NOTICE
@@ -170,7 +176,7 @@ module MU
             end
 
           else
-            MU::Cloud::Google::Role.bindFromConfig("serviceAccount", @cloud_id.gsub(/.*?\/([^\/]+)$/, '\1'), @config['roles'], credentials: @config['credentials'])
+            MU::Cloud.resourceClass("Google", "Role").bindFromConfig("serviceAccount", @cloud_id.gsub(/.*?\/([^\/]+)$/, '\1'), @config['roles'], credentials: @config['credentials'])
             if @config['create_api_key']
               resp = MU::Cloud::Google.iam(credentials: @config['credentials']).list_project_service_account_keys(
                 cloud_desc.name
@@ -187,21 +193,36 @@ module MU
           end
         end
 
+        @cloud_desc_cache = nil
         # Retrieve the cloud descriptor for this resource.
         # @return [Google::Apis::Core::Hashable]
-        def cloud_desc
+        def cloud_desc(use_cache: true)
+          return @cloud_desc_cache if @cloud_desc_cache and use_cache
           if @config['type'] == "interactive" or !@config['type']
              @config['type'] ||= "interactive"
             if !@config['external']
-              return MU::Cloud::Google.admin_directory(credentials: @config['credentials']).get_user(@cloud_id)
+              @cloud_id ||= @config['email']
+              @cloud_desc_cache = MU::Cloud::Google.admin_directory(credentials: @config['credentials']).get_user(@cloud_id)
             else
               return nil
             end
           else
             @config['type'] ||= "service"
-            MU::Cloud::Google.iam(credentials: @config['credentials']).get_project_service_account(@cloud_id)
+            # this often fails even when it succeeded earlier, so try to be
+            # resilient on GCP's behalf
+            retries = 0
+            begin
+              @cloud_desc_cache = MU::Cloud::Google.iam(credentials: @config['credentials']).get_project_service_account(@cloud_id)
+            rescue ::Google::Apis::ClientError => e
+              if e.message.match(/notFound:/) and retries < 10
+                sleep 3
+                retries += 1
+                retry
+              end
+            end
           end
 
+          @cloud_desc_cache
         end
 
         # Return the metadata for this user configuration
@@ -212,7 +233,7 @@ module MU
           else
             {}
           end
-          description.delete(:etag)
+          description.delete(:etag) if description
           description
         end
 
@@ -232,12 +253,17 @@ module MU
         # Remove all users associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
-        # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
-          my_domains = MU::Cloud::Google.getDomains(credentials)
+        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
+          MU::Cloud::Google.getDomains(credentials)
           my_org = MU::Cloud::Google.getOrg(credentials)
 
+          filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
+          if !ignoremaster and MU.mu_public_ip
+            filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
+          end
+          MU.log "Placeholder: Google User artifacts do not support labels, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: filter
+
           # We don't have a good way of tagging directory users, so we rely
           # on the known parameter, which is pulled from deployment metadata
           if flags['known'] and my_org
@@ -256,15 +282,15 @@ module MU
                 next if user_email.nil?
                 next if !user_email.match(/^[^\/]+@[^\/]+$/)
 
-                MU::Cloud::Google::Role.removeBindings("user", user_email, credentials: credentials, noop: noop)
+                MU::Cloud.resourceClass("Google", "Role").removeBindings("user", user_email, credentials: credentials, noop: noop)
               }
 
             end
           end
 
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
+          flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
           resp = MU::Cloud::Google.iam(credentials: credentials).list_project_service_accounts(
-            "projects/"+flags["project"]
+            "projects/"+flags["habitat"]
           )
 
           if resp and resp.accounts and MU.deploy_id
@@ -319,7 +345,7 @@ module MU
               MU::Cloud::Google.iam(credentials: args[:credentials]).list_project_service_accounts(
                 "projects/"+args[:project]
               )
-            rescue ::Google::Apis::ClientError => e
+            rescue ::Google::Apis::ClientError
               MU.log "Do not have permissions to retrieve service accounts for project #{args[:project]}", MU::WARN
             end
 
@@ -337,8 +363,10 @@ module MU
           else
             if cred_cfg['masquerade_as']
               resp = MU::Cloud::Google.admin_directory(credentials: args[:credentials]).list_users(customer: MU::Cloud::Google.customerID(args[:credentials]), show_deleted: false)
+# XXX this ain't exactly performant, do some caching or something
               if resp and resp.users
                 resp.users.each { |u|
+                  next if args[:cloud_id] and !args[:cloud_id] != u.primary_email
                   found[u.primary_email] = u
                 }
               end
@@ -355,6 +383,7 @@ module MU
           name.match(/\b\d+\-compute@developer\.gserviceaccount\.com$/) or
           name.match(/\bproject-\d+@storage-transfer-service\.iam\.gserviceaccount\.com$/) or
           name.match(/\b\d+@cloudbuild\.gserviceaccount\.com$/) or
+          name.match(/\b\d+@cloudservices\.gserviceaccount\.com$/) or
           name.match(/\bservice-\d+@containerregistry\.iam\.gserviceaccount\.com$/) or
           name.match(/\bservice-\d+@container-analysis\.iam\.gserviceaccount\.com$/) or
           name.match(/\bservice-\d+@compute-system\.iam\.gserviceaccount\.com$/) or
@@ -382,7 +411,7 @@ module MU
         # Reverse-map our cloud description into a runnable config hash.
         # We assume that any values we have in +@config+ are placeholders, and
         # calculate our own accordingly based on what's live in the cloud.
-        def toKitten(rootparent: nil, billing: nil, habitats: nil)
+        def toKitten(**_args)
           if MU::Cloud::Google::User.cannedServiceAcctName?(@cloud_id)
             return nil
           end
@@ -397,7 +426,7 @@ module MU
             return nil
           end
 
-          user_roles = MU::Cloud::Google::Role.getAllBindings(@config['credentials'])["by_entity"]
+          user_roles = MU::Cloud.resourceClass("Google", "Role").getAllBindings(@config['credentials'])["by_entity"]
 
           if cloud_desc.nil?
             MU.log "FAILED TO FIND CLOUD DESCRIPTOR FOR #{self}", MU::ERR, details: @config
@@ -410,6 +439,10 @@ module MU
 
           if bok['type'] == "service"
             bok['name'].gsub!(/@.*/, '')
+            if cloud_desc.description and !cloud_desc.description.empty? and
+               !cloud_desc.description.match(/^[A-Z0-9_-]+-[A-Z0-9_-]+-\d{10}-[A-Z]{2}$/)
+              bok['description'] = cloud_desc.description
+            end
             bok['project'] = @project_id
             keys = MU::Cloud::Google.iam(credentials: @config['credentials']).list_project_service_account_keys(@cloud_id)
 
@@ -420,13 +453,13 @@ module MU
             if user_roles["serviceAccount"] and
                user_roles["serviceAccount"][bok['cloud_id']] and
                user_roles["serviceAccount"][bok['cloud_id']].size > 0
-              bok['roles'] = MU::Cloud::Google::Role.entityBindingsToSchema(user_roles["serviceAccount"][bok['cloud_id']])
+              bok['roles'] = MU::Cloud.resourceClass("Google", "Role").entityBindingsToSchema(user_roles["serviceAccount"][bok['cloud_id']])
             end
           else
             if user_roles["user"] and
                user_roles["user"][bok['cloud_id']] and
                user_roles["user"][bok['cloud_id']].size > 0
-              bok['roles'] = MU::Cloud::Google::Role.entityBindingsToSchema(user_roles["user"][bok['cloud_id']], credentials: @config['credentials'])
+              bok['roles'] = MU::Cloud.resourceClass("Google", "Role").entityBindingsToSchema(user_roles["user"][bok['cloud_id']], credentials: @config['credentials'])
             end
             bok['given_name'] = cloud_desc.name.given_name if cloud_desc.name.given_name and !cloud_desc.name.given_name.empty?
             bok['family_name'] = cloud_desc.name.family_name if cloud_desc.name.family_name and !cloud_desc.name.family_name.empty?
@@ -441,9 +474,9 @@ module MU
        end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
             "name" => {
@@ -482,6 +515,10 @@ If we are binding (rather than creating) a user and no roles are specified, we w
               "type" => "string",
               "description" => "Alias for +family_name+"
             },
+            "description" => {
+              "type" => "string",
+              "description" => "Comment field for service accounts, which we normally use to store the originating deploy's deploy id, since GCP service accounts do not have labels. This field is only honored if +scrub_mu_isms+ is set."
+            },
             "email" => {
               "type" => "string",
               "description" => "Canonical email address for a +directory+ user. If not specified, will be set to +name@domain+."
@@ -509,7 +546,7 @@ If we are binding (rather than creating) a user and no roles are specified, we w
             "roles" => {
               "type" => "array",
               "description" => "One or more Google IAM roles to associate with this user.",
-              "items" => MU::Cloud::Google::Role.ref_schema
+              "items" => MU::Cloud.resourceClass("Google", "Role").ref_schema
             }
           }
           [toplevel_required, schema]
@@ -517,9 +554,9 @@ If we are binding (rather than creating) a user and no roles are specified, we w
 
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::users}, bare and unvalidated.
         # @param user [Hash]: The resource to process and validate
-        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+        # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
         # @return [Boolean]: True if validation succeeded, False otherwise
-        def self.validateConfig(user, configurator)
+        def self.validateConfig(user, _configurator)
           ok = true
 
           my_domains = MU::Cloud::Google.getDomains(user['credentials'])
@@ -595,15 +632,11 @@ If we are binding (rather than creating) a user and no roles are specified, we w
             ok = false
           end
 
-          user['dependencies'] ||= []
           if user['roles']
             user['roles'].each { |r|
               if r['role'] and r['role']['name'] and
                  (!r['role']['deploy_id'] and !r['role']['id'])
-                user['dependencies'] << {
-                  "type" => "role",
-                  "name" => r['role']['name']
-                }
+                MU::Config.addDependency(user, r['role']['name'], "role")
               end
 
               if !r["projects"] and !r["organizations"] and !r["folders"]
@@ -621,7 +654,38 @@ If we are binding (rather than creating) a user and no roles are specified, we w
           ok
         end
 
-        private
+        # Create and inject a service account on behalf of the parent resource.
+        # Return the modified parent configuration hash with references to the
+        # new addition.
+        # @param parent [Hash]
+        # @param configurator [MU::Config]
+        # @return [Hash]
+        def self.genericServiceAccount(parent, configurator)
+          user = {
+            "name" => parent['name'],
+            "cloud" => "Google",
+            "project" => parent["project"],
+            "credentials" => parent["credentials"],
+            "type" => "service"
+          }
+          if user["name"].length < 6
+            user["name"] += Password.pronounceable(6)
+          end
+          if parent['roles']
+            user['roles'] = parent['roles'].dup
+          end
+          configurator.insertKitten(user, "users", true)
+          parent['service_account'] = MU::Config::Ref.get(
+            type: "users",
+            cloud: "Google",
+            name: user["name"],
+            project: user["project"],
+            credentials: user["credentials"]
+          )
+          MU::Config.addDependency(parent, user['name'], "user")
+
+          parent
+        end
 
       end
     end