cloud-mu 2.1.0beta → 3.0.0beta

data/modules/mu/clouds/google/server_pool.rb

@@ -18,38 +18,30 @@ module MU
       # A server pool as configured in {MU::Config::BasketofKittens::server_pools}
       class ServerPool < MU::Cloud::ServerPool
 
-        @deploy = nil
-        @project_id = nil
-        @config = nil
-        attr_reader :mu_name
-        attr_reader :cloud_id
-        attr_reader :config
-
-        # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
-        # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::server_pools}
-        def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
-          @deploy = mommacat
-          @config = MU::Config.manxify(kitten_cfg)
-          @cloud_id ||= cloud_id
-          if !mu_name.nil?
-            @mu_name = mu_name
-            @config['project'] ||= MU::Cloud::Google.defaultProject(@config['credentials'])
-            if !@project_id
-              project = MU::Cloud::Google.projectLookup(@config['project'], @deploy, sibling_only: true, raise_on_fail: false)
-              @project_id = project.nil? ? @config['project'] : project.cloudobj.cloud_id
-            end
-          elsif @config['scrub_mu_isms']
-            @mu_name = @config['name']
-          else
-            @mu_name = @deploy.getResourceName(@config['name'])
-          end
+        # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like <tt>@vpc</tt>, for us.
+        # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
+        def initialize(**args)
+          super
+          @mu_name ||= @deploy.getResourceName(@config['name'])
         end
 
         # Called automatically by {MU::Deploy#createResources}
         def create
-          @project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloudobj.cloud_id
           port_objs = []
 
+          sa = MU::Config::Ref.get(@config['service_account'])
+          if !sa or !sa.kitten or !sa.kitten.cloud_desc
+            raise MuError, "Failed to get service account cloud id from #{@config['service_account'].to_s}"
+          end
+          @service_acct = MU::Cloud::Google.compute(:ServiceAccount).new(
+            email: sa.kitten.cloud_desc.email,
+            scopes: @config['scopes']
+          )
+          if !@config['scrub_mu_isms']
+            MU::Cloud::Google.grantDeploySecretAccess(@service_acct.email, credentials: @config['credentials'])
+          end
+
+
           @config['named_ports'].each { |port_cfg|
             port_objs << MU::Cloud::Google.compute(:NamedPort).new(
               name: port_cfg['name'],
@@ -77,20 +69,30 @@ module MU
             az = MU::Cloud::Google.listAZs(@config['region']).sample
           end
 
+          metadata = { # :items?
+            "startup-script" => @userdata
+          }
+          if @config['metadata']
+            desc[:metadata] = Hash[@config['metadata'].map { |m|
+              [m["key"], m["value"]]
+            }]
+          end
+          deploykey = @config['ssh_user']+":"+@deploy.ssh_public_key
+          if desc[:metadata]["ssh-keys"]
+            desc[:metadata]["ssh-keys"] += "\n"+deploykey
+          else
+            desc[:metadata]["ssh-keys"] = deploykey
+          end
+
           instance_props = MU::Cloud::Google.compute(:InstanceProperties).new(
             can_ip_forward: !@config['src_dst_check'],
             description: @deploy.deploy_id,
-#            machine_type: "zones/"+az+"/machineTypes/"+size,
             machine_type: size,
+            service_accounts: [@service_acct],
             labels: labels,
             disks: MU::Cloud::Google::Server.diskConfig(@config, false, false, credentials: @config['credentials']),
             network_interfaces: MU::Cloud::Google::Server.interfaceConfig(@config, @vpc),
-            metadata: {
-              :items => [
-                :key => "ssh-keys",
-                :value => @config['ssh_user']+":"+@deploy.ssh_public_key
-              ]
-            },
+            metadata: metadata,
             tags: MU::Cloud::Google.compute(:Tags).new(items: [MU::Cloud::Google.nameStr(@mu_name)])
           )
 
@@ -132,9 +134,9 @@ module MU
 # TODO this thing supports based on CPU usage, LB usage, or an arbitrary Cloud
 # Monitoring metric. The default is "sustained 60%+ CPU usage". We should
 # support all that.
-# http://www.rubydoc.info/github/google/google-api-ruby-client/Google/Apis/ComputeBeta/AutoscalingPolicyCpuUtilization
-# http://www.rubydoc.info/github/google/google-api-ruby-client/Google/Apis/ComputeBeta/AutoscalingPolicyLoadBalancingUtilization
-# http://www.rubydoc.info/github/google/google-api-ruby-client/Google/Apis/ComputeBeta/AutoscalingPolicyCustomMetricUtilization
+# http://www.rubydoc.info/github/google/google-api-ruby-client/Google/Apis/ComputeV1/AutoscalingPolicyCpuUtilization
+# http://www.rubydoc.info/github/google/google-api-ruby-client/Google/Apis/ComputeV1/AutoscalingPolicyLoadBalancingUtilization
+# http://www.rubydoc.info/github/google/google-api-ruby-client/Google/Apis/ComputeV1/AutoscalingPolicyCustomMetricUtilization
           policy_obj = MU::Cloud::Google.compute(:AutoscalingPolicy).new(
             cooldown_period_sec: @config['default_cooldown'],
             max_num_replicas: @config['max_size'],
@@ -166,16 +168,156 @@ module MU
         end
 
         # Locate an existing ServerPool or ServerPools and return an array containing matching Google resource descriptors for those that match.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region
-        # @param tag_key [String]: A tag key to search.
-        # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
-        # @param flags [Hash]: Optional flags
-        # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching ServerPools
-        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, flags: {}, credentials: nil)
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
-          MU.log "XXX ServerPool.find not yet implemented", MU::WARN
-          return {}
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching ServerPools
+        def self.find(**args)
+          args[:project] ||= args[:habitat]
+          args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
+
+          regions = if args[:region]
+            [args[:region]]
+          else
+            MU::Cloud::Google.listRegions
+          end
+          found = {}
+
+          regions.each { |r|
+            begin
+              resp = MU::Cloud::Google.compute(credentials: args[:credentials]).list_region_instance_group_managers(args[:project], args[:region])
+              if resp and resp.items
+                resp.items.each { |igm|
+                  found[igm.name] = igm
+                }
+              end
+            rescue ::Google::Apis::ClientError => e
+              raise e if !e.message.match(/forbidden: /)
+            end
+
+            begin
+# XXX can these guys have name collisions? test this
+              MU::Cloud::Google.listAZs(r).each { |az|
+                resp = MU::Cloud::Google.compute(credentials: args[:credentials]).list_instance_group_managers(args[:project], az)
+                if resp and resp.items
+                  resp.items.each { |igm|
+                    found[igm.name] = igm
+                  }
+                end
+              }
+            rescue ::Google::Apis::ClientError => e
+              raise e if !e.message.match(/forbidden: /)
+            end
+          }
+
+          return found
+        end
+
+        # Reverse-map our cloud description into a runnable config hash.
+        # We assume that any values we have in +@config+ are placeholders, and
+        # calculate our own accordingly based on what's live in the cloud.
+        def toKitten(rootparent: nil, billing: nil, habitats: nil)
+          bok = {
+            "cloud" => "Google",
+            "credentials" => @credentials,
+            "cloud_id" => @cloud_id,
+            "region" => @config['region'],
+            "project" => @project_id,
+          }
+          bok['name'] = cloud_desc.name
+
+          scalers = if cloud_desc.zone and cloud_desc.zone.match(/-[a-z]$/)
+            bok['availability_zone'] = cloud_desc.zone.sub(/.*?\/([^\/]+)$/, '\1')
+            MU::Cloud::Google.compute(credentials: @credentials).list_autoscalers(@project_id, bok['availability_zone'])
+          else
+            MU::Cloud::Google.compute(credentials: @credentials).list_region_autoscalers(@project_id, @config['region'], filter: "target eq #{cloud_desc.self_link}")
+          end
+
+          if scalers and scalers.items and scalers.items.size > 0
+            scaler = scalers.items.first
+MU.log bok['name'], MU::WARN, details: scaler.autoscaling_policy
+# scaler.cpu_utilization.utilization_target
+# scaler.cool_down_period_sec
+            bok['min_size'] = scaler.autoscaling_policy.min_num_replicas
+            bok['max_size'] = scaler.autoscaling_policy.max_num_replicas
+          else
+            bok['min_size'] = bok['max_size'] = cloud_desc.target_size
+          end
+if cloud_desc.auto_healing_policies and cloud_desc.auto_healing_policies.size > 0
+MU.log bok['name'], MU::WARN, details: cloud_desc.auto_healing_policies
+end
+
+          template = MU::Cloud::Google.compute(credentials: @credentials).get_instance_template(@project_id, cloud_desc.instance_template.sub(/.*?\/([^\/]+)$/, '\1'))
+
+          iface = template.properties.network_interfaces.first
+          iface.network.match(/(?:^|\/)projects\/(.*?)\/.*?\/networks\/([^\/]+)(?:$|\/)/)
+          vpc_proj = Regexp.last_match[1]
+          vpc_id = Regexp.last_match[2]
+
+          bok['vpc'] = MU::Config::Ref.get(
+            id: vpc_id,
+            cloud: "Google",
+            habitat: MU::Config::Ref.get(
+              id: vpc_proj,
+              cloud: "Google",
+              credentials: @credentials,
+              type: "habitats"
+            ),
+            credentials: @credentials,
+            type: "vpcs",
+            subnet_pref: "any" # "anywhere in this VPC" is what matters
+          )
+
+          bok['basis'] = {
+            "launch_config" => {
+              "name" => bok['name']
+            }
+          }
+
+          template.properties.disks.each { |disk|
+            if disk.initialize_params.source_image and disk.boot
+              bok['basis']['launch_config']['image_id'] ||= disk.initialize_params.source_image.sub(/^https:\/\/www\.googleapis\.com\/compute\/[^\/]+\//, '')
+            elsif disk.type != "SCRATCH"
+              bok['basis']['launch_config']['storage'] ||= []
+              storage_blob = {
+                "size" => disk.initialize_params.disk_size_gb,
+                "device" => "/dev/xvd"+(disk.index+97).chr.downcase
+              }
+              bok['basis']['launch_config']['storage'] <<  storage_blob
+            else
+              MU.log "Need to sort out scratch disks", MU::WARN, details: disk
+            end
+            
+          }
+
+          if template.properties.labels
+            bok['tags'] = template.properties.labels.keys.map { |k| { "key" => k, "value" => template.properties.labels[k] } }
+          end
+          if template.properties.tags and template.properties.tags.items and template.properties.tags.items.size > 0
+            bok['network_tags'] = template.properties.tags.items
+          end
+          bok['src_dst_check'] = !template.properties.can_ip_forward
+          bok['basis']['launch_config']['size'] = template.properties.machine_type.sub(/.*?\/([^\/]+)$/, '\1')
+          bok['project'] = @project_id
+          if template.properties.service_accounts
+            bok['scopes'] = template.properties.service_accounts.map { |sa| sa.scopes }.flatten.uniq
+          end
+          if template.properties.metadata and template.properties.metadata.items
+            bok['metadata'] = template.properties.metadata.items.map { |m| MU.structToHash(m) }
+          end
+
+          # Skip nodes that are just members of GKE clusters
+          if bok['name'].match(/^gke-.*?-[a-f0-9]+-[a-z0-9]+$/) and
+             bok['basis']['launch_config']['image_id'].match(/(:?^|\/)projects\/gke-node-images\//)
+            gke_ish = true
+            bok['network_tags'].each { |tag|
+              gke_ish = false if !tag.match(/^gke-/)
+            }
+            if gke_ish
+              MU.log "ServerPool #{bok['name']} appears to belong to a ContainerCluster, skipping adoption", MU::NOTICE
+              return nil
+            end
+          end
+#MU.log bok['name'], MU::WARN, details: [cloud_desc, template]
+
+          bok
         end
 
         # Cloud-specific configuration properties.
@@ -184,6 +326,15 @@ module MU
         def self.schema(config)
           toplevel_required = []
           schema = {
+            "ssh_user" => MU::Cloud::Google::Server.schema(config)[1]["ssh_user"],
+            "metadata" => MU::Cloud::Google::Server.schema(config)[1]["metadata"],
+            "service_account" => MU::Cloud::Google::Server.schema(config)[1]["service_account"],
+            "scopes" => MU::Cloud::Google::Server.schema(config)[1]["scopes"],
+            "network_tags" => MU::Cloud::Google::Server.schema(config)[1]["network_tags"],
+            "availability_zone" => {
+              "type" => "string",
+              "description" => "Target a specific availability zone for this pool, which will create zonal instance managers and scalers instead of regional ones."
+            },
             "named_ports" => {
               "type" => "array",
               "items" => {
@@ -211,6 +362,38 @@ module MU
         # @return [Boolean]: True if validation succeeded, False otherwise
         def self.validateConfig(pool, configurator)
           ok = true
+start = Time.now
+          pool['project'] ||= MU::Cloud::Google.defaultProject(pool['credentials'])
+          if pool['service_account']
+            pool['service_account']['cloud'] = "Google"
+            pool['service_account']['habitat'] ||= pool['project']
+            found = MU::Config::Ref.get(pool['service_account'])
+            if found.id and !found.kitten
+              MU.log "GKE pool #{pool['name']} failed to locate service account #{pool['service_account']} in project #{pool['project']}", MU::ERR
+              ok = false
+            end
+          else
+            user = {
+              "name" => pool['name'],
+              "cloud" => "Google",
+              "project" => pool["project"],
+              "credentials" => pool["credentials"],
+              "type" => "service"
+            }
+            configurator.insertKitten(user, "users", true)
+            pool['dependencies'] ||= []
+            pool['service_account'] = MU::Config::Ref.get(
+              type: "users",
+              cloud: "Google",
+              name: pool["name"],
+              project: pool["project"],
+              credentials: pool["credentials"]
+            )
+            pool['dependencies'] << {
+              "type" => "user",
+              "name" => pool["name"]
+            }
+          end
 
           pool['named_ports'] ||= []
           if !pool['named_ports'].include?({"name" => "ssh", "port" => 22})
@@ -224,8 +407,9 @@ module MU
             ok = false if launch['size'].nil?
 
             if launch['image_id'].nil?
-              if MU::Config.google_images.has_key?(pool['platform'])
-                launch['image_id'] = configurator.getTail("server_pool"+pool['name']+"Image", value: MU::Config.google_images[pool['platform']], prettyname: "server_pool"+pool['name']+"Image", cloudtype: "Google::Apis::ComputeBeta::Image")
+              img_id = MU::Cloud.getStockImage("Google", platform: pool['platform'])
+              if img_id
+                launch['image_id'] = configurator.getTail("server_pool"+pool['name']+"Image", value: img_id, prettyname: "server_pool"+pool['name']+"Image", cloudtype: "Google::Apis::ComputeV1::Image")
               else
                 MU.log "No image specified for #{pool['name']} and no default available for platform #{pool['platform']}", MU::ERR, details: launch
                 ok = false
@@ -270,6 +454,7 @@ module MU
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
+          return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
 
           if !flags["global"]
             ["region_autoscaler", "region_instance_group_manager"].each { |type|

data/modules/mu/clouds/google/user.rb

@@ -17,45 +17,160 @@ module MU
     class Google
       # A user as configured in {MU::Config::BasketofKittens::users}
       class User < MU::Cloud::User
-        @deploy = nil
-        @config = nil
-        attr_reader :mu_name
-        attr_reader :config
-        attr_reader :cloud_id
-
-        # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
-        # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::users}
-        def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
-          @deploy = mommacat
-          @config = MU::Config.manxify(kitten_cfg)
-          @cloud_id ||= cloud_id
-          @mu_name ||= @deploy.getResourceName(@config["name"])
+
+        # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
+        # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
+        def initialize(**args)
+          super
+
+          # If we're being reverse-engineered from a cloud descriptor, use that
+          # to determine what sort of account we are.
+          if args[:from_cloud_desc]
+            MU::Cloud::Google.admin_directory
+            MU::Cloud::Google.iam
+            if args[:from_cloud_desc].class == ::Google::Apis::AdminDirectoryV1::User
+              @config['type'] = "interactive"
+            elsif args[:from_cloud_desc].class == ::Google::Apis::IamV1::ServiceAccount
+              @config['type'] = "service"
+              @config['name'] = args[:from_cloud_desc].display_name
+              if @config['name'].nil? or @config['name'].empty?
+                @config['name'] = args[:from_cloud_desc].name.sub(/.*?\/([^\/@]+)(?:@[^\/]*)?$/, '\1')
+              end
+              @cloud_id = args[:from_cloud_desc].name
+            else
+              raise MuError, "Google::User got from_cloud_desc arg of class #{args[:from_cloud_desc].class.name}, but doesn't know what to do with it"
+            end
+          end
+
+          @mu_name ||= if (@config['unique_name'] or @config['type'] == "service") and !@config['scrub_mu_isms']
+            @deploy.getResourceName(@config["name"])
+          else
+            @config['name']
+          end
+
         end
 
         # Called automatically by {MU::Deploy#createResources}
         def create
-          if @config['type'] == "interactive"
-            bind_human_user
-          else
+          if @config['type'] == "service"
+            acct_id = @config['scrub_mu_isms'] ? @config['name'] : @deploy.getResourceName(@config["name"], max_length: 30).downcase
             req_obj = MU::Cloud::Google.iam(:CreateServiceAccountRequest).new(
-              account_id: @deploy.getResourceName(@config["name"], max_length: 30).downcase,
+              account_id: acct_id,
               service_account: MU::Cloud::Google.iam(:ServiceAccount).new(
-                display_name: @mu_name
+                display_name: @mu_name,
+                description: @config['scrub_mu_isms'] ? nil : @deploy.deploy_id
+              )
+            )
+            if @config['use_if_exists']
+              # XXX maybe just set @cloud_id to projects/#{@project_id}/serviceAccounts/#{@mu_name}@#{@project_id}.iam.gserviceaccount.com and see if cloud_desc returns something
+              found = MU::Cloud::Google::User.find(project: @project_id, cloud_id: @mu_name)
+              if found.size == 1
+                @cloud_id = found.keys.first
+                MU.log "Service account #{@cloud_id} already existed, using it"
+              end
+            end
+
+            if !@cloud_id
+              MU.log "Creating service account #{@mu_name}"
+              resp = MU::Cloud::Google.iam(credentials: @config['credentials']).create_service_account(
+                "projects/"+@config['project'],
+                req_obj
               )
+              @cloud_id = resp.name
+            end
+
+            # make sure we've been created before moving on
+            begin
+              cloud_desc
+            rescue ::Google::Apis::ClientError => e
+              if e.message.match(/notFound:/)
+                sleep 3
+                retry
+              end
+            end
+          elsif @config['external']
+            @cloud_id = @config['email']
+            MU::Cloud::Google::Role.bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
+          else
+            if !@config['email']
+              domains = MU::Cloud::Google.admin_directory(credentials: @credentials).list_domains(@customer)
+              @config['email'] = @mu_name.gsub(/@.*/, "")+"@"+domains.domains.first.domain_name
+            end
+
+            username_obj = MU::Cloud::Google.admin_directory(:UserName).new(
+              given_name: (@config['given_name'] || @config['name']),
+              family_name: (@config['family_name'] || @deploy.deploy_id),
+              full_name: @mu_name
             )
-            MU.log "Creating service account #{@mu_name}"
-            MU::Cloud::Google.iam(credentials: @config['credentials']).create_service_account(
-              "projects/"+@config['project'],
-              req_obj
+
+            user_obj = MU::Cloud::Google.admin_directory(:User).new(
+              name: username_obj,
+              primary_email: @config['email'],
+              suspended: @config['suspend'],
+              is_admin: @config['admin'],
+              password: MU.generateWindowsPassword,
+              change_password_at_next_login: (@config.has_key?('force_password_change') ? @config['force_password_change'] : true)
             )
+
+            MU.log "Creating user #{@mu_name}", details: user_obj
+            resp = MU::Cloud::Google.admin_directory(credentials: @credentials).insert_user(user_obj)
+            @cloud_id = resp.primary_email
+
           end
         end
 
         # Called automatically by {MU::Deploy#createResources}
         def groom
-          if @config['type'] == "interactive"
-            bind_human_user
+          if @config['external']
+            MU::Cloud::Google::Role.bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
+          elsif @config['type'] == "interactive"
+            need_update = false
+            MU::Cloud::Google::Role.bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
+
+            if @config['force_password_change'] and !cloud_desc.change_password_at_next_login
+              MU.log "Forcing #{@mu_name} to change their password at next login", MU::NOTICE
+              need_update = true
+            elsif @config.has_key?("force_password_change") and
+                  !@config['force_password_change'] and
+                  cloud_desc.change_password_at_next_login
+              MU.log "No longer forcing #{@mu_name} to change their password at next login", MU::NOTICE
+              need_update = true
+            end
+            if @config['admin'] != cloud_desc.is_admin
+              MU.log "Setting 'is_admin' flag to #{@config['admin'].to_s} for directory user #{@mu_name}", MU::NOTICE
+              MU::Cloud::Google.admin_directory(credentials: @credentials).make_user_admin(@cloud_id, MU::Cloud::Google.admin_directory(:UserMakeAdmin).new(status: @config['admin']))
+            end
+
+            if @config['suspend'] != cloud_desc.suspended
+              need_update = true
+            end
+            if cloud_desc.name.given_name != (@config['given_name'] || @config['name']) or
+               cloud_desc.name.family_name != (@config['family_name'] || @deploy.deploy_id) or
+               cloud_desc.primary_email != @config['email']
+              need_update = true
+            end
+
+            if need_update
+              username_obj = MU::Cloud::Google.admin_directory(:UserName).new(
+                given_name: (@config['given_name'] || @config['name']),
+                family_name: (@config['family_name'] || @deploy.deploy_id),
+                full_name: @mu_name
+              )
+              user_obj = MU::Cloud::Google.admin_directory(:User).new(
+                name: username_obj,
+                primary_email: @config['email'],
+                suspended: @config['suspend'],
+                change_password_at_next_login: (@config.has_key?('force_password_change') ? @config['force_password_change'] : true)
+              )
+
+              MU.log "Updating directory user #{@mu_name}", MU::NOTICE, details: user_obj
+
+              resp = MU::Cloud::Google.admin_directory(credentials: @credentials).update_user(@cloud_id, user_obj)
+              @cloud_id = resp.primary_email
+            end
+
           else
+            MU::Cloud::Google::Role.bindFromConfig("serviceAccount", @cloud_id.gsub(/.*?\/([^\/]+)$/, '\1'), @config['roles'], credentials: @config['credentials'])
             if @config['create_api_key']
               resp = MU::Cloud::Google.iam(credentials: @config['credentials']).list_project_service_account_keys(
                 cloud_desc.name
@@ -73,34 +188,32 @@ module MU
         end
 
         # Retrieve the cloud descriptor for this resource.
+        # @return [Google::Apis::Core::Hashable]
         def cloud_desc
-          if @config['type'] == "interactive"
-            return nil
-          else
-            resp = MU::Cloud::Google.iam(credentials: @config['credentials']).list_project_service_accounts(
-              "projects/"+@config["project"]
-            )
-
-            if resp and resp.accounts
-              resp.accounts.each { |sa|
-                if sa.display_name and sa.display_name == @mu_name
-                  return sa
-                end
-              }
+          if @config['type'] == "interactive" or !@config['type']
+             @config['type'] ||= "interactive"
+            if !@config['external']
+              return MU::Cloud::Google.admin_directory(credentials: @config['credentials']).get_user(@cloud_id)
+            else
+              return nil
             end
+          else
+            @config['type'] ||= "service"
+            return MU::Cloud::Google.iam(credentials: @config['credentials']).get_project_service_account(@cloud_id)
           end
+
         end
 
         # Return the metadata for this user configuration
         # @return [Hash]
         def notify
-          description = MU.structToHash(cloud_desc)
-          if description
-            description.delete(:etag)
-            return description
+          description = if !@config['external']
+            MU.structToHash(cloud_desc)
+          else
+            {}
           end
-          {
-          }
+          description.delete(:etag)
+          description
         end
 
         # Does this resource type exist as a global (cloud-wide) artifact, or
@@ -113,7 +226,7 @@ module MU
         # Denote whether this resource implementation is experiment, ready for
         # testing, or ready for production use.
         def self.quality
-          MU::Cloud::ALPHA
+          MU::Cloud::RELEASE
         end
 
         # Remove all users associated with the currently loaded deployment.
@@ -122,6 +235,33 @@ module MU
         # @param region [String]: The cloud provider region
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+          my_domains = MU::Cloud::Google.getDomains(credentials)
+          my_org = MU::Cloud::Google.getOrg(credentials)
+
+          # We don't have a good way of tagging directory users, so we rely
+          # on the known parameter, which is pulled from deployment metadata
+          if flags['known'] and my_org
+            dir_users = MU::Cloud::Google.admin_directory(credentials: credentials).list_users(customer: MU::Cloud::Google.customerID(credentials)).users
+            if dir_users
+              dir_users.each { |user|
+                if flags['known'].include?(user.primary_email)
+                  MU.log "Deleting user #{user.primary_email} from #{my_org.display_name}", details: user
+                  if !noop
+                    MU::Cloud::Google.admin_directory(credentials: credentials).delete_user(user.id)
+                  end
+                end
+              }
+
+              flags['known'].each { |user_email|
+                next if user_email.nil?
+                next if !user_email.match(/^[^\/]+@[^\/]+$/)
+
+                MU::Cloud::Google::Role.removeBindings("user", user_email, credentials: credentials, noop: noop)
+              }
+
+            end
+          end
+
           flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
           resp = MU::Cloud::Google.iam(credentials: credentials).list_project_service_accounts(
             "projects/"+flags["project"]
@@ -129,7 +269,8 @@ module MU
 
           if resp and resp.accounts and MU.deploy_id
             resp.accounts.each { |sa|
-              if sa.display_name and sa.display_name.match(/^#{Regexp.quote(MU.deploy_id)}-/i)
+              if (sa.description and sa.description == MU.deploy_id) or
+                 (sa.display_name and sa.display_name.match(/^#{Regexp.quote(MU.deploy_id)}-/i))
                 begin
                   MU.log "Deleting service account #{sa.name}", details: sa
                   if !noop
@@ -143,30 +284,167 @@ module MU
           end
         end
 
-        # Locate an existing user.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region.
-        # @param flags [Hash]: Optional flags
-        # @return [OpenStruct]: The cloud provider's complete descriptions of matching user group.
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {}, tag_key: nil, tag_value: nil)
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
-          found = nil
-          resp = MU::Cloud::Google.iam(credentials: credentials).list_project_service_accounts(
-            "projects/"+flags["project"]
-          )
+        # Locate and return cloud provider descriptors of this resource type
+        # which match the provided parameters, or all visible resources if no
+        # filters are specified. At minimum, implementations of +find+ must
+        # honor +credentials+ and +cloud_id+ arguments. We may optionally
+        # support other search methods, such as +tag_key+ and +tag_value+, or
+        # cloud-specific arguments like +project+. See also {MU::MommaCat.findStray}.
+        # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching resources
+        def self.find(**args)
+          cred_cfg = MU::Cloud::Google.credConfig(args[:credentials])
+          args[:project] ||= args[:habitat]
 
-          if resp and resp.accounts
-            resp.accounts.each { |sa|
-              if sa.display_name and sa.display_name == cloud_id
-                found ||= {}
-                found[cloud_id] = sa
+          found = {}
+
+          if args[:cloud_id] and args[:flags] and
+             args[:flags]["skip_provider_owned"] and
+             MU::Cloud::Google::User.cannedServiceAcctName?(args[:cloud_id])
+            return found
+          end
+
+          # If the project id is embedded in the cloud_id, honor it
+          if args[:cloud_id]
+            if args[:cloud_id].match(/projects\/(.+?)\//)
+              args[:project] = Regexp.last_match[1]
+            elsif args[:cloud_id].match(/@([^\.]+)\.iam\.gserviceaccount\.com$/)
+              args[:project] = Regexp.last_match[1]
+            end
+          end
+
+          if args[:project]
+            # project-local service accounts
+            resp = begin
+              MU::Cloud::Google.iam(credentials: args[:credentials]).list_project_service_accounts(
+                "projects/"+args[:project]
+              )
+            rescue ::Google::Apis::ClientError => e
+              MU.log "Do not have permissions to retrieve service accounts for project #{args[:project]}", MU::WARN
+            end
+
+            if resp and resp.accounts
+              resp.accounts.each { |sa|
+                if args[:flags] and args[:flags]["skip_provider_owned"] and
+                   MU::Cloud::Google::User.cannedServiceAcctName?(sa.name)
+                  next
+                end
+                if !args[:cloud_id] or (sa.display_name and sa.display_name == args[:cloud_id]) or (sa.name and sa.name == args[:cloud_id]) or (sa.email and sa.email == args[:cloud_id])
+                  found[sa.name] = sa
+                end
+              }
+            end
+          else
+            if cred_cfg['masquerade_as']
+              resp = MU::Cloud::Google.admin_directory(credentials: args[:credentials]).list_users(customer: MU::Cloud::Google.customerID(args[:credentials]), show_deleted: false)
+              if resp and resp.users
+                resp.users.each { |u|
+                  found[u.primary_email] = u
+                }
               end
-            }
+            end
           end
 
           found
         end
 
+        # Try to determine whether the given string looks like a pre-configured
+        # GCP service account, as distinct from one we might create or manage
+        def self.cannedServiceAcctName?(name)
+          return false if !name
+          name.match(/\b\d+\-compute@developer\.gserviceaccount\.com$/) or
+          name.match(/\bproject-\d+@storage-transfer-service\.iam\.gserviceaccount\.com$/) or
+          name.match(/\b\d+@cloudbuild\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@containerregistry\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@container-analysis\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@gcp-sa-bigquerydatatransfer\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@gcp-sa-cloudasset\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@gcp-sa-cloudiot\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@gcp-sa-cloudscheduler\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@compute-system\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@container-engine-robot\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@gcp-admin-robot\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@gcp-sa-containerscanning\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@dataflow-service-producer-prod\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@dataproc-accounts\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@endpoints-portal\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@cloud-filer\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@cloud-redis\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@firebase-rules\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@cloud-tpu\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@gcp-sa-vpcaccess\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@gcp-sa-websecurityscanner\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bservice-\d+@sourcerepo-service-accounts\.iam\.gserviceaccount\.com$/) or
+          name.match(/\bp\d+\-\d+@gcp-sa-logging\.iam\.gserviceaccount\.com$/)
+        end
+
+        # We can either refer to a service account, which is scoped to a project
+        # (a +Habitat+ in Mu parlance), or a "real" user, which comes from
+        # an external directory like GMail, GSuite, or Cloud Identity.
+        def self.canLiveIn
+          [:Habitat, nil]
+        end
+
+        # Reverse-map our cloud description into a runnable config hash.
+        # We assume that any values we have in +@config+ are placeholders, and
+        # calculate our own accordingly based on what's live in the cloud.
+        def toKitten(rootparent: nil, billing: nil, habitats: nil)
+          if MU::Cloud::Google::User.cannedServiceAcctName?(@cloud_id)
+            return nil
+          end
+
+          bok = {
+            "cloud" => "Google",
+            "credentials" => @config['credentials']
+          }
+
+          if cloud_desc.nil?
+            MU.log @config['name']+" couldn't fetch its cloud descriptor", MU::WARN, details: @cloud_id
+            return nil
+          end
+
+          user_roles = MU::Cloud::Google::Role.getAllBindings(@config['credentials'])["by_entity"]
+
+          if cloud_desc.nil?
+            MU.log "FAILED TO FIND CLOUD DESCRIPTOR FOR #{self}", MU::ERR, details: @config
+            return nil
+          end
+
+          bok['name'] = @config['name']
+          bok['cloud_id'] = @cloud_id
+          bok['type'] = @config['type']
+          bok['type'] ||= "service"
+          if bok['type'] == "service"
+            bok['project'] = @project_id
+            keys = MU::Cloud::Google.iam(credentials: @config['credentials']).list_project_service_account_keys(@cloud_id)
+
+            if keys and keys.keys and keys.keys.size > 0
+              bok['create_api_key'] = true
+            end
+#            MU.log "service account #{@cloud_id}", MU::NOTICE, details: MU::Cloud::Google.iam(credentials: @config['credentials']).get_project_service_account_iam_policy(cloud_desc.name)
+            if user_roles["serviceAccount"] and
+               user_roles["serviceAccount"][bok['cloud_id']] and
+               user_roles["serviceAccount"][bok['cloud_id']].size > 0
+              bok['roles'] = MU::Cloud::Google::Role.entityBindingsToSchema(user_roles["serviceAccount"][bok['cloud_id']])
+            end
+          else
+            if user_roles["user"] and
+               user_roles["user"][bok['cloud_id']] and
+               user_roles["user"][bok['cloud_id']].size > 0
+              bok['roles'] = MU::Cloud::Google::Role.entityBindingsToSchema(user_roles["user"][bok['cloud_id']], credentials: @config['credentials'])
+            end
+            bok['given_name'] = cloud_desc.name.given_name
+            bok['family_name'] = cloud_desc.name.family_name
+            bok['email'] = cloud_desc.primary_email
+            bok['suspend'] = cloud_desc.suspended
+            bok['admin'] = cloud_desc.is_admin
+          end
+
+          bok['use_if_exists'] = true
+
+          bok
+       end
+
         # Cloud-specific configuration properties.
         # @param config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
@@ -175,24 +453,68 @@ module MU
           schema = {
             "name" => {
               "type" => "string",
-              "description" => "This must be the email address of an existing Google user account (+foo@gmail.com+), or of a federated GSuite or Cloud Identity domain account from your organization."
+              "description" => "If the +type+ of this account is not +service+, this can include an optional @domain component (<tt>foo@example.com</tt>), which is equivalent to the +domain+ configuration option. The following rules apply to +directory+ (non-<tt>service</tt>) accounts only:
+
+If the domain portion is not specified, and we manage exactly one GSuite or Cloud Identity domain, we will attempt to create the user in that domain.
+
+If we do not manage any domains, and none are specified, we will assume <tt>@gmail.com</tt> for the domain and attempt to bind an existing external GMail user to roles under our jurisdiction.
+
+If the domain portion is specified, and our credentials can manage that domain via GSuite or Cloud Identity, we will attempt to create the user in that domain.
+
+If it is a domain we do not manage, we will attempt to bind an existing external user from that domain to roles under our jurisdiction.
+
+If we are binding (rather than creating) a user and no roles are specified, we will default to +roles/viewer+ at the organization scope. If our credentials do not manage an organization, we will grant this role in our default project.
+
+"
+            },
+            "domain" => {
+              "type" => "string",
+              "description" => "If creating or binding an +interactive+ user, this is the domain of which the user should be a member. This can instead be embedded in the {name} field: +foo@example.com+."
+            },
+            "given_name" => {
+              "type" => "string",
+              "description" => "Optionally set the +given_name+ field of a +directory+ account. Ignored for +service+ accounts."
+            },
+            "first_name" => {
+              "type" => "string",
+              "description" => "Alias for +given_name+"
+            },
+            "family_name" => {
+              "type" => "string",
+              "description" => "Optionally set the +family_name+ field of a +directory+ account. Ignored for +service+ accounts."
+            },
+            "last_name" => {
+              "type" => "string",
+              "description" => "Alias for +family_name+"
+            },
+            "email" => {
+              "type" => "string",
+              "description" => "Canonical email address for a +directory+ user. If not specified, will be set to +name@domain+."
+            },
+            "external" => {
+              "type" => "boolean",
+              "description" => "Explicitly flag this user as originating from an external domain. This should always autodetect correctly."
+            },
+            "admin" => {
+              "type" => "boolean",
+              "description" => "If the user is +interactive+ and resides in a domain we manage, set their +is_admin+ flag.",
+              "default" => false
+            },
+            "suspend" => {
+              "type" => "boolean",
+              "description" => "If the user is +interactive+ and resides in a domain we manage, this can be used to lock their account.",
+              "default" => false
             },
             "type" => {
               "type" => "string",
-              "description" => "'interactive' will attempt to bind an existing user; 'service' will create a service account and generate API keys"
+              "description" => "'interactive' will either attempt to bind an existing user to a role under our jurisdiction, or create a new directory user, depending on the domain of the user specified and whether we manage any directories; 'service' will create a service account and generate API keys.",
+              "enum" => ["interactive", "service"],
+              "default" => "interactive"
             },
             "roles" => {
               "type" => "array",
               "description" => "One or more Google IAM roles to associate with this user.",
-              "default" => ["roles/viewer"],
-              "items" => {
-                "type" => "string",
-                "description" => "One or more Google IAM roles to associate with this user. Google Cloud human user accounts (as distinct from service accounts) are not created directly; pre-existing Google accounts are associated with a project by being bound to one or more roles in that project. If no roles are specified, we default to +roles/viewer+, which permits read-only access project-wide."
-              }
-            },
-            "project" => {
-              "type" => "string",
-              "description" => "The project into which to deploy resources"
+              "items" => MU::Cloud::Google::Role.ref_schema
             }
           }
           [toplevel_required, schema]
@@ -205,75 +527,107 @@ module MU
         def self.validateConfig(user, configurator)
           ok = true
 
-          # admin_directory only works in a GSuite environment
-          if !user['name'].match(/@/i) and MU::Cloud::Google.credConfig(user['credentials'])['masquerade_as']
-            # XXX flesh this check out, need to test with a GSuite site
-            pp MU::Cloud::Google.admin_directory(credentials: user['credentials']).get_user(user['name'])
+          my_domains = MU::Cloud::Google.getDomains(user['credentials'])
+          my_org = MU::Cloud::Google.getOrg(user['credentials'])
+
+          # Deal with these name alias fields, here for the convenience of your
+          # easily confused english-centric type of person
+          user['given_name'] ||= user['first_name']
+          user['family_name'] ||= user['last_name']
+          user.delete("first_name")
+          user.delete("last_name")
+
+          if user['name'].match(/@(.*+)$/)
+            domain = Regexp.last_match[1].downcase
+            if domain and user['domain'] and domain != user['domain'].downcase
+              MU.log "User #{user['name']} had a domain component, but the domain field was also specified (#{user['domain']}) and they don't match."
+              ok = false
+            end
+            user['domain'] = domain
+            if user['type'] == "service"
+              MU.log "Username #{user['name']} appears to be a directory or external username, cannot use with 'service'", MU::ERR
+              ok = false
+            else
+              user['type'] = "interactive"
+              if !my_domains or !my_domains.include?(domain)
+                user['external'] = true
+
+                if !["gmail.com", "google.com"].include?(domain)
+                  MU.log "#{user['name']} appears to be a member of a domain that our credentials (#{user['credentials']}) do not manage; attempts to grant access for this user may fail!", MU::WARN
+                end
+
+                if !user['roles'] or user['roles'].empty?
+                  user['roles'] = [
+                    {
+                      "role" => {
+                        "id" => "roles/viewer"
+                      }
+                    }
+                  ]
+                  MU.log "External Google user specified with no role binding, will grant 'viewer' in #{my_org ? "organization #{my_org.display_name}" : "project #{user['project']}"}", MU::WARN
+                end
+              else # this is actually targeting a domain we manage! yay!
+              end
+            end
+          elsif user['type'] != "service"
+            if !user['domain']
+              if my_domains.size == 1
+                user['domain'] = my_domains.first
+              elsif my_domains.size > 1
+                MU.log "Google interactive User #{user['name']} did not specify a domain, and we have multiple defaults available. Must specify exactly one.", MU::ERR, details: my_domains
+                ok = false
+              else
+                user['domain'] = "gmail.com"
+              end
+            end
           end
 
-          if user['groups'] and user['groups'].size > 0 and
-             !MU::Cloud::Google.credConfig(user['credentials'])['masquerade_as']
-            MU.log "Cannot change Google group memberships in non-GSuite environments.\nVisit https://groups.google.com to manage groups.", MU::ERR
+          if user['domain']
+            user['email'] ||= user['name'].gsub(/@.*/, "")+"@"+user['domain']
+          end
+
+          if user['groups'] and user['groups'].size > 0 and my_org.nil?
+            MU.log "Cannot change Google group memberships with credentials that do not manage GSuite or Cloud Identity.\nVisit https://groups.google.com to manage groups.", MU::ERR
             ok = false
           end
 
+          if user['type'] == "service"
+            user['project'] ||= MU::Cloud::Google.defaultProject(user['credentials'])
+          end
+
           if user['type'] != "service" and user["create_api_key"]
             MU.log "Only service accounts can have API keys in Google Cloud", MU::ERR
             ok = false
           end
 
-          ok
-        end
-
-        private
-
-        def bind_human_user
-          bindings = []
-          ext_policy = MU::Cloud::Google.resource_manager(credentials: @config['credentials']).get_project_iam_policy(
-            @config['project']
-          )
+          user['dependencies'] ||= []
+          if user['roles']
+            user['roles'].each { |r|
+              if r['role'] and r['role']['name'] and
+                 (!r['role']['deploy_id'] and !r['role']['id'])
+                user['dependencies'] << {
+                  "type" => "role",
+                  "name" => r['role']['name']
+                }
+              end
 
-          change_needed = false
-          @config['roles'].each { |role|
-            seen = false
-            ext_policy.bindings.each { |b|
-              if b.role == role
-                seen = true
-                if !b.members.include?("user:"+@config['name'])
-                  change_needed = true
-                  b.members << "user:"+@config['name']
+              if !r["projects"] and !r["organizations"] and !r["folders"]
+                if my_org
+                  r["organizations"] = [my_org.name]
+                else
+                  r["projects"] = [
+                    "id" => user["project"]
+                  ]
                 end
               end
             }
-            if !seen
-              ext_policy.bindings << MU::Cloud::Google.resource_manager(:Binding).new(
-                role: role,
-                members: ["user:"+@config['name']]
-              )
-              change_needed = true
-            end
-          }
-
-          if change_needed
-            req_obj = MU::Cloud::Google.resource_manager(:SetIamPolicyRequest).new(
-              policy: ext_policy
-            )
-            MU.log "Adding #{@config['name']} to Google Cloud project #{@config['project']}", details: @config['roles']
-
-            begin
-              MU::Cloud::Google.resource_manager(credentials: @config['credentials']).set_project_iam_policy(
-                @config['project'],
-                req_obj
-              )
-            rescue ::Google::Apis::ClientError => e
-              if e.message.match(/does not exist/i) and !MU::Cloud::Google.credConfig(@config['credentials'])['masquerade_as']
-                raise MuError, "User #{@config['name']} does not exist, and we cannot create Google user in non-GSuite environments.\nVisit https://accounts.google.com to create new accounts."
-              end
-              raise e
-            end
           end
+
+          ok
         end
 
+        private
+
       end
     end
   end