cloud-mu 2.1.0beta → 3.0.0beta

data/modules/mu/clouds/google/bucket.rb

@@ -17,34 +17,17 @@ module MU
     class Google
       # Support for Google Cloud Storage
       class Bucket < MU::Cloud::Bucket
-        @deploy = nil
-        @config = nil
-        @project_id = nil
-
-        attr_reader :mu_name
-        attr_reader :config
-        attr_reader :cloud_id
-
-        # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
-        # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::logs}
-        def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
-          @deploy = mommacat
-          @config = MU::Config.manxify(kitten_cfg)
-          @cloud_id ||= cloud_id
-          if mu_name
-            @mu_name = mu_name
-            @config['project'] ||= MU::Cloud::Google.defaultProject(@config['credentials'])
-            if !@project_id
-              project = MU::Cloud::Google.projectLookup(@config['project'], @deploy, sibling_only: true, raise_on_fail: false)
-              @project_id = project.nil? ? @config['project'] : project.cloudobj.cloud_id
-            end
-          end
+
+        # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like <tt>@vpc</tt>, for us.
+        # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
+        def initialize(**args)
+          super
           @mu_name ||= @deploy.getResourceName(@config["name"])
         end
 
         # Called automatically by {MU::Deploy#createResources}
         def create
-          @project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloudobj.cloud_id
+          @project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloud_id
           MU::Cloud::Google.storage(credentials: credentials).insert_bucket(@project_id, bucket_descriptor)
           @cloud_id = @mu_name.downcase
         end
@@ -72,6 +55,18 @@ module MU
             changed = true
           end
 
+          if @config['bucket_wide_acls'] and (!current.iam_configuration or
+             !current.iam_configuration.bucket_policy_only or
+             !current.iam_configuration.bucket_policy_only.enabled)
+            MU.log "Converting Cloud Storage bucket #{@cloud_id} to use bucket-wide ACLs only", MU::NOTICE
+            changed = true
+          elsif !@config['bucket_wide_acls'] and current.iam_configuration and
+                current.iam_configuration.bucket_policy_only and
+                current.iam_configuration.bucket_policy_only.enabled
+            MU.log "Converting Cloud Storage bucket #{@cloud_id} to use bucket and object ACLs", MU::NOTICE
+            changed = true
+          end
+
           if changed
             MU::Cloud::Google.storage(credentials: credentials).patch_bucket(@cloud_id, bucket_descriptor)
           end
@@ -79,18 +74,19 @@ module MU
           if @config['policies']
             @config['policies'].each { |pol|
               pol['grant_to'].each { |grantee|
+                grantee['id'] ||= grantee["identifier"]
                 entity = if grantee["type"]
                   sibling = deploy_obj.findLitterMate(
-                    name: grantee["identifier"],
+                    name: grantee["id"],
                     type: grantee["type"]
                   )
                   if sibling
                     sibling.cloudobj.cloud_id
                   else
-                    raise MuError, "Couldn't find a #{grantee["type"]} named #{grantee["identifier"]} when generating Cloud Storage access policy"
+                    raise MuError, "Couldn't find a #{grantee["type"]} named #{grantee["id"]} when generating Cloud Storage access policy"
                   end
                 else
-                  pol['grant_to'].first['identifier']
+                  pol['grant_to'].first['id']
                 end
 
                 if entity.match(/@/) and !entity.match(/^(group|user)\-/)
@@ -123,6 +119,14 @@ module MU
           end
         end
 
+        # Upload a file to a bucket.
+        # @param url [String]: Target URL, of the form gs://bucket/folder/file
+        # @param acl [String]: Canned ACL permission to assign to the object we upload
+        # @param file [String]: Path to a local file to write to our target location. One of +file+ or +data+ must be specified.
+        # @param data [String]: Data to write to our target location. One of +file+ or +data+ must be specified.
+        def self.upload(url, acl: "private", file: nil, data: nil, credentials: nil)
+        end
+
         # Does this resource type exist as a global (cloud-wide) artifact, or
         # is it localized to a region/zone?
         # @return [Boolean]
@@ -166,18 +170,135 @@ module MU
         end
 
         # Locate an existing bucket.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region.
-        # @param flags [Hash]: Optional flags
         # @return [OpenStruct]: The cloud provider's complete descriptions of matching bucket.
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {}, tag_key: nil, tag_value: nil)
+        def self.find(**args)
+          args[:project] ||= args[:habitat]
+          args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
+
           found = {}
-          if cloud_id
-            found[cloud_id] = MU::Cloud::Google.storage(credentials: credentials).get_bucket(cloud_id)
+          if args[:cloud_id]
+            found[args[:cloud_id]] = MU::Cloud::Google.storage(credentials: args[:credentials]).get_bucket(args[:cloud_id])
+          else
+            resp = begin
+              MU::Cloud::Google.storage(credentials: args[:credentials]).list_buckets(args[:project])
+            rescue ::Google::Apis::ClientError => e
+              raise e if !e.message.match(/forbidden:/)
+            end
+
+            if resp and resp.items
+              resp.items.each { |bucket|
+                found[bucket.id] = bucket
+              }
+            end
           end
+
           found
         end
 
+        # Reverse-map our cloud description into a runnable config hash.
+        # We assume that any values we have in +@config+ are placeholders, and
+        # calculate our own accordingly based on what's live in the cloud.
+        def toKitten(rootparent: nil, billing: nil, habitats: nil)
+          bok = {
+            "cloud" => "Google",
+            "credentials" => @config['credentials'],
+            "cloud_id" => @cloud_id
+          }
+
+          bok['name'] = cloud_desc.name
+          bok['project'] = @project_id
+          bok['storage_class'] = cloud_desc.storage_class
+          if cloud_desc.versioning and cloud_desc.versioning.enabled
+            bok['versioning'] = true
+          end
+          if cloud_desc.website
+            bok['web'] = true
+            if cloud_desc.website.not_found_page
+              bok['web_error_object'] = cloud_desc.website.not_found_page
+            end
+            if cloud_desc.website.main_page_suffix
+              bok['web_index_object'] = cloud_desc.website.main_page_suffix
+            end
+            pp cloud_desc
+          end
+
+#          MU.log "get_bucket_iam_policy", MU::NOTICE, details: MU::Cloud::Google.storage(credentials: @credentials).get_bucket_iam_policy(@cloud_id)
+          pols = MU::Cloud::Google.storage(credentials: @credentials).get_bucket_iam_policy(@cloud_id)
+
+          if pols and pols.bindings and pols.bindings.size > 0
+            bok['policies'] = []
+            count = 0
+            grantees = {}
+            pols.bindings.each { |binding|
+              grantees[binding.role] ||= []
+              binding.members.each { |grantee|
+                if grantee.match(/^(user|group):(.*)/)
+                  grantees[binding.role] << MU::Config::Ref.get(
+                    id: Regexp.last_match[2],
+                    type: Regexp.last_match[1]+"s",
+                    cloud: "Google",
+                    credentials: @credentials
+                  )
+                elsif grantee == "allUsers" or
+                      grantee == "allAuthenticatedUsers" or
+                      grantee.match(/^project(?:Owner|Editor|Viewer):/)
+                  grantees[binding.role] << { "id" => grantee }
+                elsif grantee.match(/^serviceAccount:(.*)/)
+                  sa_name = Regexp.last_match[1]
+                  if MU::Cloud::Google::User.cannedServiceAcctName?(sa_name)
+                    grantees[binding.role] << { "id" => grantee }
+                  else
+                    grantees[binding.role] << MU::Config::Ref.get(
+                      id: sa_name,
+                      type: "users",
+                      cloud: "Google",
+                      credentials: @credentials
+                    )
+                  end
+                else
+                  # *shrug*
+                  grantees[binding.role] << { "id" => grantee }
+                end
+              }
+            }
+
+            # munge together roles that apply to the exact same set of
+            # principals
+            reverse_map = {}
+            grantees.each_pair { |perm, grant_to|
+              reverse_map[grant_to] ||= []
+              reverse_map[grant_to] << perm
+            }
+            already_done = []
+
+            grantees.each_pair { |perm, grant_to|
+              if already_done.include?(perm+grant_to.to_s)
+                next
+              end
+              bok['policies'] << {
+                "name" => "policy"+count.to_s,
+                "grant_to" => grant_to,
+                "permissions" => reverse_map[grant_to]
+              }
+              reverse_map[grant_to].each { |doneperm|
+                already_done << doneperm+grant_to.to_s
+              }
+              count = count+1
+            }
+          end
+
+          if cloud_desc.iam_configuration and
+             cloud_desc.iam_configuration.bucket_policy_only and
+             cloud_desc.iam_configuration.bucket_policy_only.enabled
+            bok['bucket_wide_acls'] = true
+          else
+#            MU.log "list_bucket_access_controls", MU::NOTICE, details:  MU::Cloud::Google.storage(credentials: @credentials).list_bucket_access_controls(@cloud_id)
+#            MU.log "list_default_object_access_controls", MU::NOTICE, details:  MU::Cloud::Google.storage(credentials: @credentials).list_default_object_access_controls(@cloud_id)
+          end
+
+          bok
+        end
+
         # Cloud-specific configuration properties.
         # @param config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
@@ -188,6 +309,11 @@ module MU
               "type" => "string",
               "enum" => ["MULTI_REGIONAL", "REGIONAL", "STANDARD", "NEARLINE", "COLDLINE", "DURABLE_REDUCED_AVAILABILITY"],
               "default" => "STANDARD"
+            },
+            "bucket_wide_acls" => {
+              "type" => "boolean",
+              "default" => false,
+              "description" => "Disables object-level access controls in favor of bucket-wide policies"
             }
           }
           [toplevel_required, schema]
@@ -200,6 +326,7 @@ module MU
         # @return [Boolean]: True if validation succeeded, False otherwise
         def self.validateConfig(bucket, configurator)
           ok = true
+          bucket['project'] ||= MU::Cloud::Google.defaultProject(bucket['credentials'])
 
           if bucket['policies']
             bucket['policies'].each { |pol|
@@ -245,6 +372,20 @@ module MU
             params[:versioning] = MU::Cloud::Google.storage(:Bucket)::Versioning.new(enabled: false)
           end
 
+          if @config['bucket_wide_acls']
+            params[:iam_configuration] =  MU::Cloud::Google.storage(:Bucket)::IamConfiguration.new(
+              bucket_policy_only: MU::Cloud::Google.storage(:Bucket)::IamConfiguration::BucketPolicyOnly.new(
+                enabled: @config['bucket_wide_acls']
+              )
+            )
+          else
+            params[:iam_configuration] =  MU::Cloud::Google.storage(:Bucket)::IamConfiguration.new(
+              bucket_policy_only: MU::Cloud::Google.storage(:Bucket)::IamConfiguration::BucketPolicyOnly.new(
+                enabled: false
+              )
+            )
+          end
+
           MU::Cloud::Google.storage(:Bucket).new(params)
         end
 

data/modules/mu/clouds/google/container_cluster.rb

@@ -1,4 +1,4 @@
-# Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
+# Copyright:: Copyright (c) 2019 eGlobalTech, Inc., all rights reserved
 #
 # Licensed under the BSD-3 license (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,33 +17,13 @@ module MU
     class Google
       # A Kubernetes cluster as configured in {MU::Config::BasketofKittens::container_clusters}
       class ContainerCluster < MU::Cloud::ContainerCluster
-        @deploy = nil
-        @config = nil
-        attr_reader :mu_name
-        attr_reader :cloud_id
-        attr_reader :config
-        attr_reader :groomer    
-
-        # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
-        # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::container_clusters}
-        def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
-          @deploy = mommacat
-          @config = MU::Config.manxify(kitten_cfg)
-          @cloud_id ||= cloud_id
-          # @mu_name = mu_name ? mu_name : @deploy.getResourceName(@config["name"])
-          @config["groomer"] = MU::Config.defaultGroomer unless @config["groomer"]
-          @groomclass = MU::Groomer.loadGroomer(@config["groomer"])
-
-          if !mu_name.nil?
-            @mu_name = mu_name
-            deploydata = describe[2]
-            @config['availability_zone'] = deploydata['zone']
-            @config['project'] ||= MU::Cloud::Google.defaultProject(@config['credentials'])
-            if !@project_id
-              project = MU::Cloud::Google.projectLookup(@config['project'], @deploy, sibling_only: true, raise_on_fail: false)
-              @project_id = project.nil? ? @config['project'] : project.cloudobj.cloud_id
-            end
-          else
+
+        # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like <tt>@vpc</tt>, for us.
+        # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
+        def initialize(**args)
+          super
+
+          if !@mu_name
             @mu_name ||= @deploy.getResourceName(@config["name"], max_length: 40)
           end
         end
@@ -52,16 +32,11 @@ module MU
         # Called automatically by {MU::Deploy#createResources}
         # @return [String]: The cloud provider's identifier for this GKE instance.
         def create
-          labels = {}
-          MU::MommaCat.listStandardTags.each_pair { |name, value|
-            if !value.nil?
-              labels[name.downcase] = value.downcase.gsub(/[^a-z0-9\-\_]/i, "_")
-            end
-          }
+          labels = Hash[@tags.keys.map { |k|
+            [k.downcase, @tags[k].downcase.gsub(/[^-_a-z0-9]/, '-')] }
+          ]
           labels["name"] = MU::Cloud::Google.nameStr(@mu_name)
 
-          @config['availability_zone'] ||= MU::Cloud::Google.listAZs(@config['region']).sample
-
           if @vpc.nil? and @config['vpc'] and @config['vpc']['vpc_name']
             @vpc = @deploy.findLitterMate(name: @config['vpc']['vpc_name'], type: "vpcs")
           end
@@ -70,124 +45,673 @@ module MU
             raise MuError, "ContainerCluster #{@config['name']} unable to locate its resident VPC from #{@config['vpc']}"
           end
 
-          subnet = nil
-          @vpc.subnets.each { |s|
-            if s.az == @config['region']
-              subnet = s
-              break
+
+          sa = MU::Config::Ref.get(@config['service_account'])
+          if sa.name and @deploy.findLitterMate(name: sa.name, type: "users")
+            @service_acct = @deploy.findLitterMate(name: sa.name, type: "users").cloud_desc
+          else
+            if !sa or !sa.kitten or !sa.kitten.cloud_desc
+              raise MuError, "Failed to get service account cloud id from #{@config['service_account'].to_s}"
             end
-          }
-puts @config['credentials']
-          service_acct = MU::Cloud::Google::Server.createServiceAccount(
-            @mu_name.downcase,
-            @deploy,
-            project: @config['project'],
-            credentials: @config['credentials']
-          )
-          MU::Cloud::Google.grantDeploySecretAccess(service_acct.email, credentials: @config['credentials'])
+            @service_acct = sa.kitten.cloud_desc
+          end
+          if !@config['scrub_mu_isms']
+            MU::Cloud::Google.grantDeploySecretAccess(@service_acct.email, credentials: @config['credentials'])
+          end
 
-          @config['ssh_user'] ||= "mu"
+          @config['ssh_user'] ||= "muadmin"
 
-          node_desc = {
-            :machine_type => @config['instance_type'],
-            :preemptible => @config['preemptible'],
-            :disk_size_gb => @config['disk_size_gb'],
-            :labels => labels,
-            :tags => [@mu_name.downcase],
-            :service_account => service_acct.email,
-            :oauth_scopes => ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/devstorage.read_only"],
-            :metadata => {
-              "ssh-keys" => @config['ssh_user']+":"+@deploy.ssh_public_key
-            }
-          }
-          [:local_ssd_count, :min_cpu_platform, :image_type].each { |field|
-            if @config[field.to_s]
-              node_desc[field] = @config[field.to_s]
-            end
-          }
 
-          nodeobj = MU::Cloud::Google.container(:NodeConfig).new(node_desc)
+          nodeobj = if @config['min_size'] and @config['max_size']
+            MU::Cloud::Google.container(:NodePool).new(
+              name: @mu_name.downcase,
+              initial_node_count: @config['instance_count'] || @config['min_size'],
+              autoscaling: MU::Cloud::Google.container(:NodePoolAutoscaling).new(
+                enabled: true,
+                min_node_count: @config['min_size'],
+                max_node_count: @config['max_size'],
+              ),
+              management: MU::Cloud::Google.container(:NodeManagement).new(
+                auto_upgrade: @config['auto_upgrade'],
+                auto_repair: @config['auto_repair']
+              ),
+              config: MU::Cloud::Google.container(:NodeConfig).new(node_desc)
+            )
+          else
+            MU::Cloud::Google.container(:NodeConfig).new(node_desc)
+          end
+          locations = if @config['availability_zone']
+            [@config['availability_zone']]
+          else
+            MU::Cloud::Google.listAZs(@config['region'])
+          end
+
+          master_user = @config['master_user']
+          # We'll create a temporary basic auth config so that we can grant
+          # useful permissions to the Client Certificate user
+          master_user ||= "master_user"
+          master_pw = Password.pronounceable(18)
 
           desc = {
             :name => @mu_name.downcase,
             :description => @deploy.deploy_id,
             :network => @vpc.cloud_id,
-            :subnetwork => subnet.cloud_id,
-            :labels => labels,
+            :enable_tpu => @config['tpu'],
             :resource_labels => labels,
-            :initial_cluster_version => @config['kubernetes']['version'],
-            :initial_node_count => @config['instance_count'],
-            :locations => MU::Cloud::Google.listAZs(@config['region']),
-            :node_config => nodeobj
+            :locations => locations,
+            :master_auth => MU::Cloud::Google.container(:MasterAuth).new(
+              :client_certificate_config => MU::Cloud::Google.container(:ClientCertificateConfig).new(
+                :issue_client_certificate => true
+              ),
+              :username => master_user,
+              :password => master_pw
+            ),
           }
 
+          if @config['kubernetes']
+            desc[:addons_config] = MU::Cloud::Google.container(:AddonsConfig).new(
+              horizontal_pod_autoscaling: MU::Cloud::Google.container(:HorizontalPodAutoscaling).new(
+                disabled: !@config['kubernetes']['horizontal_pod_autoscaling']
+              ),
+              http_load_balancing: MU::Cloud::Google.container(:HttpLoadBalancing).new(
+                disabled: !@config['kubernetes']['http_load_balancing']
+              ),
+              kubernetes_dashboard: MU::Cloud::Google.container(:KubernetesDashboard).new(
+                disabled: !@config['kubernetes']['dashboard']
+              ),
+              network_policy_config: MU::Cloud::Google.container(:NetworkPolicyConfig).new(
+                disabled: !@config['kubernetes']['network_policy_addon']
+              )
+            )
+          end
+
+          # Pick an existing subnet from our VPC, if we're not going to create
+          # one.
+          if !@config['custom_subnet']
+            @vpc.subnets.each { |s|
+              if s.az == @config['region']
+                desc[:subnetwork] = s.cloud_id
+                break
+              end
+            }
+          end
+          if @config['log_facility'] == "kubernetes"
+            desc[:logging_service] = "logging.googleapis.com/kubernetes"
+            desc[:monitoring_service] = "monitoring.googleapis.com/kubernetes"
+          elsif @config['log_facility'] == "basic"
+            desc[:logging_service] = "logging.googleapis.com"
+            desc[:monitoring_service] = "monitoring.googleapis.com"
+          else
+            desc[:logging_service] = "none"
+            desc[:monitoring_service] = "none"
+          end
+
+          if nodeobj.is_a?(::Google::Apis::ContainerV1::NodeConfig)
+            desc[:node_config] = nodeobj
+            desc[:initial_node_count] = @config['instance_count']
+          else
+            desc[:node_pools] = [nodeobj]
+          end
+
+          if @config['kubernetes']
+            if @config['kubernetes']['version']
+              desc[:initial_cluster_version] = @config['kubernetes']['version']
+            end
+            if @config['kubernetes']['alpha']
+              desc[:enable_kubernetes_alpha] = @config['kubernetes']['alpha']
+            end
+          end
+
+          if @config['preferred_maintenance_window']
+            desc[:maintenance_policy] = MU::Cloud::Google.container(:MaintenancePolicy).new(
+              window: MU::Cloud::Google.container(:MaintenanceWindow).new(
+                daily_maintenance_window: MU::Cloud::Google.container(:DailyMaintenanceWindow).new(
+                  start_time: @config['preferred_maintenance_window']
+                )
+              )
+            )
+          end
+
+          if @config['private_cluster']
+            desc[:private_cluster_config] = MU::Cloud::Google.container(:PrivateClusterConfig).new(
+              enable_private_endpoint: @config['private_cluster']['private_master'],
+              enable_private_nodes: @config['private_cluster']['private_nodes'],
+              master_ipv4_cidr_block: @config['private_cluster']['master_ip_block']
+            )
+            desc[:ip_allocation_policy] = MU::Cloud::Google.container(:IpAllocationPolicy).new(
+              use_ip_aliases: true
+            )
+          end
+
+          if @config['ip_aliases'] or @config['custom_subnet'] or
+             @config['services_ip_block'] or @config['services_ip_block_name'] or
+             @config['pod_ip_block'] or @config['pod_ip_block_name'] or
+             @config['tpu_ip_block']
+            alloc_desc = { :use_ip_aliases => @config['ip_aliases'] }
+
+            if @config['custom_subnet']
+              alloc_desc[:create_subnetwork] = true
+              alloc_desc[:subnetwork_name] = if @config['custom_subnet']['name']
+                @config['custom_subnet']['name']
+              else
+                @mu_name.downcase
+              end
+
+              if @config['custom_subnet']['node_ip_block']
+                alloc_desc[:node_ipv4_cidr_block] = @config['custom_subnet']['node_ip_block']
+              end
+            else
+              if @config['pod_ip_block_name']
+                alloc_desc[:cluster_secondary_range_name] = @config['pod_ip_block_name']
+              end
+              if @config['services_ip_block_name']
+                alloc_desc[:services_secondary_range_name] = @config['services_ip_block_name']
+              end
+            end
+
+            if @config['services_ip_block']
+              alloc_desc[:services_ipv4_cidr_block] = @config['services_ip_block']
+            end
+            if @config['tpu_ip_block']
+              alloc_desc[:tpu_ipv4_cidr_block] = @config['tpu_ip_block']
+            end
+            if @config['pod_ip_block']
+              alloc_desc[:cluster_ipv4_cidr_block] = @config['pod_ip_block']
+            end
+
+            desc[:ip_allocation_policy] = MU::Cloud::Google.container(:IpAllocationPolicy).new(alloc_desc)
+            pp alloc_desc
+          end
+
+          if @config['authorized_networks'] and @config['authorized_networks'].size > 0
+            desc[:master_authorized_networks_config] = MU::Cloud::Google.container(:MasterAuthorizedNetworksConfig).new(
+              enabled: true,
+              cidr_blocks: @config['authorized_networks'].map { |n|
+                MU::Cloud::Google.container(:CidrBlock).new(
+                  cidr_block: n['ip_block'],
+                  display_name: n['label']
+                )
+              }
+            )
+          end
+
+          if @config['kubernetes'] and @config['kubernetes']['max_pods'] and
+             @config['ip_aliases']
+            desc[:default_max_pods_constraint] = MU::Cloud::Google.container(:MaxPodsConstraint).new(
+              max_pods_per_node: @config['kubernetes']['max_pods']
+            )
+          end
+
           requestobj = MU::Cloud::Google.container(:CreateClusterRequest).new(
-            :cluster => MU::Cloud::Google.container(:Cluster).new(desc)
+            :cluster => MU::Cloud::Google.container(:Cluster).new(desc),
           )
 
-          MU.log "Creating GKE cluster #{@mu_name.downcase}", details: desc
-          pp @vpc.subnets.map { |x| x.config['name'] }
-          pp requestobj
-          cluster = MU::Cloud::Google.container(credentials: @config['credentials']).create_cluster(
-            @config['project'],
-            @config['availability_zone'],
+          MU.log "Creating GKE cluster #{@mu_name.downcase}", details: requestobj
+          @config['master_az'] = @config['region']
+          parent_arg = "projects/"+@config['project']+"/locations/"+@config['master_az']
+
+          cluster = MU::Cloud::Google.container(credentials: @config['credentials']).create_project_location_cluster(
+            parent_arg,
             requestobj
           )
+          @cloud_id = parent_arg+"/clusters/"+@mu_name.downcase
 
           resp = nil
           begin
-            resp = MU::Cloud::Google.container(credentials: @config['credentials']).get_zone_cluster(@config["project"], @config['availability_zone'], @mu_name.downcase)
+            resp = MU::Cloud::Google.container(credentials: @config['credentials']).get_project_location_cluster(@cloud_id)
+            if resp.status == "ERROR"
+              MU.log "GKE cluster #{@cloud_id} failed", MU::ERR, details: resp.status_message
+              raise MuError, "GKE cluster #{@cloud_id} failed: #{resp.status_message}"
+            end
             sleep 30 if resp.status != "RUNNING"
           end while resp.nil? or resp.status != "RUNNING"
-#          labelCluster # XXX need newer API release
-          @cloud_id = @mu_name.downcase
 
-# XXX wait until the thing is ready
+          writeKubeConfig
+
+        end
+
+
+        # Called automatically by {MU::Deploy#createResources}
+        def groom
+          labelCluster 
+
+          me = cloud_desc
+
+          parent_arg = "projects/"+@config['project']+"/locations/"+me.location
+
+          # Enable/disable basic auth
+          authcfg = {}
+          action = nil
+          if @config['master_user'] and (me.master_auth.username != @config['master_user'] or !me.master_auth.password)
+            authcfg[:username] = @config['master_user']
+            authcfg[:password] = Password.pronounceable(16..18)
+            MU.log "Enabling basic auth for GKE cluster #{@mu_name.downcase}", MU::NOTICE, details: authcfg
+          elsif !@config['master_user'] and me.master_auth.username
+            authcfg[:username] = ""
+            MU.log "Disabling basic auth for GKE cluster #{@mu_name.downcase}", MU::NOTICE
+          end
+          if authcfg.size > 0
+            MU::Cloud::Google.container(credentials: @config['credentials']).set_project_location_cluster_master_auth(
+              @cloud_id,
+              MU::Cloud::Google.container(:SetMasterAuthRequest).new(
+                name: @cloud_id,
+                action: "SET_USERNAME",
+                update: MU::Cloud::Google.container(:MasterAuth).new(
+                  authcfg
+                )
+              )
+            )
+            me = cloud_desc(use_cache: false)
+          end
+
+          # Now go through all the things that use update_project_location_cluster
+          updates = []
+
+          locations = if @config['availability_zone']
+            [@config['availability_zone']]
+          else
+            MU::Cloud::Google.listAZs(@config['region'])
+          end
+          if me.locations != locations
+            updates << { :desired_locations => locations }
+          end
+
+          if @config['min_size'] and @config['max_size'] and
+             (me.node_pools.first.autoscaling.min_node_count != @config['min_size'] or
+             me.node_pools.first.autoscaling.max_node_count != @config['max_size'])
+            updates << {
+              :desired_node_pool_autoscaling => MU::Cloud::Google.container(:NodePoolAutoscaling).new(
+                enabled: true,
+                max_node_count: @config['max_size'],
+                min_node_count: @config['min_size']
+              )
+            }
+          end
+
+          if @config['authorized_networks'] and @config['authorized_networks'].size > 0
+            desired = @config['authorized_networks'].map { |n|
+              MU::Cloud::Google.container(:CidrBlock).new(
+                cidr_block: n['ip_block'],
+                display_name: n['label']
+              )
+            }
+            if !me.master_authorized_networks_config or
+               !me.master_authorized_networks_config.enabled or
+               !me.master_authorized_networks_config.cidr_blocks or
+               me.master_authorized_networks_config.cidr_blocks.map {|n| n.cidr_block+n.display_name }.sort != desired.map {|n| n.cidr_block+n.display_name }.sort
+              updates << { :desired_master_authorized_networks_config => MU::Cloud::Google.container(:MasterAuthorizedNetworksConfig).new(
+                enabled: true,
+                cidr_blocks: desired
+              )}
+            end
+          elsif me.master_authorized_networks_config and
+                me.master_authorized_networks_config.enabled
+            updates << { :desired_master_authorized_networks_config => MU::Cloud::Google.container(:MasterAuthorizedNetworksConfig).new(
+              enabled: false
+            )}
+          end
+
+          if @config['log_facility'] == "kubernetes" and me.logging_service != "logging.googleapis.com/kubernetes"
+            updates << {
+              :desired_logging_service => "logging.googleapis.com/kubernetes",
+              :desired_monitoring_service => "monitoring.googleapis.com/kubernetes"
+            }
+          elsif @config['log_facility'] == "basic" and me.logging_service != "logging.googleapis.com"
+            updates << {
+              :desired_logging_service => "logging.googleapis.com",
+              :desired_monitoring_service => "monitoring.googleapis.com"
+            }
+          elsif @config['log_facility'] == "none" and me.logging_service != "none"
+            updates << {
+              :desired_logging_service => "none",
+              :desired_monitoring_service => "none"
+            }
+          end
+
+          if @config['kubernetes']
+            if (me.addons_config.horizontal_pod_autoscaling.disabled and @config['kubernetes']['horizontal_pod_autoscaling']) or
+               (!me.addons_config.horizontal_pod_autoscaling and !@config['kubernetes']['horizontal_pod_autoscaling']) or
+               (me.addons_config.http_load_balancing.disabled and @config['kubernetes']['http_load_balancing']) or
+               (!me.addons_config.http_load_balancing and !@config['kubernetes']['http_load_balancing']) or
+               (me.addons_config.kubernetes_dashboard.disabled and @config['kubernetes']['dashboard']) or
+               (!me.addons_config.kubernetes_dashboard and !@config['kubernetes']['dashboard']) or
+               (me.addons_config.network_policy_config.disabled and @config['kubernetes']['network_policy_addon']) or
+               (!me.addons_config.network_policy_config and !@config['kubernetes']['network_policy_addon'])
+              updates << { :desired_addons_config => MU::Cloud::Google.container(:AddonsConfig).new(
+                horizontal_pod_autoscaling: MU::Cloud::Google.container(:HorizontalPodAutoscaling).new(
+                  disabled: !@config['kubernetes']['horizontal_pod_autoscaling']
+                ),
+                http_load_balancing: MU::Cloud::Google.container(:HttpLoadBalancing).new(
+                  disabled: !@config['kubernetes']['http_load_balancing']
+                ),
+                kubernetes_dashboard: MU::Cloud::Google.container(:KubernetesDashboard).new(
+                  disabled: !@config['kubernetes']['dashboard']
+                ),
+                network_policy_config: MU::Cloud::Google.container(:NetworkPolicyConfig).new(
+                  disabled: !@config['kubernetes']['network_policy_addon']
+                )
+              )}
+            end 
+          end
+
+          if @config['kubernetes'] and @config['kubernetes']['version']
+            if MU.version_sort(@config['kubernetes']['version'], me.current_master_version) > 0
+              updates << {  :desired_master_version => @config['kubernetes']['version'] }
+            end
+          end
+
+          if @config['kubernetes'] and @config['kubernetes']['nodeversion']
+            if MU.version_sort(@config['kubernetes']['nodeversion'], me.current_node_version) > 0
+              updates << { :desired_node_version => @config['kubernetes']['nodeversion'] }
+            end
+          end
+
+          if updates.size > 0
+            updates.each { |mapping|
+              requestobj = MU::Cloud::Google.container(:UpdateClusterRequest).new(
+                :name => @cloud_id,
+                :update => MU::Cloud::Google.container(:ClusterUpdate).new(
+                  mapping
+                )
+              )
+              MU.log "Updating GKE Cluster #{@mu_name.downcase}", MU::NOTICE, details: mapping
+              begin
+                MU::Cloud::Google.container(credentials: @config['credentials']).update_project_location_cluster(
+                  @cloud_id,
+                  requestobj
+                )
+              rescue ::Google::Apis::ClientError => e
+                MU.log e.message, MU::WARN
+              end
+            }
+            me = cloud_desc(use_cache: false)
+          end
+
+          if @config['preferred_maintenance_window'] and
+             (!me.maintenance_policy.window or
+              !me.maintenance_policy.window.daily_maintenance_window or
+              me.maintenance_policy.window.daily_maintenance_window.start_time != @config['preferred_maintenance_window'])
+            MU.log "Setting GKE Cluster #{@mu_name.downcase} maintenance time to #{@config['preferred_maintenance_window']}", MU::NOTICE
+            MU::Cloud::Google.container(credentials: @config['credentials']).set_project_location_cluster_maintenance_policy(
+              @cloud_id,
+              MU::Cloud::Google.container(:SetMaintenancePolicyRequest).new(
+                maintenance_policy: MU::Cloud::Google.container(:MaintenancePolicy).new(
+                  window: MU::Cloud::Google.container(:MaintenanceWindow).new(
+                    daily_maintenance_window: MU::Cloud::Google.container(:DailyMaintenanceWindow).new(
+                      start_time: @config['preferred_maintenance_window']
+                    )
+                  )
+                )
+              )
+            )
+          elsif !@config['preferred_maintenance_window'] and me.maintenance_policy.window
+            MU.log "Unsetting GKE Cluster #{@mu_name.downcase} maintenance time to #{@config['preferred_maintenance_window']}", MU::NOTICE
+            MU::Cloud::Google.container(credentials: @config['credentials']).set_project_location_cluster_maintenance_policy(
+              @cloud_id,
+              nil
+            )
+          end
+
+
+          kube_conf = writeKubeConfig
+
+          if @config['kubernetes_resources']
+            MU::Master.applyKubernetesResources(
+              @config['name'], 
+              @config['kubernetes_resources'],
+              kubeconfig: kube_conf,
+              outputdir: @deploy.deploy_dir
+            )
+          end
+
+          MU.log %Q{How to interact with your Kubernetes cluster\nkubectl --kubeconfig "#{kube_conf}" get events --all-namespaces\nkubectl --kubeconfig "#{kube_conf}" get all\nkubectl --kubeconfig "#{kube_conf}" create -f some_k8s_deploy.yml\nkubectl --kubeconfig "#{kube_conf}" get nodes}, MU::SUMMARY
         end
 
+
         # Locate an existing ContainerCluster or ContainerClusters and return an array containing matching GCP resource descriptors for those that match.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region
-        # @param tag_key [String]: A tag key to search.
-        # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
-        # @param flags [Hash]: Optional flags
         # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching ContainerClusters
-        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, flags: {}, credentials: nil)
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
+        def self.find(**args)
+          args[:project] ||= args[:habitat]
+          args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
+          location = args[:region] || args[:availability_zone] || "-"
+
+          found = {}
+
+          if args[:cloud_id]
+            resp = begin
+              MU::Cloud::Google.container(credentials: args[:credentials]).get_project_location_cluster(args[:cloud_id])
+            rescue ::Google::Apis::ClientError => e
+              raise e if !e.message.match(/forbidden:/)
+            end
+            found[args[:cloud_id]] = resp if resp
+          else
+            resp = begin
+              MU::Cloud::Google.container(credentials: args[:credentials]).list_project_location_clusters("projects/#{args[:project]}/locations/#{location}")
+            rescue ::Google::Apis::ClientError => e
+              raise e if !e.message.match(/forbidden:/)
+            end
+            if resp and resp.clusters and !resp.clusters.empty?
+              resp.clusters.each { |c|
+                found[c.self_link.sub(/.*?\/projects\//, 'projects/')] = c
+              }
+            end
+          end
+
+          found
         end
 
-        # Called automatically by {MU::Deploy#createResources}
-        def groom
-          deploydata = describe[2]
-          @config['availability_zone'] ||= deploydata['zone']
-          resp = MU::Cloud::Google.container(credentials: @config['credentials']).get_zone_cluster(@config["project"], @config['availability_zone'], @mu_name.downcase)
-#          pp resp
-
-#          labelCluster # XXX need newer API release
-
-          # desired_*:
-          # addons_config
-          # image_type
-          # locations
-          # master_authorized_networks_config
-          # master_version
-          # monitoring_service
-          # node_pool_autoscaling
-          # node_pool_id
-          # node_version
-#          update = {
-
-#          }
-#          pp update
-#          requestobj = MU::Cloud::Google.container(:UpdateClusterRequest).new(
-#            :cluster => MU::Cloud::Google.container(:ClusterUpdate).new(update)
-#          )
-           # XXX do all the kubernetes stuff like we do in AWS
+        # Reverse-map our cloud description into a runnable config hash.
+        # We assume that any values we have in +@config+ are placeholders, and
+        # calculate our own accordingly based on what's live in the cloud.
+        def toKitten(rootparent: nil, billing: nil, habitats: nil)
+
+          bok = {
+            "cloud" => "Google",
+            "project" => @config['project'],
+            "credentials" => @config['credentials'],
+            "cloud_id" => @cloud_id,
+            "name" => cloud_desc.name.dup
+          }
+
+          bok['region'] = cloud_desc.location.sub(/\-[a-z]$/, "")
+          if cloud_desc.locations.size == 1
+            bok['availability_zone'] = cloud_desc.locations.first
+          end
+          bok["instance_count"] = cloud_desc.current_node_count
+          cloud_desc.network_config.network.match(/^projects\/(.*?)\/.*?\/networks\/([^\/]+)(?:$|\/)/)
+          vpc_proj = Regexp.last_match[1]
+          vpc_id = Regexp.last_match[2]
+
+          bok['vpc'] = MU::Config::Ref.get(
+            id: vpc_id,
+            cloud: "Google",
+            habitat: MU::Config::Ref.get(
+              id: vpc_proj,
+              cloud: "Google",
+              credentials: @credentials,
+              type: "habitats"
+            ),
+            credentials: @config['credentials'],
+            type: "vpcs"
+          )
+
+
+          bok['kubernetes'] = {
+            "version" => cloud_desc.current_master_version,
+            "nodeversion" => cloud_desc.current_node_version
+          }
+          if cloud_desc.default_max_pods_constraint and
+             cloud_desc.default_max_pods_constraint.max_pods_per_node
+            bok['kubernetes']['max_pods'] = cloud_desc.default_max_pods_constraint.max_pods_per_node
+          end
+
+          if cloud_desc.addons_config.horizontal_pod_autoscaling and
+             cloud_desc.addons_config.horizontal_pod_autoscaling.disabled
+            bok['kubernetes']['horizontal_pod_autoscaling'] = false
+          end
+          if cloud_desc.addons_config.http_load_balancing and
+             cloud_desc.addons_config.http_load_balancing.disabled
+            bok['kubernetes']['http_load_balancing'] = false
+          end
+          if !cloud_desc.addons_config.kubernetes_dashboard or
+             !cloud_desc.addons_config.kubernetes_dashboard.disabled
+            bok['kubernetes']['dashboard'] = true
+          end
+          if !cloud_desc.addons_config.network_policy_config or
+             !cloud_desc.addons_config.network_policy_config.disabled
+            bok['kubernetes']['network_policy_addon'] = true
+          end
+
+          if cloud_desc.ip_allocation_policy.use_ip_aliases
+            bok['ip_aliases'] = true
+          end
+          if cloud_desc.ip_allocation_policy.cluster_ipv4_cidr_block
+            bok['pod_ip_block'] = cloud_desc.ip_allocation_policy.cluster_ipv4_cidr_block
+          end
+          if cloud_desc.ip_allocation_policy.services_ipv4_cidr_block
+            bok['services_ip_block'] = cloud_desc.ip_allocation_policy.services_ipv4_cidr_block
+          end
+
+          if cloud_desc.ip_allocation_policy.create_subnetwork
+            bok['custom_subnet'] = {
+              "name" => (cloud_desc.ip_allocation_policy.subnetwork_name || cloud_desc.subnetwork)
+            }
+            if cloud_desc.ip_allocation_policy.node_ipv4_cidr_block
+              bok['custom_subnet']['node_ip_block'] = cloud_desc.ip_allocation_policy.node_ipv4_cidr_block
+            end
+          end
+
+          bok['log_facility'] = if cloud_desc.logging_service == "logging.googleapis.com"
+            "basic"
+          elsif cloud_desc.logging_service == "logging.googleapis.com/kubernetes"
+            "kubernetes"
+          else
+            "none"
+          end
+
+          if cloud_desc.master_auth and cloud_desc.master_auth.username
+            bok['master_user'] = cloud_desc.master_auth.username
+          end
+
+          if cloud_desc.maintenance_policy and
+             cloud_desc.maintenance_policy.window and
+             cloud_desc.maintenance_policy.window.daily_maintenance_window and
+             cloud_desc.maintenance_policy.window.daily_maintenance_window.start_time
+            bok['preferred_maintenance_window'] = cloud_desc.maintenance_policy.window.daily_maintenance_window.start_time
+          end
+
+          if cloud_desc.enable_tpu
+            bok['tpu'] = true
+          end
+          if cloud_desc.enable_kubernetes_alpha
+            bok['kubernetes'] ||= {}
+            bok['kubernetes']['alpha'] = true
+          end
+
+          if cloud_desc.node_pools and cloud_desc.node_pools.size > 0
+            pool = cloud_desc.node_pools.first # we don't really support multiples atm
+            bok["instance_type"] = pool.config.machine_type
+            bok["instance_count"] = pool.initial_node_count
+            bok['scopes'] = pool.config.oauth_scopes
+            if pool.config.metadata
+              bok["metadata"] = pool.config.metadata.keys.map { |k|
+                { "key" => k, "value" => pool.config.metadata[k] }
+              }
+            end
+            if pool.autoscaling and pool.autoscaling.enabled
+              bok['max_size'] = pool.autoscaling.max_node_count
+              bok['min_size'] = pool.autoscaling.min_node_count
+            end
+            bok['auto_repair'] = false
+            bok['auto_upgrade'] = false
+            if pool.management
+              bok['auto_repair'] = true if pool.management.auto_repair
+              bok['auto_upgrade'] = true if pool.management.auto_upgrade
+            end
+            [:local_ssd_count, :min_cpu_platform, :image_type, :disk_size_gb, :preemptible, :service_account].each { |field|
+              if pool.config.respond_to?(field)
+                bok[field.to_s] = pool.config.method(field).call
+                bok.delete(field.to_s) if bok[field.to_s].nil?
+              end
+            }
+          else
+            bok["instance_type"] = cloud_desc.node_config.machine_type
+            bok['scopes'] = cloud_desc.node_config.oauth_scopes
+            if cloud_desc.node_config.metadata
+              bok["metadata"] = cloud_desc.node_config.metadata.keys.map { |k|
+                { "key" => k, "value" => pool.config.metadata[k] }
+              }
+            end
+            [:local_ssd_count, :min_cpu_platform, :image_type, :disk_size_gb, :preemptible, :service_account].each { |field|
+              if cloud_desc.node_config.respond_to?(field)
+                bok[field.to_s] = cloud_desc.node_config.method(field).call
+                bok.delete(field.to_s) if bok[field.to_s].nil?
+              end
+            }
+          end
+
+          if bok['service_account']
+            found = MU::Cloud::Google::User.find(
+              credentials: bok['credentials'],
+              project: bok['project'],
+              cloud_id: bok['service_account']
+            )
+            if found and found.size == 1
+              sa = found.values.first
+              # Ignore generic Mu service accounts
+              if cloud_desc.resource_labels and
+                 cloud_desc.resource_labels["mu-id"] and 
+                 sa.description and
+                 cloud_desc.resource_labels["mu-id"].downcase == sa.description.downcase
+                bok.delete("service_account")
+              else
+                bok['service_account'] = MU::Config::Ref.get(
+                  id: found.values.first.name,
+                  cloud: "Google",
+                  credentials: @config['credentials'],
+                  type: "users"
+                )
+              end
+            else
+              bok.delete("service_account")
+            end
+          end
+
+          if cloud_desc.private_cluster_config
+            if cloud_desc.private_cluster_config.enable_private_nodes?
+              bok["private_cluster"] ||= {}
+              bok["private_cluster"]["private_nodes"] = true
+            end
+            if cloud_desc.private_cluster_config.enable_private_endpoint?
+              bok["private_cluster"] ||= {}
+              bok["private_cluster"]["private_master"] = true
+            end
+            if cloud_desc.private_cluster_config.master_ipv4_cidr_block
+              bok["private_cluster"] ||= {}
+              bok["private_cluster"]["master_ip_block"] = cloud_desc.private_cluster_config.master_ipv4_cidr_block
+            end
+          end
+
+          if cloud_desc.master_authorized_networks_config and
+             cloud_desc.master_authorized_networks_config.cidr_blocks and
+             cloud_desc.master_authorized_networks_config.cidr_blocks.size > 0
+            bok['authorized_networks'] = []
+            cloud_desc.master_authorized_networks_config.cidr_blocks.each { |c|
+              bok['authorized_networks'] << {
+                "ip_block" => c.cidr_block,
+                "label" => c.display_name
+              }
+            }
+          end
+
+          bok
         end
 
+
         # Register a description of this cluster instance with this deployment's metadata.
         def notify
-          desc = MU.structToHash(MU::Cloud::Google.container(credentials: @config['credentials']).get_zone_cluster(@config["project"], @config['availability_zone'], @mu_name.downcase))
+          resp = MU::Cloud::Google.container(credentials: @config['credentials']).get_project_location_cluster(@cloud_id)
+          desc = MU.structToHash(resp)
           desc["project"] = @config['project']
           desc["cloud_id"] = @cloud_id
           desc["project_id"] = @project_id
@@ -205,7 +729,7 @@ puts @config['credentials']
         # Denote whether this resource implementation is experiment, ready for
         # testing, or ready for production use.
         def self.quality
-          MU::Cloud::ALPHA
+          MU::Cloud::RELEASE
         end
 
         # Called by {MU::Cleanup}. Locates resources that were created by the
@@ -218,35 +742,51 @@ puts @config['credentials']
           skipsnapshots = flags["skipsnapshots"]
 
           flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
+          return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
+          clusters = []
+
+          # Make sure we catch regional *and* zone clusters
+          found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['project']}/locations/#{region}")
+          clusters.concat(found.clusters) if found and found.clusters
           MU::Cloud::Google.listAZs(region).each { |az|
-            found = MU::Cloud::Google.container(credentials: credentials).list_zone_clusters(flags["project"], az)
-            if found and found.clusters
-              found.clusters.each { |cluster|
+            found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['project']}/locations/#{az}")
+            clusters.concat(found.clusters) if found and found.clusters
+          }
 
-                if !cluster.name.match(/^#{Regexp.quote(MU.deploy_id)}\-/i) and
-                   cluster.resource_labels['mu-id'] != MU.deploy_id.downcase
-                  next
-                end
-                MU.log "Deleting GKE cluster #{cluster.name}"
-                if !noop
-                  MU::Cloud::Google.container(credentials: credentials).delete_zone_cluster(flags["project"], az, cluster.name)
-                  begin
-                    MU::Cloud::Google.container(credentials: credentials).get_zone_cluster(flags["project"], az, cluster.name)
-                    sleep 60
-                  rescue ::Google::Apis::ClientError => e
-                    if e.message.match(/is currently creating cluster/)
-                      sleep 60
-                      retry
-                    elsif !e.message.match(/notFound:/)
-                      raise e
-                    else
-                      break
-                    end
-                  end while true
+          clusters.uniq.each { |cluster|
+            if !cluster.resource_labels or (
+                 !cluster.name.match(/^#{Regexp.quote(MU.deploy_id)}\-/i) and
+                 cluster.resource_labels['mu-id'] != MU.deploy_id.downcase
+               )
+              next
+            end
+            MU.log "Deleting GKE cluster #{cluster.name}"
+            if !noop
+              cloud_id = cluster.self_link.sub(/.*?\/projects\//, 'projects/')
+              retries = 0
+              begin
+                MU::Cloud::Google.container(credentials: credentials).delete_project_location_cluster(cloud_id)
+                MU::Cloud::Google.container(credentials: credentials).get_project_location_cluster(cloud_id)
+                sleep 60
+              rescue ::Google::Apis::ClientError => e
+                if e.message.match(/notFound: /)
+                  MU.log cloud_id, MU::WARN, details: e.inspect
+                  break
+                elsif e.message.match(/failedPrecondition: /)
+                  if (retries % 5) == 0
+                    MU.log "Waiting to delete GKE cluster #{cluster.name}: #{e.message}", MU::NOTICE
+                  end
+                  sleep 60
+                  retries += 1
+                  retry
+                else
+                  MU.log cloud_id, MU::WARN, details: e.inspect
+                  raise e
                 end
-              }
+              end while true
             end
           }
+
         end
 
         # Cloud-specific configuration properties.
@@ -254,16 +794,104 @@ puts @config['credentials']
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
         def self.schema(config)
           toplevel_required = []
+          gke_defaults = defaults
           schema = {
+            "auto_upgrade" => {
+              "type" => "boolean",
+              "description" => "Automatically upgrade worker nodes during maintenance windows",
+              "default" => true
+            },
+            "auto_repair" => {
+              "type" => "boolean",
+              "description" => "Automatically replace worker nodes which fail health checks",
+              "default" => true
+            },
             "local_ssd_count" => {
               "type" => "integer",
               "description" => "The number of local SSD disks to be attached to workers. See https://cloud.google.com/compute/docs/disks/local-ssd#local_ssd_limits"
             },
+            "ssh_user" => MU::Cloud::Google::Server.schema(config)[1]["ssh_user"],
+            "metadata" => MU::Cloud::Google::Server.schema(config)[1]["metadata"],
+            "service_account" => MU::Cloud::Google::Server.schema(config)[1]["service_account"],
+            "scopes" => MU::Cloud::Google::Server.schema(config)[1]["scopes"],
+            "private_cluster" => {
+              "description" => "Set a GKE cluster to be private, that is segregated into its own hidden VPC.",
+              "type" => "object",
+              "properties" => {
+                "private_nodes" => {
+                  "type" => "boolean",
+                  "default" => true,
+                  "description" => "Whether GKE worker nodes have internal IP addresses only."
+                },
+                "private_master" => {
+                  "type" => "boolean",
+                  "default" => false,
+                  "description" => "Whether the GKE Kubernetes master's internal IP address is used as the cluster endpoint."
+                },
+                "master_ip_block" => {
+                  "type" => "string",
+                  "pattern" => MU::Config::CIDR_PATTERN,
+                  "default" => "172.20.0.0/28",
+                  "description" => "The private IP address range to use for the GKE master's network"
+                }
+              }
+            },
+            "custom_subnet" => {
+              "type" => "object",
+              "description" => "If set, GKE will create a new subnetwork specifically for this cluster",
+              "properties" => {
+                "name" => {
+                  "type" => "string",
+                  "description" => "Set a custom name for the generated subnet"
+                },
+                "node_ip_block" => {
+                  "type" => "string",
+                  "pattern" => MU::Config::CIDR_PATTERN,
+                  "description" => "The IP address range of the worker nodes in this cluster, in CIDR notation"
+                }
+              }
+            },
+            "pod_ip_block" => {
+              "type" => "string",
+              "pattern" => MU::Config::CIDR_PATTERN,
+              "description" => "The IP address range of the container pods in this cluster, in CIDR notation"
+            },
+            "pod_ip_block_name" => {
+              "type" => "string",
+              "description" => "The name of the secondary range to be used for the pod CIDR block"
+            },
+            "services_ip_block" => {
+              "type" => "string",
+              "pattern" => MU::Config::CIDR_PATTERN,
+              "description" => "The IP address range of the services in this cluster, in CIDR notation"
+            },
+            "services_ip_block_name" => {
+              "type" => "string",
+              "description" => "The name of the secondary range to be used for the services CIDR block"
+            },
+            "ip_aliases" => {
+              "type" => "boolean",
+              "description" => "Whether alias IPs will be used for pod IPs in the cluster. Will be automatically enabled for functionality, such as +private_cluster+, which requires it."
+            },
+            "tpu_ip_block" => {
+              "type" => "string",
+              "pattern" => MU::Config::CIDR_PATTERN,
+              "description" => "The IP address range of any Cloud TPUs in this cluster, in CIDR notation"
+            },
             "disk_size_gb" => {
               "type" => "integer",
               "description" => "Size of the disk attached to each worker, specified in GB. The smallest allowed disk size is 10GB",
               "default" => 100
             },
+            "min_size" => {
+              "description" => "In GKE, this is the minimum number of nodes *per availability zone*, when scaling is enabled. Setting +min_size+ and +max_size+ enables scaling."
+            },
+            "max_size" => {
+              "description" => "In GKE, this is the maximum number of nodes *per availability zone*, when scaling is enabled. Setting +min_size+ and +max_size+ enables scaling."
+            },
+            "instance_count" => {
+              "description" => "In GKE, this value is ignored if +min_size+ and +max_size+ are set."
+            },
             "min_cpu_platform" => {
               "type" => "string",
               "description" => "Minimum CPU platform to be used by workers. The instances may be scheduled on the specified or newer CPU platform. Applicable values are the friendly names of CPU platforms, such as minCpuPlatform: 'Intel Haswell' or minCpuPlatform: 'Intel Sandy Bridge'."
@@ -275,7 +903,95 @@ puts @config['credentials']
             },
             "image_type" => {
               "type" => "string",
-              "description" => "The image type to use for workers. Note that for a given image type, the latest version of it will be used."
+              "enum" => gke_defaults ? gke_defaults.valid_image_types : ["COS"],
+              "description" => "The image type to use for workers. Note that for a given image type, the latest version of it will be used.",
+              "default" => gke_defaults ? gke_defaults.default_image_type : "COS"
+            },
+            "availability_zone" => {
+              "type" => "string",
+              "description" => "Target a specific availability zone for this cluster"
+            },
+            "preferred_maintenance_window" => {
+              "type" => "string",
+              "description" => "The preferred daily time to perform node maintenance. Time format should be in [RFC3339](http://www.ietf.org/rfc/rfc3339.txt) format +HH:MM+ GMT.",
+              "pattern" => '^\d\d:\d\d$'
+            },
+            "kubernetes" => {
+              "description" => "Kubernetes-specific options",
+              "properties" => {
+                "version" => {
+                  "type" => "string"
+                },
+                "nodeversion" => {
+                  "type" => "string",
+                  "description" => "The version of Kubernetes to install on GKE worker nodes."
+                },
+                "alpha" => {
+                  "type" => "boolean",
+                  "default" => false,
+                  "description" => "Enable alpha-quality Kubernetes features on this cluster"
+                },
+                "dashboard" => {
+                  "type" => "boolean",
+                  "default" => false,
+                  "description" => "Enable the Kubernetes Dashboard"
+                },
+                "horizontal_pod_autoscaling" => {
+                  "type" => "boolean",
+                  "default" => true,
+                  "description" => "Increases or decreases the number of replica pods a replication controller has based on the resource usage of the existing pods."
+                },
+                "http_load_balancing" => {
+                  "type" => "boolean",
+                  "default" => true,
+                  "description" => "HTTP (L7) load balancing controller addon, which makes it easy to set up HTTP load balancers for services in a cluster."
+                },
+                "network_policy_addon" => {
+                  "type" => "boolean",
+                  "default" => false,
+                  "description" => "Enable the Network Policy addon"
+                }
+              }
+            },
+            "pod_ip_range" => {
+              "type" => "string",
+              "pattern" => MU::Config::CIDR_PATTERN,
+              "description" => "The IP address range of the container pods in this cluster, in CIDR notation"
+            },
+            "tpu" => {
+              "type" => "boolean",
+              "default" => false,
+              "description" => "Enable the ability to use Cloud TPUs in this cluster."
+            },
+            "log_facility" => {
+              "type" => "string",
+              "default" => "kubernetes",
+              "description" => "The +logging.googleapis.com+ and +monitoring.googleapis.com+ facilities that this cluster should use to write logs and metrics.",
+              "enum" => ["basic", "kubernetes", "none"]
+            },
+            "master_user" => {
+              "type" => "string",
+              "description" => "Enables Basic Auth for a GKE cluster with string as the master username"
+            },
+            "authorized_networks" => {
+              "type" => "array",
+              "items" => {
+                "description" => "GKE's Master authorized networks functionality",
+                "type" => "object",
+                "ip_block" => {
+                  "type" => "string",
+                  "description" => "CIDR block to allow",
+                  "pattern" => MU::Config::CIDR_PATTERN,
+                },
+                "label" =>{
+                  "description" => "Label for this CIDR block",
+                  "type" => "string",
+                }
+              }
+            },
+            "master_az" => {
+              "type" => "string",
+              "description" => "Target a specific Availability Zone for the GKE master. If not set, we will choose one which has the most current versions of Kubernetes available."
             }
           }
           [toplevel_required, schema]
@@ -287,11 +1003,148 @@ puts @config['credentials']
         # @return [Boolean]: True if validation succeeded, False otherwise
         def self.validateConfig(cluster, configurator)
           ok = true
-# XXX validate k8s versions (master and node)
-# XXX validate image types
-# MU::Cloud::Google.container.get_project_zone_serverconfig(@config["project"], @config['availability_zone'])
+          cluster['project'] ||= MU::Cloud::Google.defaultProject(cluster['credentials'])
+
+          cluster['master_az'] ||= cluster['availability_zone'] if cluster['availability_zone']
+
+          if cluster['private_cluster'] or cluster['custom_subnet'] or
+             cluster['services_ip_block'] or cluster['services_ip_block_name'] or
+             cluster['pod_ip_block'] or cluster['pod_ip_block_name'] or
+             cluster['tpu_ip_block']
+            cluster['ip_aliases'] = true
+          end
+
+          if cluster['service_account']
+            cluster['service_account']['cloud'] = "Google"
+            cluster['service_account']['habitat'] ||= MU::Config::Ref.get(
+              id: cluster['project'],
+              cloud: "Google",
+              credentials: cluster['credentials'],
+              type: "habitats"
+            )
+            if cluster['service_account']['name'] and
+               !cluster['service_account']['id']
+              cluster['dependencies'] ||= []
+              cluster['dependencies'] << {
+                "type" => "user",
+                "name" => cluster['service_account']['name']
+              }
+            end
+            found = MU::Config::Ref.get(cluster['service_account'])
+            # XXX verify that found.kitten fails when it's supposed to
+            if cluster['service_account']['id'] and !found.kitten
+              MU.log "GKE cluster #{cluster['name']} failed to locate service account #{cluster['service_account']} in project #{cluster['project']}", MU::ERR
+              ok = false
+            end
+          else
+            user = {
+              "name" => cluster['name'],
+              "cloud" => "Google",
+              "project" => cluster["project"],
+              "credentials" => cluster["credentials"],
+              "type" => "service"
+            }
+            configurator.insertKitten(user, "users", true)
+            cluster['dependencies'] ||= []
+            cluster['service_account'] = MU::Config::Ref.get(
+              type: "users",
+              cloud: "Google",
+              name: cluster["name"],
+              project: cluster["project"],
+              credentials: cluster["credentials"]
+            )
+            cluster['dependencies'] << {
+              "type" => "user",
+              "name" => cluster["name"]
+            }
+          end
+
+          if (cluster['pod_ip_block_name'] or cluster['services_ip_block_name']) and
+             cluster['custom_subnet']
+            MU.log "GKE cluster #{cluster['name']} cannot specify pod_ip_block_name or services_ip_block_name when using a custom subnet", MU::ERR
+            ok = false
+          end
+
+          # If we've enabled master authorized networks, make sure our Mu
+          # Master is one of the things allowed in.
+          if cluster['authorized_networks']
+            found_me = false
+            my_cidr = NetAddr::IPv4.parse(MU.mu_public_ip)
+            cluster['authorized_networks'].each { |block|
+              cidr_obj = NetAddr::IPv4Net.parse(block['ip_block'])
+              if cidr_obj.contains(my_cidr)
+                found_me = true
+                break
+              end
+            }
+            if !found_me
+              cluster['authorized_networks'] << {
+                "ip_block" => MU.mu_public_ip+"/32",
+                "label" => "Mu Master #{$MU_CFG['hostname']}"
+              }
+            end
+          end
+
+          master_versions = defaults(az: cluster['master_az']).valid_master_versions.sort { |a, b| MU.version_sort(a, b) }
+          if cluster['kubernetes'] and cluster['kubernetes']['version']
+            if cluster['kubernetes']['version'] == "latest"
+              cluster['kubernetes']['version'] = master_versions.last
+            elsif !master_versions.include?(cluster['kubernetes']['version'])
+              match = false
+              master_versions.each { |v|
+                if v.match(/^#{Regexp.quote(cluster['kubernetes']['version'])}/)
+                  match = true
+                  break
+                end
+              }
+              if !match
+                MU.log "No version matching #{cluster['kubernetes']['version']} available, will try floating minor revision", MU::WARN
+                cluster['kubernetes']['version'].sub!(/^(\d+\.\d+\.).*/i, '\1')
+                master_versions.each { |v|
+                  if v.match(/^#{Regexp.quote(cluster['kubernetes']['version'])}/)
+                    match = true
+                    break
+                  end
+                }
+                if !match
+                  MU.log "Failed to find a GKE master version matching #{cluster['kubernetes']['version']} among available versions in #{cluster['master_az'] || cluster['region']}.", MU::ERR, details: master_versions
+                  ok = false
+                end
+              end
+            end
+          end
+
+          node_versions = defaults(az: cluster['master_az']).valid_node_versions.sort { |a, b| MU.version_sort(a, b) }
+
+          if cluster['kubernetes'] and cluster['kubernetes']['nodeversion']
+            if cluster['kubernetes']['nodeversion'] == "latest"
+              cluster['kubernetes']['nodeversion'] = node_versions.last
+            elsif !node_versions.include?(cluster['kubernetes']['nodeversion'])
+              match = false
+              node_versions.each { |v|
+                if v.match(/^#{Regexp.quote(cluster['kubernetes']['nodeversion'])}/)
+                  match = true
+                  break
+                end
+              }
+              if !match
+                MU.log "No version matching #{cluster['kubernetes']['nodeversion']} available, will try floating minor revision", MU::WARN
+                cluster['kubernetes']['nodeversion'].sub!(/^(\d+\.\d+\.).*/i, '\1')
+                node_versions.each { |v|
+                  if v.match(/^#{Regexp.quote(cluster['kubernetes']['nodeversion'])}/)
+                    match = true
+                    break
+                  end
+                }
+                if !match
+                  MU.log "Failed to find a GKE node version matching #{cluster['kubernetes']['nodeversion']} among available versions in #{cluster['master_az'] || cluster['region']}.", MU::ERR, details: node_versions
+                  ok = false
+                end
+              end
+            end
+          end
 
-          cluster['instance_type'] = MU::Cloud::Google::Server.validateInstanceType(cluster["instance_type"], cluster["region"])
+          cluster['instance_type'] = MU::Cloud::Google::Server.validateInstanceType(cluster["instance_type"], cluster["region"], project: cluster['project'], credentials: cluster['credentials'])
           ok = false if cluster['instance_type'].nil?
 
           ok
@@ -299,19 +1152,121 @@ puts @config['credentials']
 
         private
 
-        def labelCluster
-          labels = {}
-          MU::MommaCat.listStandardTags.each_pair { |name, value|
-            if !value.nil?
-              labels[name.downcase] = value.downcase.gsub(/[^a-z0-9\-\_]/i, "_")
+        def node_desc
+          labels = Hash[@tags.keys.map { |k|
+            [k.downcase, @tags[k].downcase.gsub(/[^-_a-z0-9]/, '-')] }
+          ]
+          labels["name"] = MU::Cloud::Google.nameStr(@mu_name)
+          desc = {
+            :machine_type => @config['instance_type'],
+            :preemptible => @config['preemptible'],
+            :disk_size_gb => @config['disk_size_gb'],
+            :labels => labels,
+            :tags => [@mu_name.downcase],
+            :service_account => @service_acct.email,
+            :oauth_scopes => @config['scopes']
+          }
+          desc[:metadata] = {}
+          deploykey = @config['ssh_user']+":"+@deploy.ssh_public_key
+          if @config['metadata']
+            desc[:metadata] = Hash[@config['metadata'].map { |m|
+              [m["key"], m["value"]]
+            }]
+          end
+          if desc[:metadata]["ssh-keys"]
+            desc[:metadata]["ssh-keys"] += "\n"+deploykey
+          else
+            desc[:metadata]["ssh-keys"] = deploykey
+          end
+          [:local_ssd_count, :min_cpu_platform, :image_type].each { |field|
+            if @config[field.to_s]
+              desc[field] = @config[field.to_s]
             end
           }
+          desc
+        end
+
+        def labelCluster
+          labels = Hash[@tags.keys.map { |k|
+            [k.downcase, @tags[k].downcase.gsub(/[^-_a-z0-9]/, '-')] }
+          ]
           labels["name"] = MU::Cloud::Google.nameStr(@mu_name)
 
           labelset = MU::Cloud::Google.container(:SetLabelsRequest).new(
             resource_labels: labels
           )
-          MU::Cloud::Google.container(credentials: @config['credentials']).resource_project_zone_cluster_labels(@config["project"], @config['availability_zone'], @mu_name.downcase, labelset)
+          MU::Cloud::Google.container(credentials: @config['credentials']).set_project_location_cluster_resource_labels(@cloud_id, labelset)
+        end
+
+        @@server_config = {}
+        def self.defaults(credentials = nil, az: nil)
+          az ||= MU::Cloud::Google.listAZs.sample
+          return nil if az.nil?
+          @@server_config[credentials] ||= {}
+          if @@server_config[credentials][az]
+            return @@server_config[credentials][az]
+          end
+
+          parent_arg = "projects/"+MU::Cloud::Google.defaultProject(credentials)+"/locations/"+az
+
+          @@server_config[credentials][az] = MU::Cloud::Google.container(credentials: credentials).get_project_location_server_config(parent_arg)
+          @@server_config[credentials][az]
+        end
+
+        def writeKubeConfig
+          kube_conf = @deploy.deploy_dir+"/kubeconfig-#{@config['name']}"
+          client_binding = @deploy.deploy_dir+"/k8s-client-user-admin-binding.yaml"
+          @endpoint = "https://"+cloud_desc.endpoint
+          @cacert = cloud_desc.master_auth.cluster_ca_certificate
+          @cluster = cloud_desc.name
+          @clientcert = cloud_desc.master_auth.client_certificate
+          @clientkey = cloud_desc.master_auth.client_key
+          if cloud_desc.master_auth.username
+            @username = cloud_desc.master_auth.username
+          end
+          if cloud_desc.master_auth.password
+            @password = cloud_desc.master_auth.password
+          end
+
+          kube = ERB.new(File.read(MU.myRoot+"/cookbooks/mu-tools/templates/default/kubeconfig-gke.erb"))
+          File.open(kube_conf, "w"){ |k|
+            k.puts kube.result(binding)
+          }
+
+          # Take this opportunity to ensure that the 'client' service account
+          # used by certificate authentication exists and has appropriate
+          # privilege
+          if @username and @password
+            File.open(client_binding, "w"){ |k|
+              k.puts <<-EOF
+kind: ClusterRoleBinding 
+apiVersion: rbac.authorization.k8s.io/v1
+metadata: 
+  name: client-binding
+  namespace: kube-system
+roleRef: 
+  kind: ClusterRole 
+  name: cluster-admin
+  apiGroup: rbac.authorization.k8s.io
+subjects: 
+- kind: User
+  name: client
+  namespace: kube-system
+              EOF
+            }
+            bind_cmd = %Q{#{MU::Master.kubectl} create serviceaccount client --namespace=kube-system --kubeconfig "#{kube_conf}" ; #{MU::Master.kubectl} --kubeconfig "#{kube_conf}" apply -f #{client_binding}}
+            MU.log bind_cmd
+            system(bind_cmd)
+          end
+          # unset the variables we set just for ERB
+          [:@endpoint, :@cacert, :@cluster, :@clientcert, :@clientkey, :@username, :@password].each { |var|
+            begin
+              remove_instance_variable(var)
+            rescue NameError
+            end
+          }
+
+          kube_conf
         end
 
       end #class