cloud-mu 2.1.0beta → 3.0.0beta

data/modules/mu/clouds/google/server.rb

@@ -28,27 +28,19 @@ module MU
       # Google Cloud, this amounts to a single Instance in an Unmanaged
       # Instance Group.
       class Server < MU::Cloud::Server
-        @project_id = nil
-
-        attr_reader :mu_name
-        attr_reader :config
-        attr_reader :deploy
-        attr_reader :cloud_id
-        attr_reader :cloud_desc
-        attr_reader :groomer
-        attr_accessor :mu_windows_name
-
-        # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
-        # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::servers}
-        def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
-          @deploy = mommacat
-          @config = MU::Config.manxify(kitten_cfg)
-          @cloud_id = cloud_id
-
-          if @deploy
-            @userdata = MU::Cloud.fetchUserdata(
+
+        # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like <tt>@vpc</tt>, for us.
+        # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
+        def initialize(**args)
+          super
+
+          @userdata = if @config['userdata_script']
+            @config['userdata_script']
+          elsif @deploy and !@config['scrub_mu_isms']
+            MU::Cloud.fetchUserdata(
               platform: @config["platform"],
-              cloud: "google",
+              cloud: "Google",
+              credentials: @config['credentials'],
               template_variables: {
                 "deployKey" => Base64.urlsafe_encode64(@deploy.public_key),
                 "deploySSHKey" => @deploy.ssh_public_key,
@@ -57,6 +49,9 @@ module MU
                 "publicIP" => MU.mu_public_ip,
                 "skipApplyUpdates" => @config['skipinitialupdates'],
                 "windowsAdminName" => @config['windows_admin_username'],
+                "adminBucketName" => MU::Cloud::Google.adminBucketName(@credentials),
+                "chefVersion" => MU.chefVersion,
+                "mommaCatPort" => MU.mommaCatPort,
                 "resourceName" => @config["name"],
                 "resourceType" => "server",
                 "platform" => @config["platform"]
@@ -64,17 +59,11 @@ module MU
               custom_append: @config['userdata_script']
             )
           end
-
-          if !mu_name.nil?
-            @mu_name = mu_name
-            @config['mu_name'] = @mu_name
+# XXX writing things into @config at runtime is a bad habit and we should stop
+          if !@mu_name.nil?
+            @config['mu_name'] = @mu_name # XXX whyyyy
             # describe
             @mu_windows_name = @deploydata['mu_windows_name'] if @mu_windows_name.nil? and @deploydata
-            @config['project'] ||= MU::Cloud::Google.defaultProject(@config['credentials'])
-            if !@project_id
-              project = MU::Cloud::Google.projectLookup(@config['project'], @deploy, sibling_only: true, raise_on_fail: false)
-              @project_id = project.nil? ? @config['project'] : project.cloudobj.cloud_id
-            end
           else
             if kitten_cfg.has_key?("basis")
               @mu_name = @deploy.getResourceName(@config['name'], need_unique_string: true)
@@ -83,60 +72,88 @@ module MU
             end
             @config['mu_name'] = @mu_name
 
-            @config['instance_secret'] = Password.random(50)
           end
-          @config['ssh_user'] ||= "mu"
-          @groomer = MU::Groomer.new(self)
+          @config['instance_secret'] ||= Password.random(50)
+          @config['ssh_user'] ||= "muadmin"
 
         end
 
-        # Generate a server-class specific service account, used to grant 
-        # permission to do various API things to a node.
-        # @param rolename [String]:
-        # @param project [String]:
-        # @param scopes [Array<String>]: https://developers.google.com/identity/protocols/googlescopes
-        # XXX this should be a MU::Cloud::Google::User resource
-        def self.createServiceAccount(rolename, deploy, project: nil, scopes: ["https://www.googleapis.com/auth/compute.readonly", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/cloud-platform"], credentials: nil)
-          project ||= MU::Cloud::Google.defaultProject(credentials)
-
-#https://www.googleapis.com/auth/devstorage.read_only ?
-          name = deploy.getResourceName(rolename, max_length: 30).downcase
-
-          saobj = MU::Cloud::Google.iam(:CreateServiceAccountRequest).new(
-            account_id: name.gsub(/[^a-z]/, ""), # XXX this mangling isn't required in the console, so why is it here?
-            service_account: MU::Cloud::Google.iam(:ServiceAccount).new(
-              display_name: rolename,
-# do NOT specify project_id or name, we know that much
-            )
-          )
-
-          resp = MU::Cloud::Google.iam(credentials: credentials).create_service_account(
-            "projects/#{project}",
-            saobj
-          )
+        # Return the date/time a machine image was created.
+        # @param image_id [String]: URL to a Google disk image
+        # @param credentials [String]
+        # @return [DateTime]
+        def self.imageTimeStamp(image_id, credentials: nil)
+          begin
+            img = fetchImage(image_id, credentials: credentials)
+            return DateTime.new if img.nil?
+            return DateTime.parse(img.creation_timestamp)
+          rescue ::Google::Apis::ClientError => e
+          end
 
-          MU::Cloud::Google.compute(:ServiceAccount).new(
-            email: resp.email,
-            scopes: scopes
-          )
+          return DateTime.new
         end
 
+        @@image_id_map = {}
+
         # Retrieve the cloud descriptor for this machine image, which can be
         # a whole or partial URL. Will follow deprecation notices and retrieve
         # the latest version, if applicable.
         # @param image_id [String]: URL to a Google disk image
-        # @return [Google::Apis::ComputeBeta::Image]
+        # @param credentials [String]
+        # @return [Google::Apis::ComputeV1::Image]
         def self.fetchImage(image_id, credentials: nil)
+          return @@image_id_map[image_id] if @@image_id_map[image_id]
+
           img_proj = img_name = nil
-          begin
-            img_proj = image_id.gsub(/.*?\/?projects\/([^\/]+)\/.*/, '\1')
+          if image_id.match(/\//)
+            img_proj = image_id.gsub(/(?:https?:\/\/.*?\.googleapis\.com\/compute\/.*?\/)?.*?\/?(?:projects\/)?([^\/]+)\/.*/, '\1')
             img_name = image_id.gsub(/.*?([^\/]+)$/, '\1')
+          else
+            img_name = image_id
+          end
+
+          begin
+            @@image_id_map[image_id] = MU::Cloud::Google.compute(credentials: credentials).get_image_from_family(img_proj, img_name)
+            return @@image_id_map[image_id]
+          rescue ::Google::Apis::ClientError
+            # This is fine- we don't know that what we asked for is really an
+            # image family name, instead of just an image.
+          end
+
+          begin
             img = MU::Cloud::Google.compute(credentials: credentials).get_image(img_proj, img_name)
             if !img.deprecated.nil? and !img.deprecated.replacement.nil?
               image_id = img.deprecated.replacement
+              img_proj = image_id.gsub(/(?:https?:\/\/.*?\.googleapis\.com\/compute\/.*?\/)?.*?\/?(?:projects\/)?([^\/]+)\/.*/, '\1')
+              img_name = image_id.gsub(/.*?([^\/]+)$/, '\1')
             end
+          rescue ::Google::Apis::ClientError => e
+            # SOME people *cough* don't use deprecation or image family names
+            # and just spew out images with a version appended to the name, so
+            # let's try some crude semantic versioning list.
+            if e.message.match(/^notFound: /) and img_name.match(/-[^\-]+$/)
+              list = MU::Cloud::Google.compute(credentials: credentials).list_images(img_proj, filter: "name eq #{img_name.sub(/-[^\-]+$/, '')}-.*")
+              if list and list.items
+                latest = nil
+                list.items.each { |candidate|
+                  created = DateTime.parse(candidate.creation_timestamp)
+                  if latest.nil? or created > latest
+                    latest = created
+                    img = candidate
+                  end
+                }
+                if latest
+                  MU.log "Mapped #{image_id} to #{img.name} with semantic versioning guesswork", MU::WARN
+                  @@image_id_map[image_id] = img
+                  return @@image_id_map[image_id]
+                end
+              end
+            end
+            raise e # if our little semantic versioning party trick failed
           end while !img.deprecated.nil? and img.deprecated.state == "DEPRECATED" and !img.deprecated.replacement.nil?
-          MU::Cloud::Google.compute(credentials: credentials).get_image(img_proj, img_name)
+          final = MU::Cloud::Google.compute(credentials: credentials).get_image(img_proj, img_name)
+          @@image_id_map[image_id] = final
+          @@image_id_map[image_id]
         end
 
         # Generator for disk configuration parameters for a Compute instance
@@ -230,7 +247,7 @@ next if !create
           end
           subnet = vpc.getSubnet(name: subnet_cfg['subnet_name'], cloud_id: subnet_cfg['subnet_id'])
           if subnet.nil?
-            raise MuError, "Couldn't find subnet details while configuring Server #{config['name']} (VPC: #{vpc.mu_name})"
+            raise MuError, "Couldn't find subnet details for #{subnet_cfg['subnet_name'] || subnet_cfg['subnet_id']} while configuring Server #{config['name']} (VPC: #{vpc.mu_name})"
           end
           base_iface_obj = {
             :network => vpc.url,
@@ -249,15 +266,21 @@ next if !create
 
         # Called automatically by {MU::Deploy#createResources}
         def create
-          @project_id = MU::Cloud::Google.projectLookup(@config['project_id'], @deploy).cloudobj.cloud_id
+          @project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloud_id
+
+          sa = MU::Config::Ref.get(@config['service_account'])
 
-          service_acct = MU::Cloud::Google::Server.createServiceAccount(
-            @mu_name.downcase,
-            @deploy,
-            project: @project_id,
-            credentials: @config['credentials']
+          if !sa or !sa.kitten or !sa.kitten.cloud_desc
+            raise MuError, "Failed to get service account cloud id from #{@config['service_account'].to_s}"
+          end
+
+          @service_acct = MU::Cloud::Google.compute(:ServiceAccount).new(
+            email: sa.kitten.cloud_desc.email,
+            scopes: @config['scopes']
           )
-          MU::Cloud::Google.grantDeploySecretAccess(service_acct.email, credentials: @config['credentials'])
+          if !@config['scrub_mu_isms']
+            MU::Cloud::Google.grantDeploySecretAccess(@service_acct.email, credentials: @config['credentials'])
+          end
 
           begin
             disks = MU::Cloud::Google::Server.diskConfig(@config, credentials: @config['credentials'])
@@ -273,25 +296,36 @@ next if !create
               :name => MU::Cloud::Google.nameStr(@mu_name),
               :can_ip_forward => !@config['src_dst_check'],
               :description => @deploy.deploy_id,
-              :service_accounts => [service_acct],
+              :service_accounts => [@service_acct],
               :network_interfaces => interfaces,
               :machine_type => "zones/"+@config['availability_zone']+"/machineTypes/"+@config['size'],
-              :metadata => {
-                :items => [
-                  {
-                    :key => "ssh-keys",
-                    :value => @config['ssh_user']+":"+@deploy.ssh_public_key
-                  },
-                  {
-                    :key => "startup-script",
-                    :value => @userdata
-                  }
-                ]
-              },
               :tags => MU::Cloud::Google.compute(:Tags).new(items: [MU::Cloud::Google.nameStr(@mu_name)])
             }
             desc[:disks] = disks if disks.size > 0
 
+            metadata = {}
+            if @config['metadata']
+              metadata = Hash[@config['metadata'].map { |m|
+                [m["key"], m["value"]]
+              }]
+            end
+            metadata["startup-script"] = @userdata if @userdata and !@userdata.empty?
+
+            deploykey = @config['ssh_user']+":"+@deploy.ssh_public_key
+            if metadata["ssh-keys"]
+              metadata["ssh-keys"] += "\n"+deploykey
+            else
+              metadata["ssh-keys"] = deploykey
+            end
+            desc[:metadata] = MU::Cloud::Google.compute(:Metadata).new(
+              :items => metadata.keys.map { |k|
+                MU::Cloud::Google.compute(:Metadata)::Item.new(
+                  key: k,
+                  value: metadata[k]
+                )
+              }
+            )
+
             # Tags in GCP means something other than what we think of;
             # labels are the thing you think you mean
             desc[:labels] = {}
@@ -302,10 +336,16 @@ next if !create
             }
             desc[:labels]["name"] = @mu_name.downcase
 
+            if @config['network_tags'] and @config['network_tags'].size > 0
+              desc[:tags] = U::Cloud::Google.compute(:Tags).new(
+                items: @config['network_tags']
+              )
+            end
 
             instanceobj = MU::Cloud::Google.compute(:Instance).new(desc)
 
-            MU.log "Creating instance #{@mu_name}"
+            MU.log "Creating instance #{@mu_name}", MU::NOTICE, details: instanceobj
+
             begin
               instance = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_instance(
                 @project_id,
@@ -351,7 +391,7 @@ next if !create
                 parent_thread_id = Thread.current.object_id
                 Thread.new {
                   MU.dupGlobals(parent_thread_id)
-                  MU::Cloud::Google::Server.cleanup(noop: false, ignoremaster: false, flags: { "skipsnapshots" => true } )
+                  MU::Cloud::Google::Server.cleanup(noop: false, ignoremaster: false, flags: { "skipsnapshots" => true }, region: @config['region'] )
                 }
               end
             end
@@ -467,7 +507,7 @@ next if !create
           return false if !MU::MommaCat.lock(@cloud_id+"-orchestrate", true)
           return false if !MU::MommaCat.lock(@cloud_id+"-groom", true)
 
-#          MU::MommaCat.createStandardTags(@cloud_id, region: @config['region'])
+#          MU::Cloud::AWS.createStandardTags(@cloud_id, region: @config['region'])
 #          MU::MommaCat.createTag(@cloud_id, "Name", node, region: @config['region'])
 #
 #          if @config['optional_tags']
@@ -586,80 +626,82 @@ next if !create
         end #postBoot
 
         # Locate an existing instance or instances and return an array containing matching AWS resource descriptors for those that match.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region
-        # @param tag_key [String]: A tag key to search.
-        # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
-        # @param ip [String]: An IP address associated with the instance
-        # @param flags [Hash]: Optional flags
         # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching instances
-        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, ip: nil, flags: {}, credentials: nil)
-# XXX put that 'ip' value into flags
-          instance = nil
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
-          if !region.nil? and MU::Cloud::Google.listRegions.include?(region)
-            regions = [region]
+        def self.find(**args)
+          args[:project] ||= args[:habitat]
+          args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
+          if !args[:region].nil? and MU::Cloud::Google.listRegions.include?(args[:region])
+            regions = [args[:region]]
           else
             regions = MU::Cloud::Google.listRegions
           end
 
-          found_instances = {}
+          found = {}
           search_semaphore = Mutex.new
           search_threads = []
 
           # If we got an instance id, go get it
-          if !cloud_id.nil? and !cloud_id.empty?
-            parent_thread_id = Thread.current.object_id
-            regions.each { |region|
-              search_threads << Thread.new {
-                Thread.abort_on_exception = false
-                MU.dupGlobals(parent_thread_id)
-                MU.log "Hunting for instance with cloud id '#{cloud_id}' in #{region}", MU::DEBUG
-                MU::Cloud::Google.listAZs(region).each { |az|
-                  resp = nil
-                  begin
-                    resp = MU::Cloud::Google.compute(credentials: credentials).get_instance(
-                      flags["project"],
+          parent_thread_id = Thread.current.object_id
+          regions.each { |r|
+            search_threads << Thread.new(r) { |region|
+              Thread.abort_on_exception = false
+              MU.dupGlobals(parent_thread_id)
+              MU.log "Hunting for instance with cloud id '#{args[:cloud_id]}' in #{region}", MU::DEBUG
+              MU::Cloud::Google.listAZs(region).each { |az|
+                begin
+                  if !args[:cloud_id].nil? and !args[:cloud_id].empty?
+                    resp = MU::Cloud::Google.compute(credentials: args[:credentials]).get_instance(
+                      args[:project],
                       az,
-                      cloud_id
+                      args[:cloud_id]
+                    )
+                    search_semaphore.synchronize {
+                      found[args[:cloud_id]] = resp if !resp.nil?
+                    }
+                  else
+                    resp = MU::Cloud::Google.compute(credentials: args[:credentials]).list_instances(
+                      args[:project],
+                      az
                     )
-                  rescue ::OpenSSL::SSL::SSLError => e
-                    MU.log "Got #{e.message} looking for instance #{cloud_id} in project #{flags["project"]} (#{az}). Usually this means we've tried to query a non-functional region.", MU::DEBUG
-                  rescue ::Google::Apis::ClientError => e
-                    raise e if !e.message.match(/^notFound: /)
+                    if resp and resp.items
+                      resp.items.each { |instance|
+                        search_semaphore.synchronize {
+                          found[instance.name] = instance
+                        }
+                      }
+                    end
                   end
-                  found_instances[cloud_id] = resp if !resp.nil?
-                }
+                rescue ::OpenSSL::SSL::SSLError => e
+                  MU.log "Got #{e.message} looking for instance #{args[:cloud_id]} in project #{args[:project]} (#{az}). Usually this means we've tried to query a non-functional region.", MU::DEBUG
+                rescue ::Google::Apis::ClientError => e
+                  raise e if !e.message.match(/^(?:notFound|forbidden): /)
+                end
               }
             }
-            done_threads = []
-            begin
-              search_threads.each { |t|
-                joined = t.join(2)
-                done_threads << joined if !joined.nil?
-              }
-            end while found_instances.size < 1 and done_threads.size != search_threads.size
-          end
-
-          if found_instances.size > 0
-            return found_instances
-          end
-
+          }
+          done_threads = []
+          begin
+            search_threads.reject! { |t| t.nil? }
+            search_threads.each { |t|
+              joined = t.join(2)
+              done_threads << joined if !joined.nil?
+            }
+          end while found.size < 1 and done_threads.size != search_threads.size
           # Ok, well, let's try looking it up by IP then
-          if instance.nil? and !ip.nil?
-            MU.log "Hunting for instance by IP '#{ip}'", MU::DEBUG
-          end
+#          if instance.nil? and !args[:ip].nil?
+#            MU.log "Hunting for instance by IP '#{args[:ip]}'", MU::DEBUG
+#          end
 
-          if !instance.nil?
-            return {instance.name => instance} if !instance.nil?
-          end
+#          if !instance.nil?
+#            return {instance.name => instance} if !instance.nil?
+#          end
 
           # Fine, let's try it by tag.
-          if !tag_value.nil?
-            MU.log "Searching for instance by tag '#{tag_key}=#{tag_value}'", MU::DEBUG
-          end
+#          if !args[:tag_value].nil?
+#            MU.log "Searching for instance by tag '#{args[:tag_key]}=#{args[:tag_value]}'", MU::DEBUG
+#          end
 
-          return found_instances
+          return found
         end
 
         # Return a description of this resource appropriate for deployment
@@ -724,7 +766,7 @@ next if !create
 
         # Called automatically by {MU::Deploy#createResources}
         def groom
-          @project_id = MU::Cloud::Google.projectLookup(@config['project_id'], @deploy).cloudobj.cloud_id
+          @project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloud_id
 
           MU::MommaCat.lock(@cloud_id+"-groom")
           
@@ -801,12 +843,12 @@ next if !create
                 instance_id: @cloud_id,
                 region: @config['region'],
                 storage: @config['storage'],
-                family: ("mu-"+@config['platform']+"-"+MU.environment).downcase,
                 project: @project_id,
                 exclude_storage: img_cfg['image_exclude_storage'],
                 make_public: img_cfg['public'],
                 tags: @config['tags'],
                 zone: @config['availability_zone'],
+                family: @config['family'],
                 credentials: @config['credentials']
             )
             @deploy.notify("images", @config['name'], {"image_id" => image_id})
@@ -835,7 +877,7 @@ next if !create
         # @param region [String]: The cloud provider region
         # @param tags [Array<String>]: Extra/override tags to apply to the image.
         # @return [String]: The cloud provider identifier of the new machine image.
-        def self.createImage(name: nil, instance_id: nil, storage: {}, exclude_storage: false, project: nil, make_public: false, tags: [], region: nil, family: "mu", zone: MU::Cloud::Google.listAZs.sample, credentials: nil)
+        def self.createImage(name: nil, instance_id: nil, storage: {}, exclude_storage: false, project: nil, make_public: false, tags: [], region: nil, family: nil, zone: MU::Cloud::Google.listAZs.sample, credentials: nil)
           project ||= MU::Cloud::Google.defaultProject(credentials)
           instance = MU::Cloud::Server.find(cloud_id: instance_id, region: region)
           if instance.nil?
@@ -891,46 +933,21 @@ next if !create
           end
 
           labels["name"] = instance_id.downcase
-          imageobj = MU::Cloud::Google.compute(:Image).new(
-            name: name,
-            source_disk: bootdisk,
-            description: "Mu image created from #{name}",
-            labels: labels,
-            family: family
-          )
+          image_desc = {
+            :name => name,
+            :source_disk => bootdisk,
+            :description => "Mu image created from #{name}",
+            :labels => labels
+          }
+          image_desc[:family] = family if family
 
           newimage = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_image(
             project,
-            imageobj
+            MU::Cloud::Google.compute(:Image).new(image_desc)
           )
           newimage.name
         end
 
-#        def cloud_desc
-#          max_retries = 5
-#          retries = 0
-#          if !@cloud_id.nil?
-#            begin
-#              return MU::Cloud::Google.compute(credentials: @config['credentials']).get_instance(
-#                @project_id,
-#                @config['availability_zone'],
-#                @cloud_id
-#              )
-#            rescue ::Google::Apis::ClientError => e
-#              if e.message.match(/^notFound: /)
-#                return nil
-#              else
-#                raise e
-#              end
-#            end
-#          end
-#          nil
-#        end
-
-        def cloud_desc
-          MU::Cloud::Google::Server.find(cloud_id: @cloud_id, credentials: @config['credentials']).values.first
-        end
-
         # Return the IP address that we, the Mu server, should be using to access
         # this host via the network. Note that this does not factor in SSH
         # bastion hosts that may be in the path, see getSSHConfig if that's what
@@ -974,7 +991,8 @@ next if !create
         # @param dev [String]: Device name to use when attaching to instance
         # @param size [String]: Size (in gb) of the new volume
         # @param type [String]: Cloud storage type of the volume, if applicable
-        def addVolume(dev, size, type: "pd-standard")
+        # @param delete_on_termination [Boolean]: Value of delete_on_termination flag to set
+        def addVolume(dev, size, type: "pd-standard", delete_on_termination: false)
           devname = dev.gsub(/.*?\/([^\/]+)$/, '\1')
           resname = MU::Cloud::Google.nameStr(@mu_name+"-"+devname)
           MU.log "Creating disk #{resname}"
@@ -1007,11 +1025,13 @@ next if !create
           end
 
           attachobj = MU::Cloud::Google.compute(:AttachedDisk).new(
-            auto_delete: true,
             device_name: devname,
             source: newdisk.self_link,
-            type: "PERSISTENT"
+            type: "PERSISTENT",
+            auto_delete: delete_on_termination
           )
+
+          MU.log "Attaching disk #{resname} to #{@cloud_id} at #{devname}"
           attachment = MU::Cloud::Google.compute(credentials: @config['credentials']).attach_disk(
             @project_id,
             @config['availability_zone'],
@@ -1028,6 +1048,113 @@ next if !create
           true
         end
 
+        # Reverse-map our cloud description into a runnable config hash.
+        # We assume that any values we have in +@config+ are placeholders, and
+        # calculate our own accordingly based on what's live in the cloud.
+        def toKitten(rootparent: nil, billing: nil, habitats: nil)
+          bok = {
+            "cloud" => "Google",
+            "credentials" => @config['credentials'],
+            "cloud_id" => @cloud_id,
+            "project" => @project_id
+          }
+          if !cloud_desc
+            MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
+            return nil
+          end
+          bok['name'] = cloud_desc.name
+
+          # XXX we can have multiple network interfaces, and often do; need
+          # language to account for this
+          iface = cloud_desc.network_interfaces.first
+          iface.network.match(/(?:^|\/)projects\/(.*?)\/.*?\/networks\/([^\/]+)(?:$|\/)/)
+          vpc_proj = Regexp.last_match[1]
+          vpc_id = Regexp.last_match[2]
+
+          bok['vpc'] = MU::Config::Ref.get(
+            id: vpc_id,
+            cloud: "Google",
+            habitat: MU::Config::Ref.get(
+              id: vpc_proj,
+              cloud: "Google",
+              credentials: @credentials,
+              type: "habitats"
+            ),
+            credentials: @credentials,
+            type: "vpcs",
+            subnet_id: iface.subnetwork.sub(/.*?\/([^\/]+)$/, '\1')
+          )
+
+          cloud_desc.disks.each { |disk|
+            next if !disk.source
+            disk.source.match(/\/projects\/([^\/]+)\/zones\/([^\/]+)\/disks\/(.*)/)
+            proj = Regexp.last_match[1]
+            az = Regexp.last_match[2]
+            name = Regexp.last_match[3]
+            begin
+              disk_desc = MU::Cloud::Google.compute(credentials: @credentials).get_disk(proj, az, name)
+              if disk_desc.source_image and disk.boot
+                bok['image_id'] ||= disk_desc.source_image.sub(/^https:\/\/www\.googleapis\.com\/compute\/[^\/]+\//, '')
+              else
+                bok['storage'] ||= []
+                storage_blob = {
+                  "size" => disk_desc.size_gb,
+                  "device" => "/dev/xvd"+(disk.index+97).chr.downcase
+                }
+                bok['storage'] <<  storage_blob
+              end
+            rescue ::Google::Apis::ClientError => e
+              MU.log "Failed to retrieve disk #{name} attached to server #{@cloud_id} in #{proj}/#{az}", MU::WARN, details: e.message
+              next
+            end
+            
+          }
+
+          if cloud_desc.labels
+            bok['tags'] = cloud_desc.labels.keys.map { |k| { "key" => k, "value" => cloud_desc.labels[k] } }
+          end
+          if cloud_desc.tags and cloud_desc.tags.items and cloud_desc.tags.items.size > 0
+            bok['network_tags'] = cloud_desc.tags.items
+          end
+          bok['src_dst_check'] = !cloud_desc.can_ip_forward
+          bok['size'] = cloud_desc.machine_type.sub(/.*?\/([^\/]+)$/, '\1')
+          bok['project'] = @project_id
+          if cloud_desc.service_accounts
+            bok['scopes'] = cloud_desc.service_accounts.map { |sa| sa.scopes }.flatten.uniq
+          end
+          if cloud_desc.metadata and cloud_desc.metadata.items
+            bok['metadata'] = cloud_desc.metadata.items.map { |m| MU.structToHash(m) }
+          end
+
+          # Skip nodes that are just members of GKE clusters
+          if bok['name'].match(/^gke-.*?-[a-f0-9]+-[a-z0-9]+$/) and
+             bok['image_id'].match(/(:?^|\/)projects\/gke-node-images\//)
+            found_gke_tag = false
+            bok['network_tags'].each { |tag|
+              if tag.match(/^gke-/)
+                found_gke_tag = true
+                break
+              end
+            }
+            if found_gke_tag
+              MU.log "Server #{bok['name']} appears to belong to a ContainerCluster, skipping adoption", MU::DEBUG
+              return nil
+            end
+          end
+
+          if bok['metadata']
+            bok['metadata'].each { |item|
+              if item[:key] == "created-by" and item[:value].match(/\/instanceGroupManagers\//)
+                MU.log "Server #{bok['name']} appears to belong to a ServerPool, skipping adoption", MU::DEBUG, details: item[:value]
+                return nil
+              end
+            }
+          end
+
+
+          bok
+        end
+
         # Does this resource type exist as a global (cloud-wide) artifact, or
         # is it localized to a region/zone?
         # @return [Boolean]
@@ -1048,6 +1175,7 @@ next if !create
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
+          return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
           skipsnapshots = flags["skipsnapshots"]
           onlycloud = flags["onlycloud"]
 # XXX make damn sure MU.deploy_id is set
@@ -1105,51 +1233,140 @@ next if !create
         def self.schema(config)
           toplevel_required = []
           schema = {
-            "image_id" => {
+            "roles" => MU::Cloud::Google::User.schema(config)[1]["roles"],
+            "create_image" => {
+              "properties" => {
+                "family" => {
+                  "type" => "string",
+                  "description" => "Add a GCP image +family+ string to the created image(s)"
+                }
+              }
+            },
+            "availability_zone" => {
+              "type" => "string",
+              "description" => "Target this instance to a specific Availability Zone"
+            },
+            "ssh_user" => {
               "type" => "string",
-              "description" => "The Google Cloud Platform Image on which to base this instance. Will use the default appropriate for the platform, if not specified."
+              "description" => "Account to use when connecting via ssh. Google Cloud images don't come with predefined remote access users, and some don't work with our usual default of +root+, so we recommend using some other (non-root) username.",
+              "default" => "muadmin"
+            },
+            "network_tags" => {
+              "type" => "array",
+              "items" => {
+                "type" => "string",
+                "description" => "Add a network tag to this host, which can be used to selectively apply routes or firewall rules."
+              }
+            },
+            "service_account" => MU::Config::Ref.schema(
+              type: "users",
+              desc: "An existing service account to use instead of the default one generated by Mu during the deployment process."
+            ),
+            "metadata" => {
+              "type" => "array",
+              "items" => {
+                "type" => "object",
+                "description" => "Custom key-value pairs to be added to the metadata of Google Cloud virtual machines",
+                "required" => ["key", "value"],
+                "properties" => {
+                  "key" => {
+                    "type" => "string"
+                  },
+                  "value" => {
+                    "type" => "string"
+                  }
+                }
+              }
             },
             "routes" => {
               "type" => "array",
               "items" => MU::Config::VPC.routeschema
+            },
+            "scopes" => {
+              "type" => "array",
+              "items" => {
+                "type" => "string",
+                "description" => "API scopes to make available to this resource's service account."
+              },
+              "default" => ["https://www.googleapis.com/auth/compute.readonly", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/monitoring.write", "https://www.googleapis.com/auth/devstorage.read_only"]
             }
           }
           [toplevel_required, schema]
         end
 
+        @@instance_type_cache = {}
+
         # Confirm that the given instance size is valid for the given region.
         # If someone accidentally specified an equivalent size from some other cloud provider, return something that makes sense. If nothing makes sense, return nil.
         # @param size [String]: Instance type to check
         # @param region [String]: Region to check against
         # @return [String,nil]
-        def self.validateInstanceType(size, region)
-          types = (MU::Cloud::Google.listInstanceTypes(region))[region]
-          if types and (size.nil? or !types.has_key?(size))
+        def self.validateInstanceType(size, region, project: nil, credentials: nil)
+          size = size.dup.to_s
+          if @@instance_type_cache[project] and
+             @@instance_type_cache[project][region] and
+             @@instance_type_cache[project][region][size]
+            return @@instance_type_cache[project][region][size]
+          end
+
+          if size.match(/\/?custom-(\d+)-(\d+)(?:-ext)?$/)
+            cpus = Regexp.last_match[1].to_i
+            mem = Regexp.last_match[2].to_i
+            ok = true
+            if cpus < 1 or cpus > 32 or (cpus % 2 != 0 and cpus != 1)
+              MU.log "Custom instance type #{size} illegal: CPU count must be 1 or an even number between 2 and 32", MU::ERR
+              ok = false
+            end
+            if (mem % 256) != 0
+              MU.log "Custom instance type #{size} illegal: Memory must be a multiple of 256 (MB)", MU::ERR
+              ok = false
+            end
+            if ok
+              return "custom-#{cpus.to_s}-#{mem.to_s}"
+            else
+              return nil
+            end
+          end
+
+          @@instance_type_cache[project] ||= {}
+          @@instance_type_cache[project][region] ||= {}
+          types = (MU::Cloud::Google.listInstanceTypes(region, project: project, credentials: credentials))[project][region]
+          realsize = size.dup
+
+          if types and (realsize.nil? or !types.has_key?(realsize))
             # See if it's a type we can approximate from one of the other clouds
-            atypes = (MU::Cloud::AWS.listInstanceTypes)[MU::Cloud::AWS.myRegion]
             foundmatch = false
-            if atypes and atypes.size > 0 and atypes.has_key?(size)
-              vcpu = atypes[size]["vcpu"]
-              mem = atypes[size]["memory"]
-              ecu = atypes[size]["ecu"]
-              types.keys.sort.reverse.each { |type|
-                features = types[type]
-                next if ecu == "Variable" and ecu != features["ecu"]
-                next if features["vcpu"] != vcpu
-                if (features["memory"] - mem.to_f).abs < 0.10*mem
-                  foundmatch = true
-                  MU.log "You specified an Amazon instance type '#{size}.' Approximating with Google Compute type '#{type}.'", MU::WARN
-                  size = type
-                  break
-                end
-              }
-            end
+            MU::Cloud.availableClouds.each { |cloud|
+              next if cloud == "Google"
+              cloudbase = Object.const_get("MU").const_get("Cloud").const_get(cloud)
+              foreign_types = (cloudbase.listInstanceTypes)[cloudbase.myRegion]
+              if foreign_types and foreign_types.size > 0 and foreign_types.has_key?(size)
+                vcpu = foreign_types[size]["vcpu"]
+                mem = foreign_types[size]["memory"]
+                ecu = foreign_types[size]["ecu"]
+                types.keys.sort.reverse.each { |type|
+                  features = types[type]
+                  next if ecu == "Variable" and ecu != features["ecu"]
+                  next if features["vcpu"] != vcpu
+                  if (features["memory"] - mem.to_f).abs < 0.10*mem
+                    foundmatch = true
+                    MU.log "You specified #{cloud} instance type '#{realsize}.' Approximating with Google Compute type '#{type}.'", MU::WARN
+                    realsize = type
+                    break
+                  end
+                }
+              end
+              break if foundmatch
+            }
+
             if !foundmatch
-              MU.log "Invalid size '#{size}' for Google Compute instance in #{region}. Supported types:", MU::ERR, details: types.keys.sort.join(", ")
+              MU.log "Invalid size '#{realsize}' for Google Compute instance in #{region} (checked project #{project}). Supported types:", MU::ERR, details: types.keys.sort.join(", ")
+              @@instance_type_cache[project][region][size] = nil
               return nil
             end
           end
-          size
+          @@instance_type_cache[project][region][size] = realsize
+          @@instance_type_cache[project][region][size]
         end
 
 
@@ -1160,17 +1377,57 @@ next if !create
         def self.validateConfig(server, configurator)
           ok = true
 
-          server['size'] = validateInstanceType(server["size"], server["region"])
-          ok = false if server['size'].nil?
+          server['project'] ||= MU::Cloud::Google.defaultProject(server['credentials'])
+          size = validateInstanceType(server["size"], server["region"], project: server['project'], credentials: server['credentials'])
+
+          if size.nil?
+            MU.log "Failed to verify instance size #{server["size"]} for Server #{server['name']}", MU::WARN
+          else
+            server["size"] = size
+          end
 
           # If we're not targeting an availability zone, pick one randomly
           if !server['availability_zone']
             server['availability_zone'] = MU::Cloud::Google.listAZs(server['region']).sample
           end
 
+          if server['service_account']
+            server['service_account']['cloud'] = "Google"
+            server['service_account']['habitat'] ||= server['project']
+            found = MU::Config::Ref.get(server['service_account'])
+            if found.id and !found.kitten
+              MU.log "GKE server #{server['name']} failed to locate service account #{server['service_account']} in project #{server['project']}", MU::ERR
+              ok = false
+            end
+          else
+            user = {
+              "name" => server['name'],
+              "cloud" => "Google",
+              "project" => server["project"],
+              "credentials" => server["credentials"],
+              "type" => "service"
+            }
+            if server['roles']
+              user['roles'] = server['roles'].dup
+            end
+            configurator.insertKitten(user, "users", true)
+            server['dependencies'] ||= []
+            server['service_account'] = MU::Config::Ref.get(
+              type: "users",
+              cloud: "Google",
+              name: server["name"],
+              project: server["project"],
+              credentials: server["credentials"]
+            )
+            server['dependencies'] << {
+              "type" => "user",
+              "name" => server["name"]
+            }
+          end
+
           subnets = nil
           if !server['vpc']
-            vpcs = MU::Cloud::Google::VPC.find
+            vpcs = MU::Cloud::Google::VPC.find(credentials: server['credentials'])
             if vpcs["default"]
               server["vpc"] ||= {}
               server["vpc"]["vpc_id"] = vpcs["default"].self_link
@@ -1202,8 +1459,9 @@ next if !create
           end
 
           if server['image_id'].nil?
-            if MU::Config.google_images.has_key?(server['platform'])
-              server['image_id'] = configurator.getTail("server"+server['name']+"Image", value: MU::Config.google_images[server['platform']], prettyname: "server"+server['name']+"Image", cloudtype: "Google::::Apis::ComputeBeta::Image")
+            img_id = MU::Cloud.getStockImage("Google", platform: server['platform'])
+            if img_id
+              server['image_id'] = configurator.getTail("server"+server['name']+"Image", value: img_id, prettyname: "server"+server['name']+"Image", cloudtype: "Google::Apis::ComputeV1::Image")
             else
               MU.log "No image specified for #{server['name']} and no default available for platform #{server['platform']}", MU::ERR, details: server
               ok = false
@@ -1214,7 +1472,6 @@ next if !create
           begin
             real_image = MU::Cloud::Google::Server.fetchImage(server['image_id'].to_s, credentials: server['credentials'])
           rescue ::Google::Apis::ClientError => e
-            MU.log e.inspect, MU::WARN
           end
 
           if real_image.nil?
@@ -1226,31 +1483,38 @@ next if !create
             img_project = Regexp.last_match[1]
             img_name = Regexp.last_match[2]
             begin
+              img = MU::Cloud::Google.compute(credentials: server['credentials']).get_image(img_project, img_name)
               snaps = MU::Cloud::Google.compute(credentials: server['credentials']).list_snapshots(
                 img_project,
                 filter: "name eq #{img_name}-.*"
               )
               server['storage'] ||= []
               used_devs = server['storage'].map { |disk| disk['device'].gsub(/.*?\//, "") }
-              snaps.items.each { |snap|
-                next if !snap.labels.is_a?(Hash) or !snap.labels["mu-device-name"] or snap.labels["mu-parent-image"] != img_name
-                devname = snap.labels["mu-device-name"]
-
-                if used_devs.include?(devname)
-                  MU.log "Device name #{devname} already declared in server #{server['name']} (snapshot #{snap.name} wants the name)", MU::ERR
-                  ok = false
-                end
-                server['storage'] << {
-                  "snapshot_id" => snap.self_link,
-                  "size" => snap.disk_size_gb,
-                  "delete_on_termination" => true, 
-                  "device" => devname
+              if snaps and snaps.items
+                snaps.items.each { |snap|
+                  next if !snap.labels.is_a?(Hash) or !snap.labels["mu-device-name"] or snap.labels["mu-parent-image"] != img_name
+                  devname = snap.labels["mu-device-name"]
+
+                  if used_devs.include?(devname)
+                    MU.log "Device name #{devname} already declared in server #{server['name']} (snapshot #{snap.name} wants the name)", MU::ERR
+                    ok = false
+                  end
+                  server['storage'] << {
+                    "snapshot_id" => snap.self_link,
+                    "size" => snap.disk_size_gb,
+                    "delete_on_termination" => true, 
+                    "device" => devname
+                  }
+                  used_devs << devname
                 }
-                used_devs << devname
-              }
+                if snaps.items.size > 0
+#                  MU.log img_name, MU::WARN, details: snaps.items
+                end
+              end
             rescue ::Google::Apis::ClientError => e
               # it's ok, sometimes we don't have permission to list snapshots
               # in other peoples' projects
+#              MU.log img_name, MU::WARN, details: img
               raise e if !e.message.match(/^forbidden: /)
             end
           end